From c22484eee86cb964ba7ecdef345fdfa1115914d9 Mon Sep 17 00:00:00 2001 From: Sancarn Date: Sun, 1 Nov 2026 18:57:39 +0000 Subject: [PATCH 1/2] Unify test tooling and centralize test assets --- Tools/Testing/README.md | 23 + Tools/Testing/office.py | 263 +++++++ Tools/Testing/run.py | 192 ++++++ Tools/stdDate-test/README.md | 5 - Tools/stdDate-test/run-tests.ps1 | 38 - Tools/stdRegex-build/benchmark.ps1 | 86 +-- Tools/stdRegex-build/run-tests.ps1 | 44 +- Tools/stdRegex-build/summarize-benchmark.py | 49 +- Tools/stdRegex-build/test_build.py | 100 +-- Tools/stdRegex-thunk/README.md | 2 +- Tools/stdRegex-thunk/run.py | 194 +----- Tools/stdRegex-thunk/summarize.py | 45 +- Tools/stdRegex-thunk/test-native.py | 24 +- Tools/stdWindow-subclass/run.py | 377 +--------- Tools/stdWindow-subclass/test_thunk.py | 139 +--- changelog.md | 3 + docs.json | 38 +- src/WIP/stdWebSocket/README.md | 2 +- src/WIP/stdWebSocket/native/build.ps1 | 2 +- src/stdLambda.cls | 232 ------- tests/README.md | 24 + .../harnesses}/stdCallback-tests/Probe.bas | 0 .../harnesses}/stdCallback-tests/README.md | 2 +- .../harnesses}/stdCallback-tests/run.py | 4 +- tests/harnesses/stdDate-test/README.md | 5 + .../harnesses}/stdDate-test/TestRunner.bas | 0 tests/harnesses/stdDate-test/run-tests.ps1 | 2 + .../harnesses}/stdEnumerator-tests/Probe.bas | 0 .../harnesses}/stdEnumerator-tests/README.md | 2 +- .../harnesses}/stdEnumerator-tests/Test.cls | 0 .../harnesses}/stdEnumerator-tests/run.py | 4 +- .../harnesses}/stdHTTP-tests/fixture.py | 0 .../harnesses}/stdHTTP-tests/run.ps1 | 0 .../harnesses}/stdHTTP-tests/run.py | 2 +- .../harnesses}/stdHTTP-tests/worker.ps1 | 2 +- .../harnesses}/stdImage-tests/README.md | 4 +- .../harnesses}/stdImage-tests/run.py | 2 +- .../harnesses}/stdRegex-build/Benchmark.bas | 0 .../stdRegex-build/TestReporter.cls | 0 .../harnesses}/stdRegex-build/TestRunner.bas | 0 tests/harnesses/stdRegex-build/benchmark.ps1 | 84 +++ tests/harnesses/stdRegex-build/run-tests.ps1 | 2 + .../stdRegex-build/summarize-benchmark.py | 48 ++ .../harnesses}/stdRegex-thunk/Benchmark.bas | 0 tests/harnesses/stdRegex-thunk/run.py | 193 ++++++ tests/harnesses/stdRegex-thunk/summarize.py | 44 ++ .../harnesses}/stdRegex-thunk/test-native.c | 2 +- tests/harnesses/stdRegex-thunk/test-native.py | 23 + .../harnesses}/stdWebSocket-tests/README.md | 4 +- .../WebSocketAsyncRecorder.cls | 0 .../stdWebSocket-tests/WebSocketRecorder.cls | 0 .../harnesses}/stdWebSocket-tests/fixture.py | 0 .../stdWebSocket-tests/native_tests.c | 0 .../harnesses}/stdWebSocket-tests/run.py | 4 +- .../harnesses}/stdWebSocket-tests/worker.ps1 | 4 +- .../stdWindow-subclass/IDEProbe.bas | 0 tests/harnesses/stdWindow-subclass/run.py | 376 ++++++++++ tests/manifest.json | 647 ++++++++++++++++++ tests/native/test_window_thunk.py | 139 ++++ tests/stdCallbackTests.bas | 13 + tests/stdImageTests.bas | 2 +- tests/stdRegexTests.bas | 16 +- tests/support/Test.cls | 36 + tests/support/stdLambdaVM.cls.inc | 229 +++++++ tests/tooling/test_office_runner.py | 43 ++ tests/tooling/test_regex_build.py | 102 +++ tests/tooling/test_runner.py | 51 ++ 67 files changed, 2598 insertions(+), 1375 deletions(-) create mode 100644 Tools/Testing/README.md create mode 100644 Tools/Testing/office.py create mode 100644 Tools/Testing/run.py delete mode 100644 Tools/stdDate-test/README.md delete mode 100644 Tools/stdDate-test/run-tests.ps1 create mode 100644 tests/README.md rename {Tools => tests/harnesses}/stdCallback-tests/Probe.bas (100%) rename {Tools => tests/harnesses}/stdCallback-tests/README.md (95%) rename {Tools => tests/harnesses}/stdCallback-tests/run.py (97%) create mode 100644 tests/harnesses/stdDate-test/README.md rename {Tools => tests/harnesses}/stdDate-test/TestRunner.bas (100%) create mode 100644 tests/harnesses/stdDate-test/run-tests.ps1 rename {Tools => tests/harnesses}/stdEnumerator-tests/Probe.bas (100%) rename {Tools => tests/harnesses}/stdEnumerator-tests/README.md (94%) rename {Tools => tests/harnesses}/stdEnumerator-tests/Test.cls (100%) rename {Tools => tests/harnesses}/stdEnumerator-tests/run.py (98%) rename {Tools => tests/harnesses}/stdHTTP-tests/fixture.py (100%) rename {Tools => tests/harnesses}/stdHTTP-tests/run.ps1 (100%) rename {Tools => tests/harnesses}/stdHTTP-tests/run.py (98%) rename {Tools => tests/harnesses}/stdHTTP-tests/worker.ps1 (97%) rename {Tools => tests/harnesses}/stdImage-tests/README.md (96%) rename {Tools => tests/harnesses}/stdImage-tests/run.py (99%) rename {Tools => tests/harnesses}/stdRegex-build/Benchmark.bas (100%) rename {Tools => tests/harnesses}/stdRegex-build/TestReporter.cls (100%) rename {Tools => tests/harnesses}/stdRegex-build/TestRunner.bas (100%) create mode 100644 tests/harnesses/stdRegex-build/benchmark.ps1 create mode 100644 tests/harnesses/stdRegex-build/run-tests.ps1 create mode 100644 tests/harnesses/stdRegex-build/summarize-benchmark.py rename {Tools => tests/harnesses}/stdRegex-thunk/Benchmark.bas (100%) create mode 100644 tests/harnesses/stdRegex-thunk/run.py create mode 100644 tests/harnesses/stdRegex-thunk/summarize.py rename {Tools => tests/harnesses}/stdRegex-thunk/test-native.c (98%) create mode 100644 tests/harnesses/stdRegex-thunk/test-native.py rename {Tools => tests/harnesses}/stdWebSocket-tests/README.md (97%) rename {Tools => tests/harnesses}/stdWebSocket-tests/WebSocketAsyncRecorder.cls (100%) rename {Tools => tests/harnesses}/stdWebSocket-tests/WebSocketRecorder.cls (100%) rename {Tools => tests/harnesses}/stdWebSocket-tests/fixture.py (100%) rename {Tools => tests/harnesses}/stdWebSocket-tests/native_tests.c (100%) rename {Tools => tests/harnesses}/stdWebSocket-tests/run.py (95%) rename {Tools => tests/harnesses}/stdWebSocket-tests/worker.ps1 (93%) rename {Tools => tests/harnesses}/stdWindow-subclass/IDEProbe.bas (100%) create mode 100644 tests/harnesses/stdWindow-subclass/run.py create mode 100644 tests/manifest.json create mode 100644 tests/native/test_window_thunk.py create mode 100644 tests/support/Test.cls create mode 100644 tests/support/stdLambdaVM.cls.inc create mode 100644 tests/tooling/test_office_runner.py create mode 100644 tests/tooling/test_regex_build.py create mode 100644 tests/tooling/test_runner.py diff --git a/Tools/Testing/README.md b/Tools/Testing/README.md new file mode 100644 index 00000000..beab3d0c --- /dev/null +++ b/Tools/Testing/README.md @@ -0,0 +1,23 @@ +# Shared test runner + +Run commands from the repository root. Python 3.9+ is needed for discovery and tooling tests. Office suites require Windows, desktop Office and pywin32 (`python -m pip install pywin32`). Image tests additionally require Pillow; native suites list their prerequisites in their own harness documentation. + +```powershell +python Tools/Testing/run.py --list +python Tools/Testing/run.py +python Tools/Testing/run.py --all --json test-results.json +python Tools/Testing/run.py --suite stdCallback +python Tools/Testing/run.py --suite stdCallback --test Bind +python Tools/Testing/run.py --suite stdRegex --test MatchAllCol --host Word +python Tools/Testing/run.py --suite stdLambda.vm --test Instructions +python Tools/Testing/run.py --suite tooling.regexBuild +python Tools/Testing/run.py --suite native.regexThunk --clang C:/path/to/llvm-mingw/bin/clang.exe +``` + +Suite and case IDs are case insensitive. Repeat `--suite` to choose multiple suites. `--test` selects an exact case ID, requires a suite, and fails on an unknown name. `--list --suite stdRegex` shows the available cases. Selection is by independently callable case, rather than filtering assertion output after executing an entire suite. + +The default set runs unattended regressions. Legacy `Test.FullTest` branches are disabled unless `--full` is supplied; those branches can require UI or lengthy measurements. `--all` selects suites and does not implicitly enable these legacy branches. `--all` selects every registered suite, including UI, external-network, clipboard, native-call and experimental tests. Read `--list` requirements before using it. Host mismatches and unavailable legacy fixtures are explicitly reported as skipped. An explicitly selected skipped suite exits unsuccessfully; an aggregate run fails for any failure or if nothing passed. JSON output records each case, host, duration, status and available assertion details. Missing dependencies, imports, compile errors, dialogs, timeouts and crashes fail a case; remaining cases continue. + +Each ordinary VBA case runs in a fresh disposable Office project. The supervisor records the real host/VBE versions, checks VBA project access, respects policy, temporarily adjusts AccessVBOM only if needed, selects Break on Unhandled Errors and restores the exact original registry values/types/absence in cleanup. It watches only owned Office windows for dialogs and bounds each worker to 90 seconds. Shared runner invocations refuse overlapping Office settings transactions. Specialized adapters retain their own process isolation and fixtures, within the shared settings transaction; their aggregate execution is bounded to ten minutes. Do not terminate the supervisor forcibly while it is restoring settings. + +The former `Tools/*-tests` folders have moved under `tests/harnesses`; use this shared command instead. The Regex test command retains its generated-source check before calling the shared runner. Native builders remain build tools; regression test bodies live under `tests/`. diff --git a/Tools/Testing/office.py b/Tools/Testing/office.py new file mode 100644 index 00000000..bd0eaf06 --- /dev/null +++ b/Tools/Testing/office.py @@ -0,0 +1,263 @@ +"""Shared bounded Office lifecycle with owned-process monitoring and exact registry restoration.""" +import ctypes +from ctypes import wintypes +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import threading +import time +import winreg + +sys.dont_write_bytecode = True +ROOT = Path(__file__).resolve().parents[2] +USER = ctypes.WinDLL("user32", use_last_error=True) +USER.GetWindowThreadProcessId.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.DWORD)] +USER.ShowWindow.argtypes = [wintypes.HWND, ctypes.c_int] +USER.PostMessageW.argtypes = [wintypes.HWND, wintypes.UINT, wintypes.WPARAM, wintypes.LPARAM] +USER.GetDlgCtrlID.argtypes = [wintypes.HWND] +USER.GetDlgCtrlID.restype = ctypes.c_int +USER.GetDlgItem.argtypes = [wintypes.HWND, ctypes.c_int] +USER.GetDlgItem.restype = wintypes.HWND +USER.GetClassNameW.argtypes = [wintypes.HWND, wintypes.LPWSTR, ctypes.c_int] +USER.GetWindowTextW.argtypes = [wintypes.HWND, wintypes.LPWSTR, ctypes.c_int] +CALLBACK = ctypes.WINFUNCTYPE(wintypes.BOOL, wintypes.HWND, wintypes.LPARAM) +USER.EnumWindows.argtypes = [CALLBACK, wintypes.LPARAM] +USER.EnumChildWindows.argtypes = [wintypes.HWND, CALLBACK, wintypes.LPARAM] + + +def read_value(path, name): + try: + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, path) as key: + return winreg.QueryValueEx(key, name) + except FileNotFoundError: + return None + + +def restore_value(path, name, original): + with winreg.CreateKey(winreg.HKEY_CURRENT_USER, path) as key: + if original is None: + try: + winreg.DeleteValue(key, name) + except FileNotFoundError: + pass + else: + winreg.SetValueEx(key, name, 0, original[1], original[0]) + if read_value(path, name) != original: + raise RuntimeError("Could not restore " + path + "\\" + name) + + +def worker(host, scratch, phase, url): + import pythoncom + import win32com.client + pythoncom.CoInitialize() + import win32process + existing_pids = set(win32process.EnumProcesses()) + app = doc = None + pidpath = Path(scratch) / "office.pid" + try: + app = win32com.client.DispatchEx(host + ".Application") + pid = wintypes.DWORD() + if host == "Word": + # Word exposes a document window handle without requiring VBE trust. + doc = app.Documents.Add(Visible=False) + hwnd = app.Hwnd if host == "Excel" else doc.ActiveWindow.Hwnd + USER.GetWindowThreadProcessId(int(hwnd), ctypes.byref(pid)) + if pid.value in existing_pids: + app = None + raise RuntimeError("Office returned an existing process; refusing to use it") + pidpath.write_bytes(str(pid.value).encode("ascii")) + app.Visible = False + app.DisplayAlerts = 0 + if host == "Excel": + doc = app.Workbooks.Add() + if phase == "probe": + try: + project = doc.VBProject + access = project.VBComponents.Count >= 0 + except Exception: + access = False + info = {"office": str(app.Version), "vbe": str(app.VBE.Version) if access else None, "access": access} + (Path(scratch) / "probe.json").write_bytes(json.dumps(info).encode("utf-8")) + else: + config = json.loads(os.environ["STDVBA_TEST_CONFIG"]) + app.DisplayAlerts = config.get("display_alerts", 0) + project = doc.VBProject + for relative in config["files"]: + path = ROOT / relative + raw = path.read_bytes() + if b"\n" in raw.replace(b"\r\n", b"") or b"\r" in raw.replace(b"\r\n", b""): + raise RuntimeError("CRLF required before import: " + relative) + project.VBComponents.Import(str(path)) + macro = "UnifiedTestRunner.RunCase" + if host == "Excel": + macro = "'" + doc.Name + "'!" + macro + result = app.Run(macro) + (Path(scratch) / "result.json").write_bytes(str(result).encode("utf-8")) + finally: + try: + if doc is not None: + doc.Close(False if host == "Excel" else 0) + finally: + if app is not None: + app.Quit() + doc = app = None + pythoncom.CoUninitialize() + + +def bounded_worker(host, scratch, phase, url): + pidpath = Path(scratch) / "office.pid" + if pidpath.exists(): + pidpath.unlink() + environment = dict(os.environ) + process = subprocess.Popen([sys.executable, "-B", __file__, "--worker", host, scratch, phase, url], env=environment) + owned = None + deadline = time.monotonic() + 90 + dialogs = [] + try: + while process.poll() is None: + if pidpath.exists(): + text = pidpath.read_text() + if text: + owned = int(text) + + @CALLBACK + def inspect(hwnd, unused): + pid = wintypes.DWORD() + USER.GetWindowThreadProcessId(hwnd, ctypes.byref(pid)) + if owned and pid.value == owned: + kind = ctypes.create_unicode_buffer(256) + USER.GetClassNameW(hwnd, kind, len(kind)) + if kind.value == "#32770": + parts = [] + + @CALLBACK + def capture(child, ignored): + label = ctypes.create_unicode_buffer(1024) + USER.GetWindowTextW(child, label, len(label)) + if label.value: + parts.append(label.value) + if label.value.replace("&", "") == "OK": + control_id = USER.GetDlgCtrlID(child) + USER.PostMessageW(hwnd, 0x0111, control_id, child) + USER.PostMessageW(child, 0x00F5, 0, 0) + return True + + USER.EnumChildWindows(hwnd, capture, 0) + dialogs.append(" | ".join(parts)) + print("Detected and dismissed test dialog: " + dialogs[-1], flush=True) + # Dismiss only this test instance's dialog; never send global keys. + button = USER.GetDlgItem(hwnd, 1) + if button: + USER.PostMessageW(hwnd, 0x0111, 1, button) + USER.PostMessageW(button, 0x00F5, 0, 0) + USER.PostMessageW(hwnd, 0x0100, 13, 0) + USER.PostMessageW(hwnd, 0x0101, 13, 0) + else: + USER.PostMessageW(hwnd, 0x0010, 0, 0) + USER.ShowWindow(hwnd, 6) + return True + + USER.EnumWindows(inspect, 0) + if dialogs: + raise RuntimeError("Office dialog (failed check): " + repr(dialogs)) + if time.monotonic() > deadline: + raise RuntimeError("Office worker timed out; dialogs=" + repr(dialogs)) + time.sleep(0.05) + if dialogs: + raise RuntimeError("Office dialog (failed check): " + repr(dialogs)) + if process.returncode: + raise RuntimeError(host + " worker failed") + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + if owned: + # Only the PID reported by our DispatchEx-created instance. + subprocess.run(["taskkill", "/PID", str(owned), "/F"], capture_output=True) + + + +def _run(config, host, scratch): + os.environ["STDVBA_TEST_CONFIG"] = json.dumps(config) + bounded_worker(host, scratch, "probe", "") + info = json.loads((Path(scratch) / "probe.json").read_bytes()) + version = info["office"] + access_path = r"Software\Microsoft\Office" + "\\" + version + "\\" + host + r"\Security" + originals = {} + try: + if not info["access"]: + # Respect either user or machine managed policy. + for hive in (winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE): + policy = r"Software\Policies\Microsoft\Office" + "\\" + version + "\\" + host + r"\Security" + try: + with winreg.OpenKey(hive, policy) as key: + if winreg.QueryValueEx(key, "AccessVBOM")[0] == 0: + raise RuntimeError("Managed policy disables VBA project access for " + host) + except FileNotFoundError: + pass + originals[(access_path, "AccessVBOM")] = read_value(access_path, "AccessVBOM") + with winreg.CreateKey(winreg.HKEY_CURRENT_USER, access_path) as key: + winreg.SetValueEx(key, "AccessVBOM", 0, winreg.REG_DWORD, 1) + bounded_worker(host, scratch, "probe", "") + info = json.loads((Path(scratch) / "probe.json").read_bytes()) + if not info["access"]: + raise RuntimeError("VBA project access remains unavailable for " + host) + trap_path = r"Software\Microsoft\VBA" + "\\" + ".".join(str(int(part)) for part in info["vbe"].split(".")) + r"\Common" + for name in ("BreakOnAllErrors", "BreakOnServerErrors"): + originals[(trap_path, name)] = read_value(trap_path, name) + print(host + " original settings: " + repr(originals), flush=True) + for name in ("BreakOnAllErrors", "BreakOnServerErrors"): + with winreg.CreateKey(winreg.HKEY_CURRENT_USER, trap_path) as key: + winreg.SetValueEx(key, name, 0, winreg.REG_DWORD, 0) + if config["kind"] == "adapter": + command = ROOT / config["command"] + args = [sys.executable, "-B", str(command)] if command.suffix == ".py" else ["powershell", "-NoProfile", "-File", str(command)] + if config["id"] == "stdImage": + args += ["--hosts", host] + subprocess.run(args, cwd=ROOT, check=True, timeout=600) + else: + bounded_worker(host, scratch, "tests", "") + finally: + errors = [] + for (path, name), original in reversed(list(originals.items())): + try: + restore_value(path, name, original) + except Exception as error: + errors.append(str(error)) + if errors: + raise RuntimeError("Settings restoration failed: " + "; ".join(errors)) + print(host + ": exact registry values restored", flush=True) + + +def run(config, host, scratch): + # Registry settings are shared across Office instances. Refuse overlapping + # runner transactions; Windows releases the byte lock if this process exits. + import msvcrt + lock_path = Path(tempfile.gettempdir()) / "stdVBA-office-tests.lock" + with lock_path.open("a+b") as lock: + lock.seek(0, 2) + if lock.tell() == 0: + lock.write(b"0") + lock.flush() + lock.seek(0) + try: + msvcrt.locking(lock.fileno(), msvcrt.LK_NBLCK, 1) + except OSError as error: + raise RuntimeError("Another stdVBA Office test transaction is running") from error + original = os.environ.get("STDVBA_TEST_CONFIG") + try: + _run(config, host, scratch) + finally: + if original is None: + os.environ.pop("STDVBA_TEST_CONFIG", None) + else: + os.environ["STDVBA_TEST_CONFIG"] = original + lock.seek(0) + msvcrt.locking(lock.fileno(), msvcrt.LK_UNLCK, 1) + + +if __name__ == "__main__": + worker(*sys.argv[2:]) diff --git a/Tools/Testing/run.py b/Tools/Testing/run.py new file mode 100644 index 00000000..e4906d68 --- /dev/null +++ b/Tools/Testing/run.py @@ -0,0 +1,192 @@ +"""Run registered stdVBA suites in disposable Office projects.""" +import argparse +import json +from pathlib import Path +import sys +import tempfile +import time + +sys.dont_write_bytecode = True +ROOT = Path(__file__).resolve().parents[2] + + +def load_manifest(): + data = json.loads((ROOT / 'tests/manifest.json').read_bytes()) + seen = set() + for suite in data['suites']: + if suite['id'].casefold() in seen: + raise ValueError('Duplicate suite: ' + suite['id']) + seen.add(suite['id'].casefold()) + for file in suite.get('files', []) + ([suite['command']] if 'command' in suite else []): + path = (ROOT / file).resolve() + if not path.is_relative_to(ROOT) or not path.is_file(): + raise ValueError('Invalid manifest file: ' + file) + if not suite['cases']: + raise ValueError('Empty suite: ' + suite['id']) + return data['suites'] + + +def select(suites, names=None, case=None, all_suites=False): + wanted = {name.casefold() for name in names or []} + unknown = wanted - {s['id'].casefold() for s in suites} + if unknown: + raise ValueError('Unknown suite: ' + ', '.join(sorted(unknown))) + if case and not wanted: + raise ValueError('--test requires --suite') + selected = [] + for suite in suites: + if not (suite['id'].casefold() in wanted if wanted else all_suites or suite['default']): + continue + cases = [name for name in suite['cases'] if not case or name.casefold() == case.casefold()] + if case and not cases: + raise ValueError('Unknown test for ' + suite['id'] + ': ' + case) + selected.extend((suite, name) for name in cases) + if not selected: + raise ValueError('No tests selected') + return selected + + +def bridge(suite, case): + target = suite['cases'][case] + mode = suite.get('result', 'assertions') + call = ' Test.FullTest = ' + ('True' if suite.get('full_test') else 'False') + '\n Call ' + target + '\n' + if mode != 'assertions': + call = ' Dim result As Variant\n result = ' + target + '\n' + condition = {'pass': 'Left$(CStr(result), 4) = "PASS"', + 'count': 'IsNumeric(result) And result > 0', + 'timer': 'result = True' if case == 'testInvalidResolution' else 'Left$(CStr(result), 4) = "PASS"'}[mode] + call += ' Test.Assert "Returned result: " & CStr(result), ' + condition + '\n' + return '''Attribute VB_Name = "UnifiedTestRunner" +Option Explicit +Public Function RunCase() As String + On Error GoTo Failed + Test.Refresh +'''+call+''' If Test.checks = 0 Then Err.Raise 5, , "No assertions ran" + RunCase = "{""passed"":" & LCase$(CStr(Test.failures = 0)) & ",""assertions"":" & CStr(Test.checks) & ",""failures"":" & CStr(Test.failures) & ",""detail"":" & Quote(Test.lastCheck) & "}" + Exit Function +Failed: + RunCase = "{""passed"":false,""detail"":" & Quote("VBA error " & Err.Number & ": " & Err.Description) & "}" +End Function +Private Function Quote(ByVal value As String) As String + value = Replace(value, "\\", "\\\\") + value = Replace(value, Chr$(34), "\\" & Chr$(34)) + value = Replace(value, vbCr, "\\r") + value = Replace(value, vbLf, "\\n") + value = Replace(value, vbTab, "\\t") + Quote = Chr$(34) & value & Chr$(34) +End Function +''' + + +def execute(suite, case, host, clang=None, full=False): + if suite.get('unsupported'): + return {'status': 'skipped', 'detail': suite['unsupported']} + if host not in suite['hosts']: + return {'status': 'skipped', 'detail': 'Supported hosts: ' + ', '.join(suite['hosts'])} + if suite['kind'] == 'command': + import shutil + import subprocess + compiler = clang or shutil.which('clang') + if suite.get('compiler') and not compiler: + return {'status': 'skipped', 'detail': 'Requires llvm-mingw clang; pass --clang PATH'} + args = [sys.executable, '-B', str(ROOT / suite['command'])] + if suite.get('compiler'): + args += ['--clang', compiler] + subprocess.run(args, cwd=ROOT, check=True, timeout=300) + return {'status': 'passed'} + if suite['kind'] == 'python': + import subprocess + subprocess.run([sys.executable, '-B', str(ROOT / suite['command']), suite['cases'][case]], cwd=ROOT, check=True, timeout=300) + return {'status': 'passed'} + import office + with tempfile.TemporaryDirectory(prefix='stdVBA-tests-') as scratch: + config = dict(suite, full_test=full) + if suite['kind'] == 'vba': + path = Path(scratch) / 'UnifiedTestRunner.bas' + path.write_bytes(bridge(config, case).replace('\n', '\r\n').encode('utf-8')) + config['files'] = ['tests/support/Test.cls'] + suite['files'] + [str(path)] + if suite.get('instrument') == 'date': + source = (ROOT / 'src/WIP/stdDate.cls').read_text() + source = source.replace(' Set CName = CachedNames(key)', ' Set CName = CachedNames(key)\n StdRegexTestRunner.regexCount = CachedRegex.Count\n StdRegexTestRunner.namesCount = CachedNames.Count') + instrumented = Path(scratch) / 'stdDate.cls' + instrumented.write_bytes(source.replace('\n', '\r\n').encode('utf-8')) + config['files'][1] = str(instrumented) + if suite.get('instrument') == 'lambda-vm': + source = (ROOT / 'src/stdLambda.cls').read_text(encoding='utf-8') + fragment = (ROOT / 'tests/support/stdLambdaVM.cls.inc').read_text(encoding='utf-8') + lines = [] + for number, line in enumerate(fragment.splitlines(), 1): + if line.lstrip().startswith('Debug.Assert '): + expression = line.strip()[len('Debug.Assert '):] + line = ' Test.Assert "VM instruction line ' + str(number) + '", ' + expression + lines.append(line) + instrumented = Path(scratch) / 'stdLambda.cls' + instrumented.write_bytes((source + '\n' + '\n'.join(lines)).replace('\n', '\r\n').encode('utf-8')) + config['files'][config['files'].index('src/stdLambda.cls')] = str(instrumented) + try: + office.run(config, host, scratch) + except Exception as error: + result = Path(scratch) / 'result.json' + if result.exists(): + detail = json.loads(result.read_bytes()) + detail['passed'] = False + detail['detail'] = detail.get('detail', '') + '; ' + str(error) + return dict(detail, status='failed') + raise error + result = Path(scratch) / 'result.json' + if result.exists(): + detail = json.loads(result.read_bytes()) + return dict(detail, status='passed' if detail['passed'] else 'failed') + if suite['kind'] == 'vba': + raise RuntimeError('Office worker did not produce a result') + return {'status': 'passed'} + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--suite', action='append', help='Suite ID; repeat to select several') + parser.add_argument('--test', help='Exact case ID from --list') + parser.add_argument('--full', action='store_true', help='Enable legacy Test.FullTest branches (may require interaction)') + parser.add_argument('--clang', help='llvm-mingw clang executable for native Regex tests') + parser.add_argument('--all', action='store_true', help='Include opt-in and unavailable suites') + parser.add_argument('--list', action='store_true') + parser.add_argument('--host', choices=['Excel', 'Word', 'Access'], default='Excel') + parser.add_argument('--json', type=Path, help='Write structured results') + args = parser.parse_args(argv) + try: + suites = load_manifest() + if args.list: + for suite in suites: + if args.suite and suite['id'].casefold() not in {n.casefold() for n in args.suite}: + continue + print(suite['id'] + ' [' + ', '.join(suite['hosts']) + '] ' + ('default' if suite['default'] else 'opt-in')) + for case in suite['cases']: + print(' ' + case) + if suite.get('unsupported') or suite.get('requirements'): + print(' Requires: ' + suite.get('unsupported', suite.get('requirements', ''))) + if args.suite: + select(suites, args.suite, args.test, args.all) + return 0 + selected = select(suites, args.suite, args.test, args.all) + except ValueError as error: + parser.error(str(error)) + results = [] + for suite, case in selected: + start = time.monotonic() + try: + result = execute(suite, case, args.host, args.clang, args.full) + except Exception as error: + result = {'status': 'failed', 'detail': str(error)} + result.update(suite=suite['id'], case=case, host=args.host, seconds=round(time.monotonic()-start, 3)) + results.append(result) + print(result['status'].upper() + ' ' + suite['id'] + '.' + case + ': ' + result.get('detail', ''), flush=True) + if args.json: + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_bytes((json.dumps({'results': results}, indent=2) + '\n').replace('\n', '\r\n').encode()) + counts = {status: sum(r['status'] == status for r in results) for status in ['passed', 'failed', 'skipped']} + print(', '.join(str(count) + ' ' + status for status, count in counts.items())) + return 1 if counts['failed'] or not counts['passed'] or (args.suite and counts['skipped']) else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/Tools/stdDate-test/README.md b/Tools/stdDate-test/README.md deleted file mode 100644 index cdc5d611..00000000 --- a/Tools/stdDate-test/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Run `powershell -NoProfile -File Tools/stdDate-test/run-tests.ps1` from this worktree. - -Uses a new hidden Excel application and scratch workbook; closes only these objects and changes no security settings. Requires existing trust for VBA project access. A temporary imported stdDate copy reports actual static dictionary counts to the runner; production source/API is not instrumented. Tests cover US/UK defaults, all seven dates and explicit first days, abbreviated/full names, regex reuse/options, malformed errors and cache retention. The temporary class file is retained in the system temp directory for inspection. - -2026-10-31: execution blocked at VBProject access by the existing trust setting. No VBA assertions ran and compilation was not verified. The WIP class also contains a pre-existing Exit Function inside the isLeap Property Get; this work does not repair unrelated WIP functionality. diff --git a/Tools/stdDate-test/run-tests.ps1 b/Tools/stdDate-test/run-tests.ps1 deleted file mode 100644 index 37676a41..00000000 --- a/Tools/stdDate-test/run-tests.ps1 +++ /dev/null @@ -1,38 +0,0 @@ -$HostApplication = 'Excel' -$ErrorActionPreference = 'Stop' -$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path -$office = $null -$scratch = Join-Path ([IO.Path]::GetTempPath()) ([guid]::NewGuid().ToString() + '.cls') -$source = Get-Content (Join-Path $repoRoot 'src/WIP/stdDate.cls') -Raw -$source = $source.Replace(' Set CName = CachedNames(key)', ' Set CName = CachedNames(key)' + "`r`n" + ' StdRegexTestRunner.regexCount = CachedRegex.Count' + "`r`n" + ' StdRegexTestRunner.namesCount = CachedNames.Count') -Set-Content $scratch $source -$document = $null -try { - $office = New-Object -ComObject 'Excel.Application' - $office.Visible = $false - $office.DisplayAlerts = 0 - if ($HostApplication -eq 'Excel') { $document = $office.Workbooks.Add() } - else { $document = $office.Documents.Add() } - $project = $document.GetType().InvokeMember('VBProject', [System.Reflection.BindingFlags]::GetProperty, $null, $document, $null) - foreach ($file in @('src\WIP\stdDate.cls', 'Tools\stdRegex-build\TestReporter.cls', 'Tools\stdDate-test\TestRunner.bas', 'tests\stdDateTests.bas')) { - if ($file -eq 'src\WIP\stdDate.cls') { $null = $project.VBComponents.Import($scratch) } - else { $null = $project.VBComponents.Import((Join-Path $repoRoot $file)) } - } - if ($HostApplication -eq 'Excel') { - $result = $office.Run("'" + $document.Name + "'!StdRegexTestRunner.RunTests") - } else { - $result = $office.Run('StdRegexTestRunner.RunTests') - } - if ($result -is [string]) { throw $result } - if ($result -le 0) { throw 'No stdDate assertions ran.' } - Write-Output "Passed $result stdDate checks in $HostApplication VBA." -} finally { - try { - if ($null -ne $document) { - if ($HostApplication -eq 'Excel') { $document.Close($false) } - else { $document.Close(0) } - } - } finally { - if ($null -ne $office) { $office.Quit() } - } -} diff --git a/Tools/stdRegex-build/benchmark.ps1 b/Tools/stdRegex-build/benchmark.ps1 index e6141a85..7eaf9b5f 100644 --- a/Tools/stdRegex-build/benchmark.ps1 +++ b/Tools/stdRegex-build/benchmark.ps1 @@ -1,84 +1,2 @@ -param( - [string]$BaselineRef = 'c02d86fd7e5c99b648f9729d7a0c6160d492a642', - [ValidateSet('Excel', 'Word')][string]$HostApplication = 'Word', - [ValidateRange(3, 31)][int]$Samples = 7, - [ValidateRange(0.01, 1.0)][double]$TargetSeconds = 0.05, - [string]$OutputDir = (Join-Path $PSScriptRoot 'benchmarks\latest') -) - -# The baseline is the pure-VBA stdRegex before the list/array changes, not the -# older VBScript wrapper. Both classes run in the same hidden Office process. -$ErrorActionPreference = 'Stop' -$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path -$baselineCommit = (& git -C $repoRoot rev-parse --verify --end-of-options ($BaselineRef + '^{commit}')).Trim() -if ($LASTEXITCODE -ne 0) { throw 'Cannot resolve baseline commit.' } -$baselineLines = & git -C $repoRoot show ($baselineCommit + ':src/stdRegex.cls') -if ($LASTEXITCODE -ne 0) { throw 'Cannot read baseline class.' } -$baseline = [regex]::Replace(($baselineLines -join "`r`n"), '\bstdRegex\b', 'stdRegexBaseline') + "`r`n" -$tempRoot = [System.IO.Path]::GetFullPath([System.IO.Path]::GetTempPath()) -$workDir = Join-Path $tempRoot ('stdRegex-benchmark-' + [guid]::NewGuid().ToString('N')) -$null = New-Item -ItemType Directory -Path $workDir -$baselinePath = Join-Path $workDir 'stdRegexBaseline.cls' -[System.IO.File]::WriteAllText($baselinePath, $baseline, [System.Text.UTF8Encoding]::new($false)) -$office = $null -$document = $null -$rows = [System.Collections.Generic.List[object]]::new() -try { - $office = New-Object -ComObject ($HostApplication + '.Application') - $office.Visible = $false - $office.DisplayAlerts = 0 - if ($HostApplication -eq 'Excel') { $document = $office.Workbooks.Add() } - else { $document = $office.Documents.Add() } - $project = $document.GetType().InvokeMember('VBProject', [System.Reflection.BindingFlags]::GetProperty, $null, $document, $null) - foreach ($path in @($baselinePath, (Join-Path $repoRoot 'src\stdRegex.cls'), (Join-Path $PSScriptRoot 'Benchmark.bas'))) { - $null = $project.VBComponents.Import($path) - } - $prefix = if ($HostApplication -eq 'Excel') { "'" + $document.Name + "'!" } else { '' } - $officeVersion = $office.Version - $vbaArchitecture = $office.Run($prefix + 'StdRegexBenchmark.HostDetails') - for ($caseId = 0; $caseId -lt 6; $caseId++) { - $values = $office.Run($prefix + 'StdRegexBenchmark.RunCase', $caseId, $Samples, $TargetSeconds) - if ($values -is [string]) { throw $values } - for ($i = 1; $i -le $values.GetLength(0); $i++) { - $rows.Add([pscustomobject]@{ - case = $values[$i, 1]; matches = [int]$values[$i, 2]; captures = [int]$values[$i, 3] - input_characters = [int]$values[$i, 4]; operation = $values[$i, 5] - sample = [int]$values[$i, 6]; iterations = [int]$values[$i, 7] - elapsed_ms = [double]$values[$i, 8]; ms_per_call = [double]$values[$i, 9] - }) - } - Write-Output ('Measured: ' + $values[1, 1]) - } - $metadata = [ordered]@{ - measured_at_utc = [DateTime]::UtcNow.ToString('o') - host_application = $HostApplication; office_version = $officeVersion; vba_architecture = $vbaArchitecture - operating_system = [System.Environment]::OSVersion.VersionString - baseline_commit = $baselineCommit - current_class_sha256 = (Get-FileHash -LiteralPath (Join-Path $repoRoot 'src\stdRegex.cls') -Algorithm SHA256).Hash - engine = (Get-Content -LiteralPath (Join-Path $PSScriptRoot 'engine.json') -Raw | ConvertFrom-Json) - samples = $Samples; minimum_batch_seconds = $TargetSeconds - methodology = 'QueryPerformanceCounter; compiled regexes reused; 2 warm-up calls; calibrated batches; rotated operation order; includes result allocation and release; output equality checked before timing.' - } - $null = New-Item -ItemType Directory -Path $OutputDir -Force - $metadata | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $OutputDir 'environment.json') -Encoding utf8 - $rows | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $OutputDir 'samples.json') -Encoding utf8 - $rows | Export-Csv -LiteralPath (Join-Path $OutputDir 'samples.csv') -NoTypeInformation -Encoding utf8 - & python (Join-Path $PSScriptRoot 'summarize-benchmark.py') $OutputDir - if ($LASTEXITCODE -ne 0) { throw 'Benchmark summarization failed.' } -} finally { - try { - if ($null -ne $document) { - if ($HostApplication -eq 'Excel') { $document.Close($false) } - else { $document.Close(0) } - } - } finally { - try { if ($null -ne $office) { $office.Quit() } } - finally { - $resolvedWork = [System.IO.Path]::GetFullPath($workDir) - $resolvedTemp = [System.IO.Path]::GetFullPath($tempRoot).TrimEnd('\', '/') + [System.IO.Path]::DirectorySeparatorChar - if (-not $resolvedWork.StartsWith($resolvedTemp, [System.StringComparison]::OrdinalIgnoreCase) -or - -not ([System.IO.Path]::GetFileName($resolvedWork)).StartsWith('stdRegex-benchmark-')) { throw 'Unsafe scratch cleanup path.' } - Remove-Item -LiteralPath $resolvedWork -Recurse -Force - } - } -} +& (Join-Path $PSScriptRoot "../../tests/harnesses/stdRegex-build/benchmark.ps1") @args +exit $LASTEXITCODE diff --git a/Tools/stdRegex-build/run-tests.ps1 b/Tools/stdRegex-build/run-tests.ps1 index 7e7addb9..ff5dda4c 100644 --- a/Tools/stdRegex-build/run-tests.ps1 +++ b/Tools/stdRegex-build/run-tests.ps1 @@ -1,42 +1,8 @@ param( - [string]$EngineRoot = (Join-Path $PSScriptRoot '..\..\..\vba-regex'), - [ValidateSet('Excel', 'Word')][string]$HostApplication = 'Excel' + [string]$EngineRoot = (Join-Path $PSScriptRoot '..\..\..\vba-regex'), + [ValidateSet('Excel','Word')][string]$HostApplication = 'Excel' ) - -# Requires an Office host with "Trust access to the VBA project object model". -# No security settings are changed by this runner. -$ErrorActionPreference = 'Stop' -$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..')).Path & python (Join-Path $PSScriptRoot 'build.py') --engine-root $EngineRoot --check -if ($LASTEXITCODE -ne 0) { throw 'Generated stdRegex.cls is out of date.' } - -$office = $null -$document = $null -try { - $office = New-Object -ComObject ($HostApplication + '.Application') - $office.Visible = $false - $office.DisplayAlerts = 0 - if ($HostApplication -eq 'Excel') { $document = $office.Workbooks.Add() } - else { $document = $office.Documents.Add() } - $project = $document.GetType().InvokeMember('VBProject', [System.Reflection.BindingFlags]::GetProperty, $null, $document, $null) - foreach ($file in @('src\stdRegex.cls', 'Tools\stdRegex-build\TestReporter.cls', 'Tools\stdRegex-build\TestRunner.bas', 'tests\stdRegexTests.bas')) { - $null = $project.VBComponents.Import((Join-Path $repoRoot $file)) - } - if ($HostApplication -eq 'Excel') { - $result = $office.Run("'" + $document.Name + "'!StdRegexTestRunner.RunTests") - } else { - $result = $office.Run('StdRegexTestRunner.RunTests') - } - if ($result -is [string]) { throw $result } - if ($result -le 0) { throw 'No stdRegex assertions ran.' } - Write-Output "Passed $result stdRegex checks in $HostApplication VBA." -} finally { - try { - if ($null -ne $document) { - if ($HostApplication -eq 'Excel') { $document.Close($false) } - else { $document.Close(0) } - } - } finally { - if ($null -ne $office) { $office.Quit() } - } -} +if ($LASTEXITCODE) { exit $LASTEXITCODE } +& python (Join-Path $PSScriptRoot '../Testing/run.py') --suite stdRegex --host $HostApplication +exit $LASTEXITCODE diff --git a/Tools/stdRegex-build/summarize-benchmark.py b/Tools/stdRegex-build/summarize-benchmark.py index 4a775f3f..af8c3402 100644 --- a/Tools/stdRegex-build/summarize-benchmark.py +++ b/Tools/stdRegex-build/summarize-benchmark.py @@ -1,48 +1,3 @@ -"""Summarize measured per-call samples; no synthetic timings.""" -from collections import defaultdict -import json from pathlib import Path -from statistics import median -import sys - - -def summarize(folder): - samples = json.loads((folder / 'samples.json').read_text(encoding='utf-8-sig')) - environment = json.loads((folder / 'environment.json').read_text(encoding='utf-8-sig')) - groups = defaultdict(list) - for row in samples: - groups[(row['case'], row['operation'])].append(row['ms_per_call']) - cases = [] - for case in dict.fromkeys(row['case'] for row in samples): - timing = {} - for (label, op), values in groups.items(): - if label == case: - timing[op] = {'median_ms': median(values), 'min_ms': min(values), 'max_ms': max(values)} - comparisons = [] - for before, after in [('List before', 'List after'), ('ListArr before', 'ListArr after'), ('MatchAll', 'MatchAllArr')]: - old, new = timing[before]['median_ms'], timing[after]['median_ms'] - comparisons.append({'before': before, 'after': after, 'before_ms': old, 'after_ms': new, - 'speedup': old / new, 'time_reduction_percent': 100 * (old - new) / old}) - cases.append({'case': case, 'timings': timing, 'comparisons': comparisons}) - result = {'environment': environment, 'cases': cases} - (folder / 'summary.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') - lines = ['# stdRegex before/after benchmark', '', - f"Host: {environment['host_application']} {environment['office_version']}, {environment['vba_architecture']}.", - f"Measured UTC: {environment['measured_at_utc']}.", - f"Baseline: `{environment['baseline_commit']}` (pure VBA engine, before list optimizations).", '', - f"{environment['samples']} samples per operation/case; medians in milliseconds per call. " + environment['methodology'], '', - 'ListArr uses two output columns. MatchAllArr materializes all numbered captures; MatchAll additionally creates dictionaries and named entries. Their contracts differ.', ''] - for case in cases: - lines += ['## ' + case['case'], ''] - for item in case['comparisons']: - change = item['time_reduction_percent'] - direction = 'less time' if change >= 0 else 'more time' - lines.append(f"- {item['before']} → {item['after']}: **{item['before_ms']:.3f} → {item['after_ms']:.3f} ms**, {item['speedup']:.2f}× relative speed ({abs(change):.1f}% {direction}).") - lines.append('') - lines += ['These measurements describe this host and these workloads. Sparse/no-match workloads spend more time in the unchanged search engine, so wrapper-only improvements may be small or negative.', ''] - (folder / 'results.md').write_text('\n'.join(lines), encoding='utf-8') - print('\n'.join(lines).replace('\u2192', '->').replace('\u00d7', 'x')) - - -if __name__ == '__main__': - summarize(Path(sys.argv[1])) +import runpy +runpy.run_path(str(Path(__file__).resolve().parents[2] / "tests/harnesses/stdRegex-build/summarize-benchmark.py"), run_name="__main__") diff --git a/Tools/stdRegex-build/test_build.py b/Tools/stdRegex-build/test_build.py index 168c91c7..559a60cf 100644 --- a/Tools/stdRegex-build/test_build.py +++ b/Tools/stdRegex-build/test_build.py @@ -1,99 +1,3 @@ -"""Version-selection and failure-safety tests for the standalone-class builder.""" -from contextlib import redirect_stderr, redirect_stdout -import io -import json from pathlib import Path -import subprocess -from tempfile import TemporaryDirectory -import unittest -from unittest.mock import patch - -import build - - -class BuildSelectionTests(unittest.TestCase): - def setUp(self): - self.temp = TemporaryDirectory() - self.addCleanup(self.temp.cleanup) - root = Path(self.temp.name) - self.here = root / 'stdVBA' / 'Tools' / 'stdRegex-build' - self.here.mkdir(parents=True) - self.target = root / 'stdVBA' / 'src' / 'stdRegex.cls' - self.target.parent.mkdir() - self.target.write_bytes(b'previous class\r\n') - self.engine = root / 'engine' - self.engine.mkdir() - self.git('init', '-q') - self.git('config', 'user.name', 'Builder test') - self.git('config', 'user.email', 'test@example.invalid') - (self.engine / 'LICENSE').write_text('first engine revision') - self.git('add', 'LICENSE') - self.git('commit', '-q', '-m', 'first') - self.first = self.git('rev-parse', 'HEAD').strip() - (self.engine / 'LICENSE').write_text('second engine revision') - self.git('commit', '-qam', 'second') - self.second = self.git('rev-parse', 'HEAD').strip() - self.manifest = self.here / 'engine.json' - self.manifest.write_text(json.dumps({'repository': 'https://github.com/sihlfall/vba-regex', 'revision': self.first}) + '\n') - self.saved_manifest = self.manifest.read_bytes() - self.saved_class = self.target.read_bytes() - - def git(self, *args): - return subprocess.check_output(['git', '-C', str(self.engine), *args], text=True) - - def invoke(self, *args, fail=False): - def generate(root, selected): - if fail: - raise ValueError('incompatible engine') - return 'generated ' + selected['revision'] + '\n' - with patch.object(build, 'HERE', self.here), patch.object(build, 'generate', side_effect=generate), redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): - build.main(['--engine-root', str(self.engine), *args]) - - def test_default_reproduces_recorded_commit_even_when_head_is_newer(self): - self.invoke() - self.assertEqual(self.target.read_text(), 'generated ' + self.first + '\n') - self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) - - def test_ref_override_does_not_adopt_revision_implicitly(self): - self.invoke('--engine-ref', 'HEAD') - self.assertEqual(self.target.read_text(), 'generated ' + self.second + '\n') - self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) - - def test_update_records_resolved_sha_and_matching_class(self): - self.invoke('--engine-ref', 'HEAD', '--update-engine') - self.assertEqual(json.loads(self.manifest.read_text())['revision'], self.second) - self.assertEqual(self.target.read_bytes(), ('generated ' + self.second + '\r\n').encode()) - - def test_update_requires_a_ref(self): - with self.assertRaises(SystemExit): - self.invoke('--update-engine') - self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) - self.assertEqual(self.target.read_bytes(), self.saved_class) - - def test_check_is_read_only(self): - self.target.write_text('generated ' + self.first + '\n') - before = self.target.read_bytes() - self.invoke('--check') - self.assertEqual(self.target.read_bytes(), before) - self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) - - def test_check_cannot_update_manifest(self): - with self.assertRaises(SystemExit): - self.invoke('--check', '--engine-ref', 'HEAD', '--update-engine') - self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) - - def test_incompatible_engine_does_not_change_existing_files(self): - with self.assertRaises(ValueError): - self.invoke('--engine-ref', 'HEAD', '--update-engine', fail=True) - self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) - self.assertEqual(self.target.read_bytes(), self.saved_class) - - def test_missing_revision_does_not_change_existing_files(self): - with self.assertRaises(subprocess.CalledProcessError): - self.invoke('--engine-ref', 'missing-engine-ref', '--update-engine') - self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) - self.assertEqual(self.target.read_bytes(), self.saved_class) - - -if __name__ == '__main__': - unittest.main() +import runpy +runpy.run_path(str(Path(__file__).resolve().parents[2] / "tests/tooling/test_regex_build.py"), run_name="__main__") diff --git a/Tools/stdRegex-thunk/README.md b/Tools/stdRegex-thunk/README.md index 6f5cd08f..c0d4a8c1 100644 --- a/Tools/stdRegex-thunk/README.md +++ b/Tools/stdRegex-thunk/README.md @@ -104,7 +104,7 @@ zero code relocations. This checks executable code, not just cross-compilation. `test-native.c` contains 18 fixture checks repeated 1,000 times per architecture. Build with the same Clang using `--target=i686-w64-windows-gnu -O2 -static -Tools/stdRegex-thunk/test-native.c -luser32 -o /test-x86.exe`, then pass +tests/harnesses/stdRegex-thunk/test-native.c -luser32 -o /test-x86.exe`, then pass `Tools/stdRegex-thunk/matcher-x86.bin` to that executable. Repeat with `--target=x86_64-w64-windows-gnu` and `matcher.bin` for 64-bit. diff --git a/Tools/stdRegex-thunk/run.py b/Tools/stdRegex-thunk/run.py index 38eea387..ffae6e1d 100644 --- a/Tools/stdRegex-thunk/run.py +++ b/Tools/stdRegex-thunk/run.py @@ -1,193 +1,3 @@ -"""Disposable Word x64 test host with bounded watchdog and exact registry cleanup. - -Requires Python/pywin32 and desktop Word. Run from any directory. No user Office -instance is attached to or closed. The supervisor owns settings and timeout. -""" from pathlib import Path -import ctypes -import hashlib -import json -import os -import re -import subprocess -import sys -import tempfile -import time -import winreg - -HERE = Path(__file__).resolve().parent -ROOT = HERE.parent.parent - -def write(path, text): - path.write_bytes(text.replace('\r\n', '\n').replace('\n', '\r\n').encode('utf-8')) - -def worker(folder, test_only=False): - import win32com.client - import win32process - app = doc = None - result = {'measured_at_utc': __import__('datetime').datetime.now(__import__('datetime').timezone.utc).isoformat()} - try: - app = win32com.client.DispatchEx('Word.Application') - app.Visible = False - app.DisplayAlerts = 0 - doc = app.Documents.Add() - pid = win32process.GetWindowThreadProcessId(doc.ActiveWindow.Hwnd)[1] - (folder/'pid').write_text(str(pid)) - result['office_version'] = app.Version - result['office_build'] = app.Build - result['cpu'] = __import__('platform').processor() - result['windows'] = __import__('platform').platform() - project = doc.VBProject # Check actual application's project access. - paths = [folder/'stdRegex.cls', - ROOT/'Tools/stdRegex-build/TestReporter.cls', - ROOT/'Tools/stdRegex-build/TestRunner.bas', folder/'stdRegexTests.bas', - folder/'ThunkTests.bas', folder/'ThunkTestRunner.bas', HERE/'Benchmark.bas'] - for path in paths: - data = path.read_bytes() - assert b'\n' not in data.replace(b'\r\n', b'') and b'\r' not in data.replace(b'\r\n', b''), f'Not CRLF: {path}' - project.VBComponents.Import(str(path)) - for name in ['ThunkBenchmark.Differential','StdRegexTestRunner.RunTests','ThunkTestRunner.RunTests']: - print('Running '+name, flush=True) - value = app.Run(name) - result[name] = value - if isinstance(value, str) or value <= 0: - raise RuntimeError(f'{name}: {value}') - result['samples'] = [] - for case in range(0 if test_only else 6): - print('Benchmark case '+str(case), flush=True) - rows = app.Run('ThunkBenchmark.RunCase', case, 7, 0.05) - if isinstance(rows, str): raise RuntimeError(rows) - for row in rows: - result['samples'].append(dict(zip(['case','matches','captures','characters','operation','sample','iterations','elapsed_ms','ms_per_call'],row))) - result['status'] = 'passed' - except Exception as exc: - result['status'] = 'failed' - result['error'] = repr(exc) - raise - finally: - write(folder/'result.json', json.dumps(result,indent=2)+'\n') - try: - if doc is not None: doc.Close(0) - finally: - if app is not None: app.Quit(0) - -def supervise(test_only=False): - import win32gui - import win32process - import win32con - import psutil - # Refuse managed AccessVBOM settings rather than overriding policy. - with winreg.OpenKey(winreg.HKEY_CLASSES_ROOT, r'Word.Application\CurVer') as key: - version = winreg.QueryValueEx(key, '')[0].rsplit('.',1)[1]+'.0' - for hive in (winreg.HKEY_CURRENT_USER,winreg.HKEY_LOCAL_MACHINE): - try: - with winreg.OpenKey(hive,rf'Software\Policies\Microsoft\Office\{version}\Word\Security') as key: - value,_ = winreg.QueryValueEx(key,'AccessVBOM') - if not value: raise RuntimeError('Managed policy disables VBA project access') - except FileNotFoundError: pass - settings = [(rf'Software\Microsoft\Office\{version}\Word\Security','AccessVBOM',1), - (r'Software\Microsoft\VBA\7.1\Common','BreakOnAllErrors',0), - (r'Software\Microsoft\VBA\7.1\Common','BreakOnServerErrors',0)] - saved=[] - output=HERE/('verification-x64' if test_only else 'results') - output.mkdir(exist_ok=True) - started=time.time() - original_pids={p.pid for p in psutil.process_iter()} - dialogs=[] - process=None - office_pid=None - with tempfile.TemporaryDirectory(prefix='stdRegex-thunk-test-') as tmp: - folder=Path(tmp) - combined=(ROOT/'src/stdRegex.cls').read_text(encoding='utf-8') - combined += '\nPublic Function CreateNative(ByVal pattern As String, Optional ByVal flags As String = "", Optional ByVal stepsLimit As Long = 10000) As stdRegex\n Set CreateNative = Create(pattern, flags, stepsLimit, True)\nEnd Function\n' - combined += '\nPublic Function CreateVBA(ByVal pattern As String, Optional ByVal flags As String = "", Optional ByVal stepsLimit As Long = 10000) As stdRegex\n Set CreateVBA = Create(pattern, flags, stepsLimit, False)\nEnd Function\n' - write(folder/'stdRegex.cls', combined) - baseline_tests=(ROOT/'tests/stdRegexTests.bas').read_text(encoding='utf-8') - write(folder/'stdRegexTests.bas', baseline_tests.replace('stdRegex.Create(', 'stdRegex.CreateVBA(')) - source=(ROOT/'tests/stdRegexTests.bas').read_text(encoding='utf-8') - source=source.replace('stdRegex.Create(', 'stdRegex.CreateNative(') - source=re.sub(r'\bstdRegexTests\b','ThunkTests',source) - write(folder/'ThunkTests.bas',source) - source=(ROOT/'Tools/stdRegex-build/TestRunner.bas').read_text(encoding='utf-8') - write(folder/'ThunkTestRunner.bas',source.replace('StdRegexTestRunner','ThunkTestRunner').replace('stdRegexTests','ThunkTests')) - try: - for path,name,value in settings: - try: - key=winreg.OpenKey(winreg.HKEY_CURRENT_USER,path,0,winreg.KEY_READ|winreg.KEY_WRITE) - existed=True - except FileNotFoundError: - key=winreg.CreateKey(winreg.HKEY_CURRENT_USER,path); existed=False - with key: - try: old,kind=winreg.QueryValueEx(key,name); present=True - except FileNotFoundError: old=kind=None; present=False - saved.append(dict(path=path,name=name,key_existed=existed,present=present,value=old,kind=kind)) - winreg.SetValueEx(key,name,0,winreg.REG_DWORD,value) - write(output/'settings-before.json',json.dumps(saved,indent=2)+'\n') - process=subprocess.Popen([sys.executable,str(Path(__file__)), '--worker',str(folder)]+(['--test-only'] if test_only else [])) - deadline=time.monotonic()+240 - while process.poll() is None: - if (folder/'pid').exists(): office_pid=int((folder/'pid').read_text()) - if office_pid and office_pid not in original_pids: - def visit(hwnd,_): - if win32process.GetWindowThreadProcessId(hwnd)[1]!=office_pid: return - if not win32gui.IsWindowVisible(hwnd): return - klass=win32gui.GetClassName(hwnd) - if klass=='#32770': - texts=[]; buttons=[] - def child(ch,_): - texts.append(win32gui.GetWindowText(ch)) - if win32gui.GetClassName(ch)=='Button': buttons.append(ch) - win32gui.EnumChildWindows(hwnd,child,None) - dialogs.append(' | '.join(texts)) - if buttons: win32gui.PostMessage(buttons[0],win32con.BM_CLICK,0,0) - else: win32gui.PostMessage(hwnd,win32con.WM_CLOSE,0,0) - ctypes.windll.user32.ShowWindowAsync(hwnd,6) - win32gui.EnumWindows(visit,None) - if time.monotonic()>deadline: - raise TimeoutError('Office test exceeded 240 seconds') - time.sleep(0.05) - if (folder/'result.json').exists(): - data=json.loads((folder/'result.json').read_text()) - else: data={'status':'failed','error':'No worker result'} - data.update(dialogs=dialogs,elapsed_seconds=time.time()-started, - baseline_commit=subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(), - native_class_sha256=hashlib.sha256((ROOT/'src/stdRegex.cls').read_bytes()).hexdigest(), - architecture='Windows x64',methodology='QPC; 7 rotated samples; warmed cached patterns; calibrated >=50ms batches; allocation/release included; output equality checked') - if dialogs: data['status']='failed' - write(output/'results.json',json.dumps(data,indent=2)+'\n') - if process.returncode or data['status']!='passed': raise RuntimeError(data.get('error',str(dialogs))) - print('All requested checks passed',flush=True) - finally: - cleanup_error = None - try: - if process and process.poll() is None: process.kill(); process.wait(timeout=10) - # Close only the newly created test process, including after a COM hang. - if office_pid and office_pid not in original_pids and psutil.pid_exists(office_pid): - p=psutil.Process(office_pid) - if p.name().lower()=='winword.exe': p.terminate(); p.wait(timeout=10) - except Exception as exc: - cleanup_error = repr(exc) - for entry in reversed(saved): - with winreg.OpenKey(winreg.HKEY_CURRENT_USER,entry['path'],0,winreg.KEY_WRITE) as key: - if entry['present']: winreg.SetValueEx(key,entry['name'],0,entry['kind'],entry['value']) - else: - try: winreg.DeleteValue(key,entry['name']) - except FileNotFoundError: pass - if not entry['key_existed']: - try: winreg.DeleteKey(winreg.HKEY_CURRENT_USER,entry['path']) - except OSError: pass - verified=[] - for entry in saved: - try: - with winreg.OpenKey(winreg.HKEY_CURRENT_USER,entry['path']) as key: - value,kind=winreg.QueryValueEx(key,entry['name']) - assert entry['present'] and value==entry['value'] and kind==entry['kind'] - except FileNotFoundError: - assert not entry['present'] - verified.append(entry['name']) - write(output/'settings-restored.json',json.dumps({'verified':verified,'cleanup_error':cleanup_error},indent=2)+'\n') - if cleanup_error: raise RuntimeError('Office cleanup failed: '+cleanup_error) - -if __name__=='__main__': - if len(sys.argv)>1 and sys.argv[1]=='--worker': worker(Path(sys.argv[2]), '--test-only' in sys.argv) - else: supervise('--test-only' in sys.argv) +import runpy +runpy.run_path(str(Path(__file__).resolve().parents[2] / "tests/harnesses/stdRegex-thunk/run.py"), run_name="__main__") diff --git a/Tools/stdRegex-thunk/summarize.py b/Tools/stdRegex-thunk/summarize.py index d9c64238..b5a7f10c 100644 --- a/Tools/stdRegex-thunk/summarize.py +++ b/Tools/stdRegex-thunk/summarize.py @@ -1,44 +1,3 @@ -"""Summarize the measured samples and create a standalone comparison chart.""" -import json from pathlib import Path -import statistics - -HERE=Path(__file__).resolve().parent -def write(path,text): - path.write_bytes(text.replace('\r\n','\n').replace('\n','\r\n').encode('utf-8')) - -data=json.loads((HERE/'results/results.json').read_text()) -assert data['status']=='passed' and not data['dialogs'] -summary=[] -for case in dict.fromkeys(r['case'] for r in data['samples']): - for operation in ['Test','List','ListArr','MatchAllArr']: - record=dict(case=case,operation=operation) - for engine in ['VBA','native']: - rows=[r for r in data['samples'] if r['case']==case and r['operation']==operation+' '+engine] - values=[r['ms_per_call'] for r in rows] - assert len(values)==7 - record[engine]=dict(median_ms=statistics.median(values),min_ms=min(values),max_ms=max(values)) - record['speedup']=record['VBA']['median_ms']/record['native']['median_ms'] - summary.append(record) -write(HERE/'results/summary.json',json.dumps(summary,indent=2)+'\n') -html=''' - -

Native matcher speedup over current VBA (ratio of median call times; larger is faster).

- -
WorkloadVBA msNative msSpeedup
-

Source: local Word 16.0, Windows x64, 1 November 2026. Seven rotated samples per operation; warmed patterns; batches calibrated to at least 50 ms. Returned values are allocated and released inside timing. Test stops at the first match. Compiler and public formatting remain VBA. These are workload-specific results, not a universal speed guarantee.

-'''.replace('DATA',json.dumps(summary)) -write(HERE/'results/comparison.html',html) -for r in summary: - print(f"{r['case']} | {r['operation']}: {r['speedup']:.2f}x") +import runpy +runpy.run_path(str(Path(__file__).resolve().parents[2] / "tests/harnesses/stdRegex-thunk/summarize.py"), run_name="__main__") diff --git a/Tools/stdRegex-thunk/test-native.py b/Tools/stdRegex-thunk/test-native.py index b37b0622..2600087f 100644 --- a/Tools/stdRegex-thunk/test-native.py +++ b/Tools/stdRegex-thunk/test-native.py @@ -1,23 +1,3 @@ -"""Build real x86/x64 Windows test hosts and execute the embedded blobs.""" -import argparse -import hashlib -import json from pathlib import Path -import subprocess -import tempfile - -HERE = Path(__file__).resolve().parent -parser = argparse.ArgumentParser() -parser.add_argument('--clang', required=True) -args = parser.parse_args() -results = {} -with tempfile.TemporaryDirectory(prefix='stdRegex-abi-') as scratch: - for arch, target, blob in [('x86', 'i686', 'matcher-x86.bin'), ('x64', 'x86_64', 'matcher.bin')]: - exe = Path(scratch) / (arch+'.exe') - subprocess.run([args.clang, '--target='+target+'-w64-windows-gnu', '-O2', '-static', - str(HERE/'test-native.c'), '-luser32', '-o', str(exe)], check=True) - result = subprocess.check_output([str(exe), str(HERE/blob)], text=True, timeout=30).strip() - assert '18000 checks passed' in result - print(result) - results[arch] = dict(result=result, blob_sha256=hashlib.sha256((HERE/blob).read_bytes()).hexdigest()) -(HERE/'verification-native.json').write_bytes((json.dumps(results, indent=2)+'\n').replace('\n','\r\n').encode()) +import runpy +runpy.run_path(str(Path(__file__).resolve().parents[2] / "tests/harnesses/stdRegex-thunk/test-native.py"), run_name="__main__") diff --git a/Tools/stdWindow-subclass/run.py b/Tools/stdWindow-subclass/run.py index fc464210..1f1b7c25 100644 --- a/Tools/stdWindow-subclass/run.py +++ b/Tools/stdWindow-subclass/run.py @@ -1,376 +1,3 @@ -"""Bounded isolated Excel regression runner; requires pywin32 and Office. - -Discovers the installed Office/VBE versions, restores exact registry values, -and dismisses dialogs only in the newly created test process. -""" -import json from pathlib import Path -import subprocess -import sys -import threading -import tempfile -import winreg -import gc -import os - -ROOT = Path(__file__).resolve().parents[2] -FILES = ['src/stdICallable.cls', 'src/stdCallback.cls', 'src/stdWindow.cls', - 'tests/stdWindowSubclassProbe.cls', 'tests/stdWindowSubclassTests.bas', - 'Tools/stdWindow-subclass/IDEProbe.bas'] - - -def worker(): - print('Starting isolated Office worker', flush=True) - import win32com.client - import win32process - import win32gui - import win32con - existing = set() - win32gui.EnumWindows(lambda hwnd, _: existing.add(win32process.GetWindowThreadProcessId(hwnd)[1]), None) - app = win32com.client.DispatchEx('Excel.Application') - print('Created isolated Office application', flush=True) - pid = win32process.GetWindowThreadProcessId(app.Hwnd)[1] - if pid in existing: - raise RuntimeError('Worker returned an existing Office instance; refusing to use it.') - Path(sys.argv[2]).write_text(str(pid), encoding='ascii') - print('PID ' + str(pid), flush=True) - stop = threading.Event() - dialogs = [] - - def monitor(): - while not stop.wait(.03): - def visit(hwnd, unused): - if win32process.GetWindowThreadProcessId(hwnd)[1] != pid: - return - if win32gui.GetClassName(hwnd) == '#32770' and win32gui.IsWindowVisible(hwnd): - labels = [] - win32gui.EnumChildWindows(hwnd, lambda child, _: labels.append(win32gui.GetWindowText(child)), None) - dialogs.append(' | '.join(labels)) - print('DIALOG ' + dialogs[-1], flush=True) - win32gui.PostMessage(hwnd, win32con.WM_COMMAND, 1, 0) - elif win32gui.GetClassName(hwnd) in ('XLMAIN', 'wndclass_desked_gsk') and win32gui.IsWindowVisible(hwnd): - win32gui.ShowWindow(hwnd, win32con.SW_MINIMIZE) - win32gui.EnumWindows(visit, None) - - thread = threading.Thread(target=monitor, daemon=True) - thread.start() - book = None - try: - app.Visible = False - app.DisplayAlerts = False - no_vbom = '--no-vbom' in sys.argv - if no_vbom: - book = app.Workbooks.Open(sys.argv[3], ReadOnly=True) - try: - book.VBProject.VBComponents.Count - except Exception as error: - if 'not trusted' not in str(error).lower(): - raise - print('Verified VBOM access is disabled in this test instance', flush=True) - else: - raise RuntimeError('Negative access test did not disable VBOM') - else: - book = app.Workbooks.Add() - project = book.VBProject - project.References.AddFromGuid('{1EA4DBF0-3C3B-11CF-810C-00AA00389B71}', 1, 1) - project.VBComponents.Add(3).Name = 'stdWindowTestForm' - for file in FILES: - raw = (ROOT / file).read_bytes().replace(b'\r\n', b'') - assert b'\n' not in raw and b'\r' not in raw, file - project.VBComponents.Import(str(ROOT / file)) - app.VBE.CommandBars.FindControl(1, 578).Execute() - if dialogs: - raise RuntimeError('Compilation failed') - book.SaveAs(sys.argv[3], 52) - result = app.Run("'" + book.Name + "'!stdWindowSubclassTests.RunTests") - print(json.dumps({'result': result, 'dialogs': dialogs}), flush=True) - if dialogs or not str(result).startswith('Passed '): - raise RuntimeError('Office regression failed') - result = app.Run("'" + book.Name + "'!stdWindowSubclassTests.RunUserFormTests") - print(json.dumps({'result': result, 'dialogs': dialogs}), flush=True) - if dialogs or not str(result).startswith('Passed '): - raise RuntimeError('UserForm regression failed') - result = app.Run("'" + book.Name + "'!stdWindowSubclassTests.RunResolverTests") - print(json.dumps({'result': result, 'dialogs': dialogs}), flush=True) - if dialogs or not str(result).startswith('Passed '): - raise RuntimeError('EbMode resolver regression failed') - if not no_vbom: - ide_checks(app, book, project, pid) - if dialogs: - raise RuntimeError('Office/VBE dialog occurred during IDE checks') - finally: - try: - if book is not None: - print('Closing disposable workbook', flush=True) - book.Close(False) - finally: - try: - print('Closing isolated Office application', flush=True) - app.Quit() - print('Isolated Office application closed', flush=True) - finally: - stop.set() - thread.join(2) - # Office has closed and assertions have completed. In this disposable - # process, skip late pywin32 proxy finalizers: after a denied VBOM query - # they can block interpreter teardown even after Excel.Quit returned. - # The parent still owns the Office PID and performs failure-safe cleanup. - sys.stdout.flush() - sys.stderr.flush() - os._exit(0) - - -def ide_checks(app, book, project, pid): - import ctypes - from ctypes import wintypes - user = ctypes.WinDLL('user32', use_last_error=True) - kernel = ctypes.WinDLL('kernel32', use_last_error=True) - user.SendMessageTimeoutW.argtypes = [wintypes.HWND, wintypes.UINT, wintypes.WPARAM, wintypes.LPARAM, - wintypes.UINT, wintypes.UINT, ctypes.POINTER(ctypes.c_size_t)] - user.SendMessageTimeoutW.restype = ctypes.c_size_t - user.DestroyWindow.argtypes = [wintypes.HWND] - user.IsWindow.argtypes = [wintypes.HWND] - kernel.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] - kernel.OpenProcess.restype = wintypes.HANDLE - kernel.ReadProcessMemory.argtypes = [wintypes.HANDLE, ctypes.c_void_p, ctypes.c_void_p, - ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)] - kernel.CloseHandle.argtypes = [wintypes.HANDLE] - process = kernel.OpenProcess(0x10, False, pid) - if not process: - raise ctypes.WinError(ctypes.get_last_error()) - - def run(name): - return app.Run("'" + book.Name + "'!WindowIDEProbe." + name) - - def counts(thunk): - counters = (wintypes.DWORD * 2)() - copied = ctypes.c_size_t() - if not kernel.ReadProcessMemory(process, thunk+4096, counters, ctypes.sizeof(counters), ctypes.byref(copied)): - raise ctypes.WinError(ctypes.get_last_error()) - return tuple(counters) - - def send(hwnd): - result = ctypes.c_size_t() - if not user.SendMessageTimeoutW(hwnd, 0x8001, 1, 2, 2, 2000, ctypes.byref(result)): - raise ctypes.WinError(ctypes.get_last_error()) - return result.value - - def paused_probe(hwnd, thunk, reset=False): - import pythoncom - import win32com.client - import pywintypes - import time - import win32gui - vbe_hwnd = app.VBE.MainWindow.HWnd - run_id = reset_id = None - for menu in app.VBE.CommandBars('Menu Bar').Controls: - if menu.Caption.replace('&', '').lower() == 'run': - for control in menu.Controls: - caption = control.Caption.replace('&', '').lower() - if caption.startswith('run sub/') or caption == 'run macro': - run_id = control.Id - if caption == 'reset': - reset_id = control.Id - assert run_id and reset_id, 'Could not find the test VBE Continue/Reset controls' - stream = pythoncom.CoMarshalInterThreadInterfaceInStream(pythoncom.IID_IDispatch, app.VBE._oleobj_) - errors = [] - - def coordinator(): - pythoncom.CoInitialize() - remote = None - try: - remote = win32com.client.Dispatch(pythoncom.CoGetInterfaceAndReleaseStream(stream, pythoncom.IID_IDispatch)) - deadline = time.monotonic()+8 - while time.monotonic() < deadline: - if '[break]' in win32gui.GetWindowText(vbe_hwnd).lower(): - break - time.sleep(.03) - else: - raise RuntimeError('Test VBE did not enter break mode: ' + win32gui.GetWindowText(vbe_hwnd)) - assert send(hwnd) == 0, 'Break mode must forward without entering VBA' - assert counts(thunk) == (0, 1), 'Break mode should preserve the subscription' - remote.CommandBars.FindControl(1, reset_id if reset else run_id).Execute() - except Exception as error: - errors.append(str(error)) - print('Break-mode coordinator failed:', error, flush=True) - if remote is not None: - try: - remote.CommandBars.FindControl(1, reset_id).Execute() - except Exception: - pass - finally: - remote = None - pythoncom.CoUninitialize() - - helper = threading.Thread(target=coordinator, daemon=True) - helper.start() - try: - result = run('StopAndResume') - except pywintypes.com_error as error: - if not reset or not error.excepinfo or error.excepinfo[5] != -2146788248: - raise - result = None - helper.join(10) - assert not helper.is_alive(), 'Break-mode coordinator did not finish' - assert not errors, errors - if not reset: - assert result == 2468, 'Continue must resume callback dispatch' - - try: - hwnd, thunk = run('ArmModeless') - assert send(hwnd) == 2468, 'Modeless callbacks must work after the starter macro returns' - run('Cleanup') - print('Passed real modeless UserForm callback after starter macro returns', flush=True) - - hwnd, thunk = run('Arm') - print('IDE mode after macro:', project.Mode, 'native counters', counts(thunk), flush=True) - assert send(hwnd) == 0, 'Design mode must forward without executing VBA' - assert counts(thunk) == (0, 0), 'Design mode must remove the registration natively' - assert run('CheckDetached'), 'Wrapper must reconcile native detachment' - run('Cleanup') - print('Passed real design-mode detachment', flush=True) - - hwnd, thunk = run('Arm') - paused_probe(hwnd, thunk) - run('Cleanup') - print('Passed real break/Continue forwarding', flush=True) - - hwnd, thunk = run('Arm') - paused_probe(hwnd, thunk, reset=True) - if user.IsWindow(hwnd): - assert send(hwnd) == 0 - assert counts(thunk) == (0, 0) - print('Passed real VBE Reset detachment', flush=True) - - hwnd, thunk = run('Arm') - import pywintypes - try: - run('EndNow') - except pywintypes.com_error as error: - # VBE returns its explicit execution-aborted HRESULT for End. - if not error.excepinfo or error.excepinfo[5] != -2146788248: - raise - if user.IsWindow(hwnd): - assert send(hwnd) == 0, 'End must not call a stale VBA object' - assert counts(thunk) == (0, 0), 'End must detach without VBA globals' - # The disposable Office process owns this orphan after End; process - # teardown reclaims it. Do not enter its vanished VBA wrapper again. - print('Passed real End/reset detachment', flush=True) - finally: - kernel.CloseHandle(process) - - -def launch_worker(template, no_vbom=False): - handle = tempfile.NamedTemporaryFile(prefix='stdvba86-', suffix='.pid', delete=False) - pidfile = Path(handle.name) - handle.close() - args = [sys.executable, __file__, '--worker', str(pidfile), str(template)] - if no_vbom: - args.append('--no-vbom') - child = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) - timed_out = False - output = '' - try: - output, _ = child.communicate(timeout=45) - except subprocess.TimeoutExpired: - timed_out = True - child.kill() - finally: - owned_pid = pidfile.read_text().strip() - if owned_pid: - subprocess.run(['taskkill', '/PID', owned_pid, '/F'], capture_output=True) - if child.poll() is None: - child.kill() - tail, _ = child.communicate(timeout=5) - if timed_out: - output = tail - pidfile.unlink() - print(output, flush=True) - if timed_out: - raise RuntimeError('Office test timeout; partial worker output shown above') - if child.returncode: - raise RuntimeError('Office worker failed: ' + str(child.returncode)) - - -def main(): - import win32com.client - import win32process - import win32gui - existing = set() - win32gui.EnumWindows(lambda hwnd, _: existing.add(win32process.GetWindowThreadProcessId(hwnd)[1]), None) - app = win32com.client.DispatchEx('Excel.Application') - discovery_pid = win32process.GetWindowThreadProcessId(app.Hwnd)[1] - if discovery_pid in existing: - raise RuntimeError('Discovery returned an existing Office instance; refusing to use it.') - print('Discovery PID', discovery_pid, flush=True) - try: - version = app.Version - vbe = '.'.join(str(int(part)) for part in app.VBE.Version.split('.')) - book = app.Workbooks.Add() - try: - access = book.VBProject.VBComponents.Count > 0 - except Exception: - access = False - finally: - book.Close(False) - finally: - app.Quit() - del app - gc.collect() - subprocess.run(['taskkill', '/PID', str(discovery_pid), '/F'], capture_output=True) - print('Excel', version, 'VBE', vbe, 'VBProject access', access, flush=True) - policy_enables_access = False - for hive in (winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE): - try: - with winreg.OpenKey(hive, fr'Software\Policies\Microsoft\Office\{version}\Excel\Security') as key: - policy, _ = winreg.QueryValueEx(key, 'AccessVBOM') - if not policy: - raise RuntimeError('Managed policy disables Excel VBA project access.') - policy_enables_access = True - except FileNotFoundError: - pass - template_handle = tempfile.NamedTemporaryFile(prefix='stdvba86-', suffix='.xlsm', delete=False) - template = Path(template_handle.name) - template_handle.close() - changes = [] - try: - settings = [(fr'Software\Microsoft\VBA\{vbe}\Common', 'BreakOnAllErrors', 0), - (fr'Software\Microsoft\VBA\{vbe}\Common', 'BreakOnServerErrors', 0)] - if not access: - settings.append((fr'Software\Microsoft\Office\{version}\Excel\Security', 'AccessVBOM', 1)) - for path, name, value in settings: - with winreg.CreateKey(winreg.HKEY_CURRENT_USER, path) as key: - try: - old, kind = winreg.QueryValueEx(key, name) - except FileNotFoundError: - old, kind = None, None - changes.append((path, name, old, kind)) - print('SETTING', path, name, old, kind, flush=True) - winreg.SetValueEx(key, name, 0, winreg.REG_DWORD, value) - launch_worker(template) - if policy_enables_access: - print('Skipped disabled-VBOM variant: managed policy forces access on.', flush=True) - else: - path = fr'Software\Microsoft\Office\{version}\Excel\Security' - with winreg.CreateKey(winreg.HKEY_CURRENT_USER, path) as key: - try: - old, kind = winreg.QueryValueEx(key, 'AccessVBOM') - except FileNotFoundError: - old, kind = None, None - changes.append((path, 'AccessVBOM', old, kind)) - print('SETTING disabled-VBOM variant', path, 'AccessVBOM', old, kind, flush=True) - winreg.SetValueEx(key, 'AccessVBOM', 0, winreg.REG_DWORD, 0) - launch_worker(template, no_vbom=True) - finally: - for path, name, old, kind in reversed(changes): - with winreg.OpenKey(winreg.HKEY_CURRENT_USER, path, 0, winreg.KEY_SET_VALUE) as key: - if kind is None: - winreg.DeleteValue(key, name) - else: - winreg.SetValueEx(key, name, 0, kind, old) - template.unlink(missing_ok=True) - print('Original registry values restored.', flush=True) - - -if __name__ == '__main__': - worker() if '--worker' in sys.argv else main() +import runpy +runpy.run_path(str(Path(__file__).resolve().parents[2] / "tests/harnesses/stdWindow-subclass/run.py"), run_name="__main__") diff --git a/Tools/stdWindow-subclass/test_thunk.py b/Tools/stdWindow-subclass/test_thunk.py index dbca98f7..3c6f06f1 100644 --- a/Tools/stdWindow-subclass/test_thunk.py +++ b/Tools/stdWindow-subclass/test_thunk.py @@ -1,136 +1,3 @@ -"""Check both native ABIs without Office. Dev dependencies: keystone-engine, unicorn.""" -import struct -import unittest -from build import assemble -from unicorn import Uc, UC_ARCH_X86, UC_MODE_32, UC_MODE_64, UC_HOOK_CODE -from unicorn.x86_const import * - - -class ThunkTests(unittest.TestCase): - def check_abi(self, bits, fail=False, mode=1, message=0x8001, remove_success=True, unsafe=False): - size = bits // 8 - fmt = '`: stdCallback.probes, stdDate, stdHTTP.local, stdImage stdWebSocket.local or stdEnumerator.local respectively. Image host selection is now `--host Excel` or `--host Word` through the shared command (formerly `--hosts`). Test bodies and specialized harnesses now live under tests/harnesses. diff --git a/docs.json b/docs.json index 4b6b9122..980a9339 100644 --- a/docs.json +++ b/docs.json @@ -10954,22 +10954,7 @@ "todos": [ "Case \"callbyname\": evaluateFunc = CallByName(args(iArgStart))", "Switch caching and use of dictionary would be good here", - "Better error handling", - "iAccess_General", - "iSet_General", - "Test other functions", - "iReturn_WithValue - this is a utility used for returning values from functions, need to think how to test it properly", - "iReturn_NoValue - as above.", - "iObject_MethodCall", - "iObject_MethodCall_WithArgs", - "iObject_FieldCall as method", - "iObject_FieldCall_WithArgs as method", - "iObject_FieldCall as property", - "iObject_FieldCall_WithArgs as property", - "iObject_PropLet", - "iObject_PropLet_WithArgs", - "iObject_PropSet", - "iObject_PropSet_WithArgs" + "Better error handling" ], "requires": [], "methods": [ @@ -11179,25 +11164,6 @@ "throws": [], "requires": [], "isStatic": false - }, - { - "name": "protRunVMTests", - "description": "Runs tests on the class", - "remarks": [], - "examples": [], - "params": [], - "returns": null, - "deprecation": { - "status": false, - "message": "" - }, - "isDefaultMember": false, - "devNotes": [], - "todos": [], - "isProtected": false, - "throws": [], - "requires": [], - "isStatic": false } ], "properties": [ @@ -19513,4 +19479,4 @@ "events": [], "implements": [] } -] \ No newline at end of file +] diff --git a/src/WIP/stdWebSocket/README.md b/src/WIP/stdWebSocket/README.md index e5d9a4fe..ae99f9d4 100644 --- a/src/WIP/stdWebSocket/README.md +++ b/src/WIP/stdWebSocket/README.md @@ -86,4 +86,4 @@ Send and close have a native operation deadline. Connection send/response phases ## Verification -See [the local test runner](../../../Tools/stdWebSocket-tests/README.md). It uses disposable local servers and a separate hidden Excel instance. A native helper is necessary at runtime; no built DLL is committed. The x86 and x64 builds are both checked. Office runtime testing depends on the architecture installed on the test machine. +See [the local test runner](../../../tests/harnesses/stdWebSocket-tests/README.md). It uses disposable local servers and a separate hidden Excel instance. A native helper is necessary at runtime; no built DLL is committed. The x86 and x64 builds are both checked. Office runtime testing depends on the architecture installed on the test machine. diff --git a/src/WIP/stdWebSocket/native/build.ps1 b/src/WIP/stdWebSocket/native/build.ps1 index c457901e..d6e615ca 100644 --- a/src/WIP/stdWebSocket/native/build.ps1 +++ b/src/WIP/stdWebSocket/native/build.ps1 @@ -29,7 +29,7 @@ foreach ($architecture in @('x86','x64')) { $installer = Split-Path $vswhere $commands = "@echo off`r`nset `"PATH=$installer;%PATH%`"`r`ncall `"$dev`" -no_logo -arch=$architecture`r`nif errorlevel 1 exit /b 1`r`ncl /nologo /W4 /WX /O2 /LD /MT /Fo`"$obj`" `"$source`" /link winhttp.lib user32.lib oleaut32.lib uuid.lib /DEF:`"$definition`" /OUT:`"$dll`" /IMPLIB:`"$lib`"`r`nif errorlevel 1 exit /b 1`r`n" if ($WithTests) { - $test = (Resolve-Path (Join-Path $PSScriptRoot '../../../../Tools/stdWebSocket-tests/native_tests.c')).Path + $test = (Resolve-Path (Join-Path $PSScriptRoot '../../../../tests/harnesses/stdWebSocket-tests/native_tests.c')).Path $testObj = Join-Path $out 'native_tests.obj' $testExe = Join-Path $out 'native_tests.exe' $commands += "cl /nologo /W4 /WX /O2 /MT /Fo`"$testObj`" /Fe`"$testExe`" `"$test`" /link winhttp.lib user32.lib oleaut32.lib uuid.lib`r`n" diff --git a/src/stdLambda.cls b/src/stdLambda.cls index 63d84aff..9c4737dc 100644 --- a/src/stdLambda.cls +++ b/src/stdLambda.cls @@ -2371,235 +2371,3 @@ macJmpCall: Case 30: Call CopyVariant(macCallByName, CallByName(obj, funcName, callerType, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27, a28, a29)) End Select End Function - -'-------------------- -' Testing functions -'-------------------- - -'Creates an operation with the given instruction and value -'@param instruction - The instruction to use for the operation -'@param value - The value to use for the operation -'@returns - The created operation -Private Function testCreateOp(ByVal instruction As IInstruction, ByVal value As Variant) As Operation - Dim op As Operation - op.Instruction = instruction - Call CopyVariant(op.value, value) - testCreateOp = op -End Function - -'Creates an array of operations with the given parameters -'@param opsParams - A set of IInstruction/Value pairs to create operations from -'@returns - An array of created operations -Private Function testCreateOps(ParamArray opsParams()) As Operation() - Dim ops() As Operation - If (UBound(opsParams) + 1) Mod 2 <> 0 Then Call Throw("Expected an even number of parameters") - - Dim size As Long: size = (UBound(opsParams) + 1) \ 2 - ReDim ops(0 To size) - - Dim i As Long - For i = 0 To size - 1 - ops(i) = testCreateOp(opsParams(i * 2), opsParams(i * 2 + 1)) - Next i - - testCreateOps = ops -End Function - -'Runs tests on the class -'@returns - An array of booleans indicating whether each test passed -Public Sub protRunVMTests() - 'iPush - Debug.Assert evaluate(testCreateOps(iPush, 10), Array()) = 10 - - 'iPop - ' Cannot directly assert by return value of evaluate unless combined with other ops - ' To make it testable: Push 10, Push 20, Pop. If pop was a no-op, result would be 20. If pop works result should be 10. - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iPop, Empty), Array()) = 10 - - ' iMerge - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iMerge, Empty), Array()) = 20 - - 'TODO: iAccess_General - - ' iAccess_Argument - Debug.Assert evaluate(testCreateOps(iAccess_Argument, 1), Array("Hello")) = "Hello" - Debug.Assert evaluate(testCreateOps(iAccess_Argument, 1), Array("Hello", "World")) = "Hello" - Debug.Assert evaluate(testCreateOps(iAccess_Argument, 2), Array("Hello", "World")) = "World" - Debug.Assert evaluate(testCreateOps(iAccess_Argument, 1), Array(Application)) Is Application - - 'TODO: iSet_General - - ' Arithmetic operations - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iArithmetic_Add, Empty), Array()) = 30 - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iArithmetic_Subtract, Empty), Array()) = -10 - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iArithmetic_Multiply, Empty), Array()) = 200 - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iArithmetic_Divide, Empty), Array()) = 0.5 - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 3, iArithmetic_Power, Empty), Array()) = 1000 - Debug.Assert evaluate(testCreateOps(iPush, 20, iPush, 10, iArithmetic_Modulo, Empty), Array()) = 0 - Debug.Assert evaluate(testCreateOps(iPush, 20, iPush, 9, iArithmetic_Modulo, Empty), Array()) = 2 - Debug.Assert evaluate(testCreateOps(iPush, -20, iPush, 10, iArithmetic_Modulo, Empty), Array()) = 0 - Debug.Assert evaluate(testCreateOps(iPush, 10, iArithmetic_Negate, Empty), Array()) = -10 - - 'iLogic_And - 'A | 1 1 0 0 - 'B | 1 0 1 0 - '----------- - 'O | 1 0 0 0 - Debug.Assert evaluate(testCreateOps(iPush, True, iPush, True, iLogic_And, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, True, iPush, False, iLogic_And, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, False, iPush, True, iLogic_And, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, False, iPush, False, iLogic_And, Empty), Array()) = False - - 'iLogic_Or - 'A | 1 1 0 0 - 'B | 1 0 1 0 - '----------- - 'O | 1 1 1 0 - Debug.Assert evaluate(testCreateOps(iPush, True, iPush, True, iLogic_Or, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, True, iPush, False, iLogic_Or, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, False, iPush, True, iLogic_Or, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, False, iPush, False, iLogic_Or, Empty), Array()) = False - - 'iLogic_Not - 'A | 1 0 - '----------- - 'O | 0 1 - Debug.Assert evaluate(testCreateOps(iPush, True, iLogic_Not, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, False, iLogic_Not, Empty), Array()) = True - - 'iLogic_Xor - 'A | 1 1 0 0 - 'B | 1 0 1 0 - '----------- - 'O | 0 1 1 0 - Debug.Assert evaluate(testCreateOps(iPush, True, iPush, True, iLogic_Xor, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, True, iPush, False, iLogic_Xor, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, False, iPush, True, iLogic_Xor, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, False, iPush, False, iLogic_Xor, Empty), Array()) = False - - 'iFunc_Call - Use `Len` as an example - Debug.Assert evaluate(testCreateOps(iPush, "Len", iPush, "TestString", iPush, 1, iFunc_Call, Empty), Array()) = Len("TestString") - - 'Test oFunctExts - Set This.FunctionExtensions = CreateObject("Scripting.Dictionary") - This.FunctionExtensions.Add "Proxy", stdLambda.Create("$1") - Debug.Assert evaluate(testCreateOps(iPush, "Proxy", iPush, "TestString", iPush, 1, iFunc_Call, Empty), Array()) = "TestString" - 'Ensure returns objects too - Debug.Assert TypeName(evaluate(testCreateOps(iPush, "Proxy", iPush, Application, iPush, 1, iFunc_Call, Empty), Array())) = "Application" - Set This.FunctionExtensions = Nothing - - 'TODO: Test other functions - - 'Comparison operations - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_Equal, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iComparison_Equal, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_NotEqual, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iComparison_NotEqual, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iComparison_GreaterThan, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 9, iComparison_GreaterThan, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_GreaterThan, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 11, iComparison_GreaterThan, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 9, iComparison_GreaterThanOrEqual, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_GreaterThanOrEqual, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 11, iComparison_GreaterThanOrEqual, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 9, iComparison_LessThan, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_LessThan, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 11, iComparison_LessThan, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 9, iComparison_LessThanOrEqual, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_LessThanOrEqual, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 11, iComparison_LessThanOrEqual, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, "A", iPush, "B", iComparison_Like, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, "A", iPush, "A", iComparison_Like, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, "AB", iPush, "A*", iComparison_Like, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, "ABC", iPush, "A*", iComparison_Like, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, "AB", iPush, "A?", iComparison_Like, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, "ABC", iPush, "A?", iComparison_Like, Empty), Array()) = False - - Dim o1 As Collection: Set o1 = New Collection - Dim o2 As Collection: Set o2 = New Collection - Dim o3 As Collection - Debug.Assert evaluate(testCreateOps(iPush, o1, iPush, o2, iComparison_Is, Empty), Array()) = False - Debug.Assert evaluate(testCreateOps(iPush, o1, iPush, o1, iComparison_Is, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, Nothing, iPush, Nothing, iComparison_Is, Empty), Array()) = True - Debug.Assert evaluate(testCreateOps(iPush, o3, iPush, Nothing, iComparison_Is, Empty), Array()) = True - - 'Miscellaneous operations - Debug.Assert evaluate(testCreateOps(iPush, "Hello", iPush, " World", iMisc_Concatenate, Empty), Array()) = "Hello World" - - 'Jump instructions - 'These change instruction pointer, need careful sequencing for observable results - 'iJump_Unconditional - '00: Push 10 - '01: Jump to 3 - '02: Push 20 (skipped) - '03: - 'Should return 10 - Debug.Assert evaluate(testCreateOps(iPush, 10, iJump_Unconditional, 3, iPush, 20), Array()) = 10 - - 'iJump_IfTrue - '00 Push True - '01 JumpIfTrue 4 - '02 Push 10 (skipped) - '03 JumpUnconditional 5 (skipped) - '04 Push 20 - '05 - 'Should return 20. - Debug.Assert evaluate(testCreateOps(iPush, True, iJump_IfTrue, 4, iPush, 10, iJump_Unconditional, 5, iPush, 20), Array()) = 20 - - 'Counter-example for iJump_IfTrue - '00 Push False - '01 JumpIfTrue 4 - '02 Push 10 - '03 JumpUnconditional 5 - '04 Push 20 (skipped) - '05 - 'Should return 10. - Debug.Assert evaluate(testCreateOps(iPush, False, iJump_IfTrue, 4, iPush, 10, iJump_Unconditional, 5, iPush, 20), Array()) = 10 - - 'iJump_IfFalse (inverse of the above) - Debug.Assert evaluate(testCreateOps(iPush, False, iJump_IfFalse, 4, iPush, 10, iJump_Unconditional, 5, iPush, 20), Array()) = 20 - Debug.Assert evaluate(testCreateOps(iPush, True, iJump_IfFalse, 4, iPush, 10, iJump_Unconditional, 5, iPush, 20), Array()) = 10 - - 'TODO: iReturn_WithValue - this is a utility used for returning values from functions, need to think how to test it properly - 'TODO: iReturn_NoValue - as above. - - - 'iObject_PropGet - 'TEST1 - With object return value - '00: Push Application 'Application _ - '01: Push "ThisWorkbook" ' .ThisWorkbook - '02: Object_PropGet - 'Should return `ThisWorkbook` - Debug.Assert evaluate(testCreateOps(iPush, Application, iPush, "ThisWorkbook", iObject_PropGet, Empty), Array()) Is ThisWorkbook - - 'TEST2 - With literal return value - '00: Push Application 'Application _ - '01: Push "ThisWorkbook" ' .Name - '02: Object_PropGet - 'Should return Application.Name e.g. "Microsoft Excel" - Debug.Assert evaluate(testCreateOps(iPush, Application, iPush, "Name", iObject_PropGet, Empty), Array()) = Application.Name - - 'TEST3 - With parameters - '00: Push Application 'Application _ - '01: Push "Range" ' .Range( _ - '02: Push "A1" ' "A1") - '03: Push 1 '(1 arg) - '04: Object_PropGet - 'Should return `Range("A1")` but note these won't return the same objects so `is` will still fail. - 'Instead we check the type and address. - Debug.Assert TypeName(evaluate(testCreateOps(iPush, Application, iPush, "Range", iPush, "A1", iPush, 1, iObject_PropGet, Empty), Array())) = "Range" - Debug.Assert evaluate(testCreateOps(iPush, Application, iPush, "Range", iPush, "A1", iPush, 1, iObject_PropGet, Empty), Array()).Address = "$A$1" - - 'TODO: iObject_MethodCall - 'TODO: iObject_MethodCall_WithArgs - 'TODO: iObject_FieldCall as method - 'TODO: iObject_FieldCall_WithArgs as method - 'TODO: iObject_FieldCall as property - 'TODO: iObject_FieldCall_WithArgs as property - 'TODO: iObject_PropLet - 'TODO: iObject_PropLet_WithArgs - 'TODO: iObject_PropSet - 'TODO: iObject_PropSet_WithArgs -End Sub - - diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..a0919a9c --- /dev/null +++ b/tests/README.md @@ -0,0 +1,24 @@ +# Testing stdVBA + +Use [the shared runner](../Tools/Testing/README.md) for automated execution. `manifest.json` is the explicit inventory: it lists dependencies, independent cases, supported hosts, requirements and default inclusion. Root `*Tests.bas` modules remain in their established locations so existing workbook imports keep working. Test-only probes, recorders, fixture servers and specialized harnesses live in `harnesses/`; shared support lives in `support/`; Python runner/build tests live in `tooling/`, and native ABI tests in `native/`. + +## Adding or changing a test + +1. Keep behavioural assertions outside production classes. Put tests for a class in its existing `tests/stdTests.bas` module, or a new module in this folder. Preserve documentation examples in production source. +2. Prefer small public, parameterless test procedures with stable case IDs. Register each procedure explicitly in `manifest.json`; arbitrary public helpers are not discovered as tests. Make every case establish its own state and clean up its resources. Keep `testAll` wrappers for existing manual callers where useful. +3. List only required production sources and test helpers. Test framework dependencies belong to scratch projects, never to distributed production classes. +4. Use `Test.Assert label, condition`. The shared reporter counts failures without raising, so legacy `On Error Resume Next` cannot swallow failed assertions. Unexpected errors reaching the bridge fail the case; zero assertions also fail. Existing returned PASS/count/Boolean contracts have explicit adapters. Avoid new broad error suppression: it can still conceal an unexpected error before any assertion. +5. Mark host, platform and fixture requirements honestly. Native crash probes run in isolated processes and remain opt-in. A crash is a failure, never an expected pass. Keep benchmarks separate from correctness cases. +6. Check actual CRLF bytes before every import, including generated helpers. Preserve encodings. Run the selected case, its class suite, and the appropriate aggregate checks. + +## Private internals and existing suites + +Lambda VM tests now live in `support/stdLambdaVM.cls.inc`. The runner injects this fragment into a temporary stdLambda copy and converts Debug.Assert into counted assertions. The production class has no test dependency. Use `--suite stdLambda.vm`; the former production `protRunVMTests` test entry point has been removed. + +Date cache tests similarly instrument a disposable source copy. WIP classes remain experimental; existing compile or behaviour failures are reported, not patched around by the runner. + +Some legacy suites remain aggregate cases; split those as they are maintained. Callback has ModuleCalls, Bind and PointerCalls; Regex has Contract, NativeEngine, CaptureArraysAndLists and MatchAllCol. Root `Main.bas` and the worksheet/Immediate Window reporters in `lib/` are legacy manual-workbook tools. They are not the authoritative complete inventory. New automated suites use `support/Test.cls`. + +Access host automation is not implemented. The old Clipboard suite requires a workbook-specific worksheet builder, contains hard-coded paths and documents known failures; it is explicitly unavailable in the automated inventory. The old WebSocket module is commented-out placeholder code; use `stdWebSocket.local`. The historical Regex thunk comparison also requires removed experimental CreateVBA/CreateNative APIs and is explicitly unavailable; current Regex cases cover the distributed class. These are visible skips under `--all`, not passing coverage. Existing WIP prototype classes can contain exploratory test methods; they are not production regression coverage. + +Run the runner's own contract tests without Office with `python -B tests/tooling/test_runner.py`. diff --git a/Tools/stdCallback-tests/Probe.bas b/tests/harnesses/stdCallback-tests/Probe.bas similarity index 100% rename from Tools/stdCallback-tests/Probe.bas rename to tests/harnesses/stdCallback-tests/Probe.bas diff --git a/Tools/stdCallback-tests/README.md b/tests/harnesses/stdCallback-tests/README.md similarity index 95% rename from Tools/stdCallback-tests/README.md rename to tests/harnesses/stdCallback-tests/README.md index f2543d98..8ff21d24 100644 --- a/Tools/stdCallback-tests/README.md +++ b/tests/harnesses/stdCallback-tests/README.md @@ -1,6 +1,6 @@ # Issue 85 object argument probes -Run `python Tools/stdCallback-tests/run.py` from the repository root with Windows, +Run `python tests/harnesses/stdCallback-tests/run.py` from the repository root with Windows, Excel, Python and pywin32 installed. Each scenario uses a separate disposable Excel instance. Do not combine the ByRef probes into the ordinary test suite: passing an object value where a pointer to an object variable is required can diff --git a/Tools/stdCallback-tests/run.py b/tests/harnesses/stdCallback-tests/run.py similarity index 97% rename from Tools/stdCallback-tests/run.py rename to tests/harnesses/stdCallback-tests/run.py index afd4521d..febd5779 100644 --- a/Tools/stdCallback-tests/run.py +++ b/tests/harnesses/stdCallback-tests/run.py @@ -6,8 +6,8 @@ import threading import winreg -ROOT = Path(__file__).resolve().parents[2] -FILES = ['src/stdICallable.cls', 'src/stdCallback.cls', 'Tools/stdCallback-tests/Probe.bas'] +ROOT = Path(__file__).resolve().parents[3] +FILES = ['src/stdICallable.cls', 'src/stdCallback.cls', 'tests/harnesses/stdCallback-tests/Probe.bas'] NAMES = ['Object ByVal inferred', 'Object ByVal explicit', 'Object ByRef inferred', 'Object ByRef explicit', 'Object ByRef VarPtr', 'Variant ByVal explicit', 'Collection ByVal inferred', 'Object ByVal RunEx', 'Nothing ByVal', 'Dictionary ByVal'] diff --git a/tests/harnesses/stdDate-test/README.md b/tests/harnesses/stdDate-test/README.md new file mode 100644 index 00000000..d2fbefc4 --- /dev/null +++ b/tests/harnesses/stdDate-test/README.md @@ -0,0 +1,5 @@ +Run `powershell -NoProfile -File tests/harnesses/stdDate-test/run-tests.ps1` from this worktree. + +The compatibility command now delegates to `python Tools/Testing/run.py --suite stdDate`. The shared lifecycle uses a disposable hidden Excel instance, monitors modal dialogs, checks CRLF before import, and restores exact VBA project-access and error-trapping settings. A temporary class copy reports cache dictionary counts; it is discarded after the run. Production source is not instrumented. Tests cover weekday defaults/names and regex cache behaviour. stdDate remains experimental. + +2026-10-31: execution blocked at VBProject access by the existing trust setting. No VBA assertions ran and compilation was not verified. The WIP class also contains a pre-existing Exit Function inside the isLeap Property Get; this work does not repair unrelated WIP functionality. diff --git a/Tools/stdDate-test/TestRunner.bas b/tests/harnesses/stdDate-test/TestRunner.bas similarity index 100% rename from Tools/stdDate-test/TestRunner.bas rename to tests/harnesses/stdDate-test/TestRunner.bas diff --git a/tests/harnesses/stdDate-test/run-tests.ps1 b/tests/harnesses/stdDate-test/run-tests.ps1 new file mode 100644 index 00000000..21eea2b7 --- /dev/null +++ b/tests/harnesses/stdDate-test/run-tests.ps1 @@ -0,0 +1,2 @@ +& python (Join-Path $PSScriptRoot "../../../Tools/Testing/run.py") --suite stdDate @args +exit $LASTEXITCODE diff --git a/Tools/stdEnumerator-tests/Probe.bas b/tests/harnesses/stdEnumerator-tests/Probe.bas similarity index 100% rename from Tools/stdEnumerator-tests/Probe.bas rename to tests/harnesses/stdEnumerator-tests/Probe.bas diff --git a/Tools/stdEnumerator-tests/README.md b/tests/harnesses/stdEnumerator-tests/README.md similarity index 94% rename from Tools/stdEnumerator-tests/README.md rename to tests/harnesses/stdEnumerator-tests/README.md index 331d90da..fc5c7b25 100644 --- a/Tools/stdEnumerator-tests/README.md +++ b/tests/harnesses/stdEnumerator-tests/README.md @@ -1,6 +1,6 @@ # stdEnumerator Excel regressions -Run `python Tools/stdEnumerator-tests/run.py` on Windows with Excel and pywin32. +Run `python tests/harnesses/stdEnumerator-tests/run.py` on Windows with Excel and pywin32. The runner uses a separate hidden Excel process and disposable workbook. It checks actual bytes for CRLF before every import, monitors only its own Office windows, dismisses dialogs as failures, and bounds worker execution. It discovers the diff --git a/Tools/stdEnumerator-tests/Test.cls b/tests/harnesses/stdEnumerator-tests/Test.cls similarity index 100% rename from Tools/stdEnumerator-tests/Test.cls rename to tests/harnesses/stdEnumerator-tests/Test.cls diff --git a/Tools/stdEnumerator-tests/run.py b/tests/harnesses/stdEnumerator-tests/run.py similarity index 98% rename from Tools/stdEnumerator-tests/run.py rename to tests/harnesses/stdEnumerator-tests/run.py index 874ace14..53d3acea 100644 --- a/Tools/stdEnumerator-tests/run.py +++ b/tests/harnesses/stdEnumerator-tests/run.py @@ -15,7 +15,7 @@ import winreg sys.dont_write_bytecode = True -ROOT = Path(__file__).resolve().parents[2] +ROOT = Path(__file__).resolve().parents[3] USER = ctypes.WinDLL("user32", use_last_error=True) USER.GetWindowThreadProcessId.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.DWORD)] USER.ShowWindow.argtypes = [wintypes.HWND, ctypes.c_int] @@ -89,7 +89,7 @@ def worker(host, scratch, phase, url): count = project.VBComponents.Count zone = doc.Worksheets(1) project.VBComponents(zone.CodeName).Name = "TestZone" - for relative in ("src/stdICallable.cls", "src/stdEnumerator.cls", "src/stdCallback.cls", "src/stdLambda.cls", "Tools/stdEnumerator-tests/Test.cls", "tests/stdEnumeratorTests.bas", "Tools/stdEnumerator-tests/Probe.bas"): + for relative in ("src/stdICallable.cls", "src/stdEnumerator.cls", "src/stdCallback.cls", "src/stdLambda.cls", "tests/harnesses/stdEnumerator-tests/Test.cls", "tests/stdEnumeratorTests.bas", "tests/harnesses/stdEnumerator-tests/Probe.bas"): raw = (ROOT / relative).read_bytes() if b"\n" in raw.replace(b"\r\n", b"") or b"\r" in raw.replace(b"\r\n", b""): raise RuntimeError("CRLF required before import: " + relative) diff --git a/Tools/stdHTTP-tests/fixture.py b/tests/harnesses/stdHTTP-tests/fixture.py similarity index 100% rename from Tools/stdHTTP-tests/fixture.py rename to tests/harnesses/stdHTTP-tests/fixture.py diff --git a/Tools/stdHTTP-tests/run.ps1 b/tests/harnesses/stdHTTP-tests/run.ps1 similarity index 100% rename from Tools/stdHTTP-tests/run.ps1 rename to tests/harnesses/stdHTTP-tests/run.ps1 diff --git a/Tools/stdHTTP-tests/run.py b/tests/harnesses/stdHTTP-tests/run.py similarity index 98% rename from Tools/stdHTTP-tests/run.py rename to tests/harnesses/stdHTTP-tests/run.py index 80af8fc9..0aed32b3 100644 --- a/Tools/stdHTTP-tests/run.py +++ b/tests/harnesses/stdHTTP-tests/run.py @@ -15,7 +15,7 @@ from http.server import ThreadingHTTPServer from fixture import Handler -root = Path(__file__).resolve().parents[2] +root = Path(__file__).resolve().parents[3] for name in ('src/stdICallable.cls', 'src/stdHTTP.cls', 'tests/stdHTTPLocalTests.bas'): raw = (root / name).read_bytes().replace(b'\r\n', b'') if b'\n' in raw or b'\r' in raw: diff --git a/Tools/stdHTTP-tests/worker.ps1 b/tests/harnesses/stdHTTP-tests/worker.ps1 similarity index 97% rename from Tools/stdHTTP-tests/worker.ps1 rename to tests/harnesses/stdHTTP-tests/worker.ps1 index fa13f067..5af9ecb1 100644 --- a/Tools/stdHTTP-tests/worker.ps1 +++ b/tests/harnesses/stdHTTP-tests/worker.ps1 @@ -1,6 +1,6 @@ param([string]$PidFile) $ErrorActionPreference = 'Stop' -$root = (Resolve-Path "$PSScriptRoot/../..").Path +$root = (Resolve-Path "$PSScriptRoot/../../..").Path Add-Type @" using System; using System.Runtime.InteropServices; diff --git a/Tools/stdImage-tests/README.md b/tests/harnesses/stdImage-tests/README.md similarity index 96% rename from Tools/stdImage-tests/README.md rename to tests/harnesses/stdImage-tests/README.md index ef1ca3b6..06d11680 100644 --- a/Tools/stdImage-tests/README.md +++ b/tests/harnesses/stdImage-tests/README.md @@ -3,8 +3,8 @@ Run from the repository root on Windows with Microsoft Office, Python, pywin32 and Pillow installed: ```powershell -python Tools/stdImage-tests/run.py -python Tools/stdImage-tests/run.py --hosts Excel +python tests/harnesses/stdImage-tests/run.py +python tests/harnesses/stdImage-tests/run.py --hosts Excel ``` The runner creates disposable, hidden Excel and Word instances and refuses to use a process that already existed. It imports only stdImage and the standalone stdImageTests module, checking actual CRLF bytes before each import. An independent parent watches the owned process for modal dialogs, captures their text, dismisses them and minimises its Office/VBE windows; any dialog, timeout or failed assertion fails the run. diff --git a/Tools/stdImage-tests/run.py b/tests/harnesses/stdImage-tests/run.py similarity index 99% rename from Tools/stdImage-tests/run.py rename to tests/harnesses/stdImage-tests/run.py index 177a3a35..58cd3c72 100644 --- a/Tools/stdImage-tests/run.py +++ b/tests/harnesses/stdImage-tests/run.py @@ -19,7 +19,7 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer sys.dont_write_bytecode = True -ROOT = Path(__file__).resolve().parents[2] +ROOT = Path(__file__).resolve().parents[3] USER = ctypes.WinDLL("user32", use_last_error=True) USER.GetWindowThreadProcessId.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.DWORD)] USER.ShowWindow.argtypes = [wintypes.HWND, ctypes.c_int] diff --git a/Tools/stdRegex-build/Benchmark.bas b/tests/harnesses/stdRegex-build/Benchmark.bas similarity index 100% rename from Tools/stdRegex-build/Benchmark.bas rename to tests/harnesses/stdRegex-build/Benchmark.bas diff --git a/Tools/stdRegex-build/TestReporter.cls b/tests/harnesses/stdRegex-build/TestReporter.cls similarity index 100% rename from Tools/stdRegex-build/TestReporter.cls rename to tests/harnesses/stdRegex-build/TestReporter.cls diff --git a/Tools/stdRegex-build/TestRunner.bas b/tests/harnesses/stdRegex-build/TestRunner.bas similarity index 100% rename from Tools/stdRegex-build/TestRunner.bas rename to tests/harnesses/stdRegex-build/TestRunner.bas diff --git a/tests/harnesses/stdRegex-build/benchmark.ps1 b/tests/harnesses/stdRegex-build/benchmark.ps1 new file mode 100644 index 00000000..dc56b12b --- /dev/null +++ b/tests/harnesses/stdRegex-build/benchmark.ps1 @@ -0,0 +1,84 @@ +param( + [string]$BaselineRef = 'c02d86fd7e5c99b648f9729d7a0c6160d492a642', + [ValidateSet('Excel', 'Word')][string]$HostApplication = 'Word', + [ValidateRange(3, 31)][int]$Samples = 7, + [ValidateRange(0.01, 1.0)][double]$TargetSeconds = 0.05, + [string]$OutputDir = (Join-Path $PSScriptRoot '../../../Tools/stdRegex-build/benchmarks/latest') +) + +# The baseline is the pure-VBA stdRegex before the list/array changes, not the +# older VBScript wrapper. Both classes run in the same hidden Office process. +$ErrorActionPreference = 'Stop' +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..\..\..')).Path +$baselineCommit = (& git -C $repoRoot rev-parse --verify --end-of-options ($BaselineRef + '^{commit}')).Trim() +if ($LASTEXITCODE -ne 0) { throw 'Cannot resolve baseline commit.' } +$baselineLines = & git -C $repoRoot show ($baselineCommit + ':src/stdRegex.cls') +if ($LASTEXITCODE -ne 0) { throw 'Cannot read baseline class.' } +$baseline = [regex]::Replace(($baselineLines -join "`r`n"), '\bstdRegex\b', 'stdRegexBaseline') + "`r`n" +$tempRoot = [System.IO.Path]::GetFullPath([System.IO.Path]::GetTempPath()) +$workDir = Join-Path $tempRoot ('stdRegex-benchmark-' + [guid]::NewGuid().ToString('N')) +$null = New-Item -ItemType Directory -Path $workDir +$baselinePath = Join-Path $workDir 'stdRegexBaseline.cls' +[System.IO.File]::WriteAllText($baselinePath, $baseline, [System.Text.UTF8Encoding]::new($false)) +$office = $null +$document = $null +$rows = [System.Collections.Generic.List[object]]::new() +try { + $office = New-Object -ComObject ($HostApplication + '.Application') + $office.Visible = $false + $office.DisplayAlerts = 0 + if ($HostApplication -eq 'Excel') { $document = $office.Workbooks.Add() } + else { $document = $office.Documents.Add() } + $project = $document.GetType().InvokeMember('VBProject', [System.Reflection.BindingFlags]::GetProperty, $null, $document, $null) + foreach ($path in @($baselinePath, (Join-Path $repoRoot 'src\stdRegex.cls'), (Join-Path $PSScriptRoot 'Benchmark.bas'))) { + $null = $project.VBComponents.Import($path) + } + $prefix = if ($HostApplication -eq 'Excel') { "'" + $document.Name + "'!" } else { '' } + $officeVersion = $office.Version + $vbaArchitecture = $office.Run($prefix + 'StdRegexBenchmark.HostDetails') + for ($caseId = 0; $caseId -lt 6; $caseId++) { + $values = $office.Run($prefix + 'StdRegexBenchmark.RunCase', $caseId, $Samples, $TargetSeconds) + if ($values -is [string]) { throw $values } + for ($i = 1; $i -le $values.GetLength(0); $i++) { + $rows.Add([pscustomobject]@{ + case = $values[$i, 1]; matches = [int]$values[$i, 2]; captures = [int]$values[$i, 3] + input_characters = [int]$values[$i, 4]; operation = $values[$i, 5] + sample = [int]$values[$i, 6]; iterations = [int]$values[$i, 7] + elapsed_ms = [double]$values[$i, 8]; ms_per_call = [double]$values[$i, 9] + }) + } + Write-Output ('Measured: ' + $values[1, 1]) + } + $metadata = [ordered]@{ + measured_at_utc = [DateTime]::UtcNow.ToString('o') + host_application = $HostApplication; office_version = $officeVersion; vba_architecture = $vbaArchitecture + operating_system = [System.Environment]::OSVersion.VersionString + baseline_commit = $baselineCommit + current_class_sha256 = (Get-FileHash -LiteralPath (Join-Path $repoRoot 'src\stdRegex.cls') -Algorithm SHA256).Hash + engine = (Get-Content -LiteralPath (Join-Path $PSScriptRoot 'engine.json') -Raw | ConvertFrom-Json) + samples = $Samples; minimum_batch_seconds = $TargetSeconds + methodology = 'QueryPerformanceCounter; compiled regexes reused; 2 warm-up calls; calibrated batches; rotated operation order; includes result allocation and release; output equality checked before timing.' + } + $null = New-Item -ItemType Directory -Path $OutputDir -Force + $metadata | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $OutputDir 'environment.json') -Encoding utf8 + $rows | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $OutputDir 'samples.json') -Encoding utf8 + $rows | Export-Csv -LiteralPath (Join-Path $OutputDir 'samples.csv') -NoTypeInformation -Encoding utf8 + & python (Join-Path $PSScriptRoot 'summarize-benchmark.py') $OutputDir + if ($LASTEXITCODE -ne 0) { throw 'Benchmark summarization failed.' } +} finally { + try { + if ($null -ne $document) { + if ($HostApplication -eq 'Excel') { $document.Close($false) } + else { $document.Close(0) } + } + } finally { + try { if ($null -ne $office) { $office.Quit() } } + finally { + $resolvedWork = [System.IO.Path]::GetFullPath($workDir) + $resolvedTemp = [System.IO.Path]::GetFullPath($tempRoot).TrimEnd('\', '/') + [System.IO.Path]::DirectorySeparatorChar + if (-not $resolvedWork.StartsWith($resolvedTemp, [System.StringComparison]::OrdinalIgnoreCase) -or + -not ([System.IO.Path]::GetFileName($resolvedWork)).StartsWith('stdRegex-benchmark-')) { throw 'Unsafe scratch cleanup path.' } + Remove-Item -LiteralPath $resolvedWork -Recurse -Force + } + } +} diff --git a/tests/harnesses/stdRegex-build/run-tests.ps1 b/tests/harnesses/stdRegex-build/run-tests.ps1 new file mode 100644 index 00000000..7f5cd3f7 --- /dev/null +++ b/tests/harnesses/stdRegex-build/run-tests.ps1 @@ -0,0 +1,2 @@ +& (Join-Path $PSScriptRoot "../../../Tools/stdRegex-build/run-tests.ps1") @args +exit $LASTEXITCODE diff --git a/tests/harnesses/stdRegex-build/summarize-benchmark.py b/tests/harnesses/stdRegex-build/summarize-benchmark.py new file mode 100644 index 00000000..4a775f3f --- /dev/null +++ b/tests/harnesses/stdRegex-build/summarize-benchmark.py @@ -0,0 +1,48 @@ +"""Summarize measured per-call samples; no synthetic timings.""" +from collections import defaultdict +import json +from pathlib import Path +from statistics import median +import sys + + +def summarize(folder): + samples = json.loads((folder / 'samples.json').read_text(encoding='utf-8-sig')) + environment = json.loads((folder / 'environment.json').read_text(encoding='utf-8-sig')) + groups = defaultdict(list) + for row in samples: + groups[(row['case'], row['operation'])].append(row['ms_per_call']) + cases = [] + for case in dict.fromkeys(row['case'] for row in samples): + timing = {} + for (label, op), values in groups.items(): + if label == case: + timing[op] = {'median_ms': median(values), 'min_ms': min(values), 'max_ms': max(values)} + comparisons = [] + for before, after in [('List before', 'List after'), ('ListArr before', 'ListArr after'), ('MatchAll', 'MatchAllArr')]: + old, new = timing[before]['median_ms'], timing[after]['median_ms'] + comparisons.append({'before': before, 'after': after, 'before_ms': old, 'after_ms': new, + 'speedup': old / new, 'time_reduction_percent': 100 * (old - new) / old}) + cases.append({'case': case, 'timings': timing, 'comparisons': comparisons}) + result = {'environment': environment, 'cases': cases} + (folder / 'summary.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') + lines = ['# stdRegex before/after benchmark', '', + f"Host: {environment['host_application']} {environment['office_version']}, {environment['vba_architecture']}.", + f"Measured UTC: {environment['measured_at_utc']}.", + f"Baseline: `{environment['baseline_commit']}` (pure VBA engine, before list optimizations).", '', + f"{environment['samples']} samples per operation/case; medians in milliseconds per call. " + environment['methodology'], '', + 'ListArr uses two output columns. MatchAllArr materializes all numbered captures; MatchAll additionally creates dictionaries and named entries. Their contracts differ.', ''] + for case in cases: + lines += ['## ' + case['case'], ''] + for item in case['comparisons']: + change = item['time_reduction_percent'] + direction = 'less time' if change >= 0 else 'more time' + lines.append(f"- {item['before']} → {item['after']}: **{item['before_ms']:.3f} → {item['after_ms']:.3f} ms**, {item['speedup']:.2f}× relative speed ({abs(change):.1f}% {direction}).") + lines.append('') + lines += ['These measurements describe this host and these workloads. Sparse/no-match workloads spend more time in the unchanged search engine, so wrapper-only improvements may be small or negative.', ''] + (folder / 'results.md').write_text('\n'.join(lines), encoding='utf-8') + print('\n'.join(lines).replace('\u2192', '->').replace('\u00d7', 'x')) + + +if __name__ == '__main__': + summarize(Path(sys.argv[1])) diff --git a/Tools/stdRegex-thunk/Benchmark.bas b/tests/harnesses/stdRegex-thunk/Benchmark.bas similarity index 100% rename from Tools/stdRegex-thunk/Benchmark.bas rename to tests/harnesses/stdRegex-thunk/Benchmark.bas diff --git a/tests/harnesses/stdRegex-thunk/run.py b/tests/harnesses/stdRegex-thunk/run.py new file mode 100644 index 00000000..9c06d10a --- /dev/null +++ b/tests/harnesses/stdRegex-thunk/run.py @@ -0,0 +1,193 @@ +"""Disposable Word x64 test host with bounded watchdog and exact registry cleanup. + +Requires Python/pywin32 and desktop Word. Run from any directory. No user Office +instance is attached to or closed. The supervisor owns settings and timeout. +""" +from pathlib import Path +import ctypes +import hashlib +import json +import os +import re +import subprocess +import sys +import tempfile +import time +import winreg + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[2] + +def write(path, text): + path.write_bytes(text.replace('\r\n', '\n').replace('\n', '\r\n').encode('utf-8')) + +def worker(folder, test_only=False): + import win32com.client + import win32process + app = doc = None + result = {'measured_at_utc': __import__('datetime').datetime.now(__import__('datetime').timezone.utc).isoformat()} + try: + app = win32com.client.DispatchEx('Word.Application') + app.Visible = False + app.DisplayAlerts = 0 + doc = app.Documents.Add() + pid = win32process.GetWindowThreadProcessId(doc.ActiveWindow.Hwnd)[1] + (folder/'pid').write_text(str(pid)) + result['office_version'] = app.Version + result['office_build'] = app.Build + result['cpu'] = __import__('platform').processor() + result['windows'] = __import__('platform').platform() + project = doc.VBProject # Check actual application's project access. + paths = [folder/'stdRegex.cls', + ROOT/'tests/harnesses/stdRegex-build/TestReporter.cls', + ROOT/'tests/harnesses/stdRegex-build/TestRunner.bas', folder/'stdRegexTests.bas', + folder/'ThunkTests.bas', folder/'ThunkTestRunner.bas', HERE/'Benchmark.bas'] + for path in paths: + data = path.read_bytes() + assert b'\n' not in data.replace(b'\r\n', b'') and b'\r' not in data.replace(b'\r\n', b''), f'Not CRLF: {path}' + project.VBComponents.Import(str(path)) + for name in ['ThunkBenchmark.Differential','StdRegexTestRunner.RunTests','ThunkTestRunner.RunTests']: + print('Running '+name, flush=True) + value = app.Run(name) + result[name] = value + if isinstance(value, str) or value <= 0: + raise RuntimeError(f'{name}: {value}') + result['samples'] = [] + for case in range(0 if test_only else 6): + print('Benchmark case '+str(case), flush=True) + rows = app.Run('ThunkBenchmark.RunCase', case, 7, 0.05) + if isinstance(rows, str): raise RuntimeError(rows) + for row in rows: + result['samples'].append(dict(zip(['case','matches','captures','characters','operation','sample','iterations','elapsed_ms','ms_per_call'],row))) + result['status'] = 'passed' + except Exception as exc: + result['status'] = 'failed' + result['error'] = repr(exc) + raise + finally: + write(folder/'result.json', json.dumps(result,indent=2)+'\n') + try: + if doc is not None: doc.Close(0) + finally: + if app is not None: app.Quit(0) + +def supervise(test_only=False): + import win32gui + import win32process + import win32con + import psutil + # Refuse managed AccessVBOM settings rather than overriding policy. + with winreg.OpenKey(winreg.HKEY_CLASSES_ROOT, r'Word.Application\CurVer') as key: + version = winreg.QueryValueEx(key, '')[0].rsplit('.',1)[1]+'.0' + for hive in (winreg.HKEY_CURRENT_USER,winreg.HKEY_LOCAL_MACHINE): + try: + with winreg.OpenKey(hive,rf'Software\Policies\Microsoft\Office\{version}\Word\Security') as key: + value,_ = winreg.QueryValueEx(key,'AccessVBOM') + if not value: raise RuntimeError('Managed policy disables VBA project access') + except FileNotFoundError: pass + settings = [(rf'Software\Microsoft\Office\{version}\Word\Security','AccessVBOM',1), + (r'Software\Microsoft\VBA\7.1\Common','BreakOnAllErrors',0), + (r'Software\Microsoft\VBA\7.1\Common','BreakOnServerErrors',0)] + saved=[] + output=ROOT/'Tools/stdRegex-thunk'/('verification-x64' if test_only else 'results') + output.mkdir(exist_ok=True) + started=time.time() + original_pids={p.pid for p in psutil.process_iter()} + dialogs=[] + process=None + office_pid=None + with tempfile.TemporaryDirectory(prefix='stdRegex-thunk-test-') as tmp: + folder=Path(tmp) + combined=(ROOT/'src/stdRegex.cls').read_text(encoding='utf-8') + combined += '\nPublic Function CreateNative(ByVal pattern As String, Optional ByVal flags As String = "", Optional ByVal stepsLimit As Long = 10000) As stdRegex\n Set CreateNative = Create(pattern, flags, stepsLimit, True)\nEnd Function\n' + combined += '\nPublic Function CreateVBA(ByVal pattern As String, Optional ByVal flags As String = "", Optional ByVal stepsLimit As Long = 10000) As stdRegex\n Set CreateVBA = Create(pattern, flags, stepsLimit, False)\nEnd Function\n' + write(folder/'stdRegex.cls', combined) + baseline_tests=(ROOT/'tests/stdRegexTests.bas').read_text(encoding='utf-8') + write(folder/'stdRegexTests.bas', baseline_tests.replace('stdRegex.Create(', 'stdRegex.CreateVBA(')) + source=(ROOT/'tests/stdRegexTests.bas').read_text(encoding='utf-8') + source=source.replace('stdRegex.Create(', 'stdRegex.CreateNative(') + source=re.sub(r'\bstdRegexTests\b','ThunkTests',source) + write(folder/'ThunkTests.bas',source) + source=(ROOT/'tests/harnesses/stdRegex-build/TestRunner.bas').read_text(encoding='utf-8') + write(folder/'ThunkTestRunner.bas',source.replace('StdRegexTestRunner','ThunkTestRunner').replace('stdRegexTests','ThunkTests')) + try: + for path,name,value in settings: + try: + key=winreg.OpenKey(winreg.HKEY_CURRENT_USER,path,0,winreg.KEY_READ|winreg.KEY_WRITE) + existed=True + except FileNotFoundError: + key=winreg.CreateKey(winreg.HKEY_CURRENT_USER,path); existed=False + with key: + try: old,kind=winreg.QueryValueEx(key,name); present=True + except FileNotFoundError: old=kind=None; present=False + saved.append(dict(path=path,name=name,key_existed=existed,present=present,value=old,kind=kind)) + winreg.SetValueEx(key,name,0,winreg.REG_DWORD,value) + write(output/'settings-before.json',json.dumps(saved,indent=2)+'\n') + process=subprocess.Popen([sys.executable,str(Path(__file__)), '--worker',str(folder)]+(['--test-only'] if test_only else [])) + deadline=time.monotonic()+240 + while process.poll() is None: + if (folder/'pid').exists(): office_pid=int((folder/'pid').read_text()) + if office_pid and office_pid not in original_pids: + def visit(hwnd,_): + if win32process.GetWindowThreadProcessId(hwnd)[1]!=office_pid: return + if not win32gui.IsWindowVisible(hwnd): return + klass=win32gui.GetClassName(hwnd) + if klass=='#32770': + texts=[]; buttons=[] + def child(ch,_): + texts.append(win32gui.GetWindowText(ch)) + if win32gui.GetClassName(ch)=='Button': buttons.append(ch) + win32gui.EnumChildWindows(hwnd,child,None) + dialogs.append(' | '.join(texts)) + if buttons: win32gui.PostMessage(buttons[0],win32con.BM_CLICK,0,0) + else: win32gui.PostMessage(hwnd,win32con.WM_CLOSE,0,0) + ctypes.windll.user32.ShowWindowAsync(hwnd,6) + win32gui.EnumWindows(visit,None) + if time.monotonic()>deadline: + raise TimeoutError('Office test exceeded 240 seconds') + time.sleep(0.05) + if (folder/'result.json').exists(): + data=json.loads((folder/'result.json').read_text()) + else: data={'status':'failed','error':'No worker result'} + data.update(dialogs=dialogs,elapsed_seconds=time.time()-started, + baseline_commit=subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(), + native_class_sha256=hashlib.sha256((ROOT/'src/stdRegex.cls').read_bytes()).hexdigest(), + architecture='Windows x64',methodology='QPC; 7 rotated samples; warmed cached patterns; calibrated >=50ms batches; allocation/release included; output equality checked') + if dialogs: data['status']='failed' + write(output/'results.json',json.dumps(data,indent=2)+'\n') + if process.returncode or data['status']!='passed': raise RuntimeError(data.get('error',str(dialogs))) + print('All requested checks passed',flush=True) + finally: + cleanup_error = None + try: + if process and process.poll() is None: process.kill(); process.wait(timeout=10) + # Close only the newly created test process, including after a COM hang. + if office_pid and office_pid not in original_pids and psutil.pid_exists(office_pid): + p=psutil.Process(office_pid) + if p.name().lower()=='winword.exe': p.terminate(); p.wait(timeout=10) + except Exception as exc: + cleanup_error = repr(exc) + for entry in reversed(saved): + with winreg.OpenKey(winreg.HKEY_CURRENT_USER,entry['path'],0,winreg.KEY_WRITE) as key: + if entry['present']: winreg.SetValueEx(key,entry['name'],0,entry['kind'],entry['value']) + else: + try: winreg.DeleteValue(key,entry['name']) + except FileNotFoundError: pass + if not entry['key_existed']: + try: winreg.DeleteKey(winreg.HKEY_CURRENT_USER,entry['path']) + except OSError: pass + verified=[] + for entry in saved: + try: + with winreg.OpenKey(winreg.HKEY_CURRENT_USER,entry['path']) as key: + value,kind=winreg.QueryValueEx(key,entry['name']) + assert entry['present'] and value==entry['value'] and kind==entry['kind'] + except FileNotFoundError: + assert not entry['present'] + verified.append(entry['name']) + write(output/'settings-restored.json',json.dumps({'verified':verified,'cleanup_error':cleanup_error},indent=2)+'\n') + if cleanup_error: raise RuntimeError('Office cleanup failed: '+cleanup_error) + +if __name__=='__main__': + if len(sys.argv)>1 and sys.argv[1]=='--worker': worker(Path(sys.argv[2]), '--test-only' in sys.argv) + else: supervise('--test-only' in sys.argv) diff --git a/tests/harnesses/stdRegex-thunk/summarize.py b/tests/harnesses/stdRegex-thunk/summarize.py new file mode 100644 index 00000000..ac2ee308 --- /dev/null +++ b/tests/harnesses/stdRegex-thunk/summarize.py @@ -0,0 +1,44 @@ +"""Summarize the measured samples and create a standalone comparison chart.""" +import json +from pathlib import Path +import statistics + +HERE=Path(__file__).resolve().parents[3] / "Tools/stdRegex-thunk" +def write(path,text): + path.write_bytes(text.replace('\r\n','\n').replace('\n','\r\n').encode('utf-8')) + +data=json.loads((HERE/'results/results.json').read_text()) +assert data['status']=='passed' and not data['dialogs'] +summary=[] +for case in dict.fromkeys(r['case'] for r in data['samples']): + for operation in ['Test','List','ListArr','MatchAllArr']: + record=dict(case=case,operation=operation) + for engine in ['VBA','native']: + rows=[r for r in data['samples'] if r['case']==case and r['operation']==operation+' '+engine] + values=[r['ms_per_call'] for r in rows] + assert len(values)==7 + record[engine]=dict(median_ms=statistics.median(values),min_ms=min(values),max_ms=max(values)) + record['speedup']=record['VBA']['median_ms']/record['native']['median_ms'] + summary.append(record) +write(HERE/'results/summary.json',json.dumps(summary,indent=2)+'\n') +html=''' + +

Native matcher speedup over current VBA (ratio of median call times; larger is faster).

+ +
WorkloadVBA msNative msSpeedup
+

Source: local Word 16.0, Windows x64, 1 November 2026. Seven rotated samples per operation; warmed patterns; batches calibrated to at least 50 ms. Returned values are allocated and released inside timing. Test stops at the first match. Compiler and public formatting remain VBA. These are workload-specific results, not a universal speed guarantee.

+'''.replace('DATA',json.dumps(summary)) +write(HERE/'results/comparison.html',html) +for r in summary: + print(f"{r['case']} | {r['operation']}: {r['speedup']:.2f}x") diff --git a/Tools/stdRegex-thunk/test-native.c b/tests/harnesses/stdRegex-thunk/test-native.c similarity index 98% rename from Tools/stdRegex-thunk/test-native.c rename to tests/harnesses/stdRegex-thunk/test-native.c index 3a50244e..757f4a27 100644 --- a/Tools/stdRegex-thunk/test-native.c +++ b/tests/harnesses/stdRegex-thunk/test-native.c @@ -5,7 +5,7 @@ #include #include #include -#include "matcher.c" +#include "../../../Tools/stdRegex-thunk/matcher.c" _Static_assert(sizeof(I)==4, "VBA Long is 32 bits"); _Static_assert(sizeof(Args)==(sizeof(void*)==8?72:52), "VBA NativeArgs layout"); static I canon_table[65536], workspace[3*4096], captures[20]; diff --git a/tests/harnesses/stdRegex-thunk/test-native.py b/tests/harnesses/stdRegex-thunk/test-native.py new file mode 100644 index 00000000..4166b388 --- /dev/null +++ b/tests/harnesses/stdRegex-thunk/test-native.py @@ -0,0 +1,23 @@ +"""Build real x86/x64 Windows test hosts and execute the embedded blobs.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import tempfile + +HERE = Path(__file__).resolve().parents[3] / "Tools/stdRegex-thunk" +parser = argparse.ArgumentParser() +parser.add_argument('--clang', required=True) +args = parser.parse_args() +results = {} +with tempfile.TemporaryDirectory(prefix='stdRegex-abi-') as scratch: + for arch, target, blob in [('x86', 'i686', 'matcher-x86.bin'), ('x64', 'x86_64', 'matcher.bin')]: + exe = Path(scratch) / (arch+'.exe') + subprocess.run([args.clang, '--target='+target+'-w64-windows-gnu', '-O2', '-static', + str(Path(__file__).with_name('test-native.c')), '-luser32', '-o', str(exe)], check=True) + result = subprocess.check_output([str(exe), str(HERE/blob)], text=True, timeout=30).strip() + assert '18000 checks passed' in result + print(result) + results[arch] = dict(result=result, blob_sha256=hashlib.sha256((HERE/blob).read_bytes()).hexdigest()) +(HERE/'verification-native.json').write_bytes((json.dumps(results, indent=2)+'\n').replace('\n','\r\n').encode()) diff --git a/Tools/stdWebSocket-tests/README.md b/tests/harnesses/stdWebSocket-tests/README.md similarity index 97% rename from Tools/stdWebSocket-tests/README.md rename to tests/harnesses/stdWebSocket-tests/README.md index edb443bd..07844b3c 100644 --- a/Tools/stdWebSocket-tests/README.md +++ b/tests/harnesses/stdWebSocket-tests/README.md @@ -3,7 +3,7 @@ Run from the repository root: ```powershell -python Tools/stdWebSocket-tests/run.py +python tests/harnesses/stdWebSocket-tests/run.py ``` Requirements: Python with `websockets` (tested with the asyncio server API), Excel, Visual Studio C++ build tools with Windows SDK, and OpenSSL (PATH or Git for Windows' bundled executable). The runner builds x86/x64 native helpers in a temporary build directory and selects the DLL using a VBA `Win64` check in the actual Office process. @@ -22,7 +22,7 @@ To run the same assertions against the WIP class identity: ```powershell $env:STDWEBSOCKET_WIP = '1' -python Tools/stdWebSocket-tests/run.py +python tests/harnesses/stdWebSocket-tests/run.py Remove-Item Env:STDWEBSOCKET_WIP ``` diff --git a/Tools/stdWebSocket-tests/WebSocketAsyncRecorder.cls b/tests/harnesses/stdWebSocket-tests/WebSocketAsyncRecorder.cls similarity index 100% rename from Tools/stdWebSocket-tests/WebSocketAsyncRecorder.cls rename to tests/harnesses/stdWebSocket-tests/WebSocketAsyncRecorder.cls diff --git a/Tools/stdWebSocket-tests/WebSocketRecorder.cls b/tests/harnesses/stdWebSocket-tests/WebSocketRecorder.cls similarity index 100% rename from Tools/stdWebSocket-tests/WebSocketRecorder.cls rename to tests/harnesses/stdWebSocket-tests/WebSocketRecorder.cls diff --git a/Tools/stdWebSocket-tests/fixture.py b/tests/harnesses/stdWebSocket-tests/fixture.py similarity index 100% rename from Tools/stdWebSocket-tests/fixture.py rename to tests/harnesses/stdWebSocket-tests/fixture.py diff --git a/Tools/stdWebSocket-tests/native_tests.c b/tests/harnesses/stdWebSocket-tests/native_tests.c similarity index 100% rename from Tools/stdWebSocket-tests/native_tests.c rename to tests/harnesses/stdWebSocket-tests/native_tests.c diff --git a/Tools/stdWebSocket-tests/run.py b/tests/harnesses/stdWebSocket-tests/run.py similarity index 95% rename from Tools/stdWebSocket-tests/run.py rename to tests/harnesses/stdWebSocket-tests/run.py index d4d132a4..8a68e2c6 100644 --- a/Tools/stdWebSocket-tests/run.py +++ b/tests/harnesses/stdWebSocket-tests/run.py @@ -18,8 +18,8 @@ from contextlib import contextmanager from fixture import start -root = Path(__file__).resolve().parents[2] -for name in ('src/stdWebSocket.cls', 'src/WIP/stdWebSocket/stdWebSocketNew.cls', 'src/stdICallable.cls', 'Tools/stdWebSocket-tests/WebSocketRecorder.cls', 'tests/stdWebSocketLocalTests.bas', 'Tools/stdWebSocket-tests/WebSocketAsyncRecorder.cls', 'tests/stdWebSocketAsyncTests.bas'): +root = Path(__file__).resolve().parents[3] +for name in ('src/stdWebSocket.cls', 'src/WIP/stdWebSocket/stdWebSocketNew.cls', 'src/stdICallable.cls', 'tests/harnesses/stdWebSocket-tests/WebSocketRecorder.cls', 'tests/stdWebSocketLocalTests.bas', 'tests/harnesses/stdWebSocket-tests/WebSocketAsyncRecorder.cls', 'tests/stdWebSocketAsyncTests.bas'): raw = (root / name).read_bytes().replace(b'\r\n', b'') if b'\n' in raw or b'\r' in raw: raise SystemExit('Source must use CRLF before import: ' + name) diff --git a/Tools/stdWebSocket-tests/worker.ps1 b/tests/harnesses/stdWebSocket-tests/worker.ps1 similarity index 93% rename from Tools/stdWebSocket-tests/worker.ps1 rename to tests/harnesses/stdWebSocket-tests/worker.ps1 index 578c961a..3bfcd182 100644 --- a/Tools/stdWebSocket-tests/worker.ps1 +++ b/tests/harnesses/stdWebSocket-tests/worker.ps1 @@ -1,6 +1,6 @@ param([string]$PidFile) $ErrorActionPreference = 'Stop' -$root = (Resolve-Path "$PSScriptRoot/../..").Path +$root = (Resolve-Path "$PSScriptRoot/../../..").Path Add-Type @" using System; using System.Runtime.InteropServices; @@ -48,7 +48,7 @@ try { $book = $excel.Workbooks.Add() $project = $book.GetType().InvokeMember('VBProject', [System.Reflection.BindingFlags]::GetProperty, $null, $book, $null) if ($null -eq $project) { throw 'VBA project access unavailable; security settings were not changed.' } - $files = @('src/stdWebSocket.cls','Tools/stdWebSocket-tests/WebSocketRecorder.cls','tests/stdWebSocketLocalTests.bas','src/stdICallable.cls','Tools/stdWebSocket-tests/WebSocketAsyncRecorder.cls','tests/stdWebSocketAsyncTests.bas') | ForEach-Object { Join-Path $root $_ } + $files = @('src/stdWebSocket.cls','tests/harnesses/stdWebSocket-tests/WebSocketRecorder.cls','tests/stdWebSocketLocalTests.bas','src/stdICallable.cls','tests/harnesses/stdWebSocket-tests/WebSocketAsyncRecorder.cls','tests/stdWebSocketAsyncTests.bas') | ForEach-Object { Join-Path $root $_ } if ($env:STDWEBSOCKET_WIP -eq '1') { $generated = Join-Path (Split-Path $PidFile) 'generated' $null = New-Item -ItemType Directory -Path $generated -Force diff --git a/Tools/stdWindow-subclass/IDEProbe.bas b/tests/harnesses/stdWindow-subclass/IDEProbe.bas similarity index 100% rename from Tools/stdWindow-subclass/IDEProbe.bas rename to tests/harnesses/stdWindow-subclass/IDEProbe.bas diff --git a/tests/harnesses/stdWindow-subclass/run.py b/tests/harnesses/stdWindow-subclass/run.py new file mode 100644 index 00000000..ae8a78b6 --- /dev/null +++ b/tests/harnesses/stdWindow-subclass/run.py @@ -0,0 +1,376 @@ +"""Bounded isolated Excel regression runner; requires pywin32 and Office. + +Discovers the installed Office/VBE versions, restores exact registry values, +and dismisses dialogs only in the newly created test process. +""" +import json +from pathlib import Path +import subprocess +import sys +import threading +import tempfile +import winreg +import gc +import os + +ROOT = Path(__file__).resolve().parents[3] +FILES = ['src/stdICallable.cls', 'src/stdCallback.cls', 'src/stdWindow.cls', + 'tests/stdWindowSubclassProbe.cls', 'tests/stdWindowSubclassTests.bas', + 'tests/harnesses/stdWindow-subclass/IDEProbe.bas'] + + +def worker(): + print('Starting isolated Office worker', flush=True) + import win32com.client + import win32process + import win32gui + import win32con + existing = set() + win32gui.EnumWindows(lambda hwnd, _: existing.add(win32process.GetWindowThreadProcessId(hwnd)[1]), None) + app = win32com.client.DispatchEx('Excel.Application') + print('Created isolated Office application', flush=True) + pid = win32process.GetWindowThreadProcessId(app.Hwnd)[1] + if pid in existing: + raise RuntimeError('Worker returned an existing Office instance; refusing to use it.') + Path(sys.argv[2]).write_text(str(pid), encoding='ascii') + print('PID ' + str(pid), flush=True) + stop = threading.Event() + dialogs = [] + + def monitor(): + while not stop.wait(.03): + def visit(hwnd, unused): + if win32process.GetWindowThreadProcessId(hwnd)[1] != pid: + return + if win32gui.GetClassName(hwnd) == '#32770' and win32gui.IsWindowVisible(hwnd): + labels = [] + win32gui.EnumChildWindows(hwnd, lambda child, _: labels.append(win32gui.GetWindowText(child)), None) + dialogs.append(' | '.join(labels)) + print('DIALOG ' + dialogs[-1], flush=True) + win32gui.PostMessage(hwnd, win32con.WM_COMMAND, 1, 0) + elif win32gui.GetClassName(hwnd) in ('XLMAIN', 'wndclass_desked_gsk') and win32gui.IsWindowVisible(hwnd): + win32gui.ShowWindow(hwnd, win32con.SW_MINIMIZE) + win32gui.EnumWindows(visit, None) + + thread = threading.Thread(target=monitor, daemon=True) + thread.start() + book = None + try: + app.Visible = False + app.DisplayAlerts = False + no_vbom = '--no-vbom' in sys.argv + if no_vbom: + book = app.Workbooks.Open(sys.argv[3], ReadOnly=True) + try: + book.VBProject.VBComponents.Count + except Exception as error: + if 'not trusted' not in str(error).lower(): + raise + print('Verified VBOM access is disabled in this test instance', flush=True) + else: + raise RuntimeError('Negative access test did not disable VBOM') + else: + book = app.Workbooks.Add() + project = book.VBProject + project.References.AddFromGuid('{1EA4DBF0-3C3B-11CF-810C-00AA00389B71}', 1, 1) + project.VBComponents.Add(3).Name = 'stdWindowTestForm' + for file in FILES: + raw = (ROOT / file).read_bytes().replace(b'\r\n', b'') + assert b'\n' not in raw and b'\r' not in raw, file + project.VBComponents.Import(str(ROOT / file)) + app.VBE.CommandBars.FindControl(1, 578).Execute() + if dialogs: + raise RuntimeError('Compilation failed') + book.SaveAs(sys.argv[3], 52) + result = app.Run("'" + book.Name + "'!stdWindowSubclassTests.RunTests") + print(json.dumps({'result': result, 'dialogs': dialogs}), flush=True) + if dialogs or not str(result).startswith('Passed '): + raise RuntimeError('Office regression failed') + result = app.Run("'" + book.Name + "'!stdWindowSubclassTests.RunUserFormTests") + print(json.dumps({'result': result, 'dialogs': dialogs}), flush=True) + if dialogs or not str(result).startswith('Passed '): + raise RuntimeError('UserForm regression failed') + result = app.Run("'" + book.Name + "'!stdWindowSubclassTests.RunResolverTests") + print(json.dumps({'result': result, 'dialogs': dialogs}), flush=True) + if dialogs or not str(result).startswith('Passed '): + raise RuntimeError('EbMode resolver regression failed') + if not no_vbom: + ide_checks(app, book, project, pid) + if dialogs: + raise RuntimeError('Office/VBE dialog occurred during IDE checks') + finally: + try: + if book is not None: + print('Closing disposable workbook', flush=True) + book.Close(False) + finally: + try: + print('Closing isolated Office application', flush=True) + app.Quit() + print('Isolated Office application closed', flush=True) + finally: + stop.set() + thread.join(2) + # Office has closed and assertions have completed. In this disposable + # process, skip late pywin32 proxy finalizers: after a denied VBOM query + # they can block interpreter teardown even after Excel.Quit returned. + # The parent still owns the Office PID and performs failure-safe cleanup. + sys.stdout.flush() + sys.stderr.flush() + os._exit(0) + + +def ide_checks(app, book, project, pid): + import ctypes + from ctypes import wintypes + user = ctypes.WinDLL('user32', use_last_error=True) + kernel = ctypes.WinDLL('kernel32', use_last_error=True) + user.SendMessageTimeoutW.argtypes = [wintypes.HWND, wintypes.UINT, wintypes.WPARAM, wintypes.LPARAM, + wintypes.UINT, wintypes.UINT, ctypes.POINTER(ctypes.c_size_t)] + user.SendMessageTimeoutW.restype = ctypes.c_size_t + user.DestroyWindow.argtypes = [wintypes.HWND] + user.IsWindow.argtypes = [wintypes.HWND] + kernel.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + kernel.OpenProcess.restype = wintypes.HANDLE + kernel.ReadProcessMemory.argtypes = [wintypes.HANDLE, ctypes.c_void_p, ctypes.c_void_p, + ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)] + kernel.CloseHandle.argtypes = [wintypes.HANDLE] + process = kernel.OpenProcess(0x10, False, pid) + if not process: + raise ctypes.WinError(ctypes.get_last_error()) + + def run(name): + return app.Run("'" + book.Name + "'!WindowIDEProbe." + name) + + def counts(thunk): + counters = (wintypes.DWORD * 2)() + copied = ctypes.c_size_t() + if not kernel.ReadProcessMemory(process, thunk+4096, counters, ctypes.sizeof(counters), ctypes.byref(copied)): + raise ctypes.WinError(ctypes.get_last_error()) + return tuple(counters) + + def send(hwnd): + result = ctypes.c_size_t() + if not user.SendMessageTimeoutW(hwnd, 0x8001, 1, 2, 2, 2000, ctypes.byref(result)): + raise ctypes.WinError(ctypes.get_last_error()) + return result.value + + def paused_probe(hwnd, thunk, reset=False): + import pythoncom + import win32com.client + import pywintypes + import time + import win32gui + vbe_hwnd = app.VBE.MainWindow.HWnd + run_id = reset_id = None + for menu in app.VBE.CommandBars('Menu Bar').Controls: + if menu.Caption.replace('&', '').lower() == 'run': + for control in menu.Controls: + caption = control.Caption.replace('&', '').lower() + if caption.startswith('run sub/') or caption == 'run macro': + run_id = control.Id + if caption == 'reset': + reset_id = control.Id + assert run_id and reset_id, 'Could not find the test VBE Continue/Reset controls' + stream = pythoncom.CoMarshalInterThreadInterfaceInStream(pythoncom.IID_IDispatch, app.VBE._oleobj_) + errors = [] + + def coordinator(): + pythoncom.CoInitialize() + remote = None + try: + remote = win32com.client.Dispatch(pythoncom.CoGetInterfaceAndReleaseStream(stream, pythoncom.IID_IDispatch)) + deadline = time.monotonic()+8 + while time.monotonic() < deadline: + if '[break]' in win32gui.GetWindowText(vbe_hwnd).lower(): + break + time.sleep(.03) + else: + raise RuntimeError('Test VBE did not enter break mode: ' + win32gui.GetWindowText(vbe_hwnd)) + assert send(hwnd) == 0, 'Break mode must forward without entering VBA' + assert counts(thunk) == (0, 1), 'Break mode should preserve the subscription' + remote.CommandBars.FindControl(1, reset_id if reset else run_id).Execute() + except Exception as error: + errors.append(str(error)) + print('Break-mode coordinator failed:', error, flush=True) + if remote is not None: + try: + remote.CommandBars.FindControl(1, reset_id).Execute() + except Exception: + pass + finally: + remote = None + pythoncom.CoUninitialize() + + helper = threading.Thread(target=coordinator, daemon=True) + helper.start() + try: + result = run('StopAndResume') + except pywintypes.com_error as error: + if not reset or not error.excepinfo or error.excepinfo[5] != -2146788248: + raise + result = None + helper.join(10) + assert not helper.is_alive(), 'Break-mode coordinator did not finish' + assert not errors, errors + if not reset: + assert result == 2468, 'Continue must resume callback dispatch' + + try: + hwnd, thunk = run('ArmModeless') + assert send(hwnd) == 2468, 'Modeless callbacks must work after the starter macro returns' + run('Cleanup') + print('Passed real modeless UserForm callback after starter macro returns', flush=True) + + hwnd, thunk = run('Arm') + print('IDE mode after macro:', project.Mode, 'native counters', counts(thunk), flush=True) + assert send(hwnd) == 0, 'Design mode must forward without executing VBA' + assert counts(thunk) == (0, 0), 'Design mode must remove the registration natively' + assert run('CheckDetached'), 'Wrapper must reconcile native detachment' + run('Cleanup') + print('Passed real design-mode detachment', flush=True) + + hwnd, thunk = run('Arm') + paused_probe(hwnd, thunk) + run('Cleanup') + print('Passed real break/Continue forwarding', flush=True) + + hwnd, thunk = run('Arm') + paused_probe(hwnd, thunk, reset=True) + if user.IsWindow(hwnd): + assert send(hwnd) == 0 + assert counts(thunk) == (0, 0) + print('Passed real VBE Reset detachment', flush=True) + + hwnd, thunk = run('Arm') + import pywintypes + try: + run('EndNow') + except pywintypes.com_error as error: + # VBE returns its explicit execution-aborted HRESULT for End. + if not error.excepinfo or error.excepinfo[5] != -2146788248: + raise + if user.IsWindow(hwnd): + assert send(hwnd) == 0, 'End must not call a stale VBA object' + assert counts(thunk) == (0, 0), 'End must detach without VBA globals' + # The disposable Office process owns this orphan after End; process + # teardown reclaims it. Do not enter its vanished VBA wrapper again. + print('Passed real End/reset detachment', flush=True) + finally: + kernel.CloseHandle(process) + + +def launch_worker(template, no_vbom=False): + handle = tempfile.NamedTemporaryFile(prefix='stdvba86-', suffix='.pid', delete=False) + pidfile = Path(handle.name) + handle.close() + args = [sys.executable, __file__, '--worker', str(pidfile), str(template)] + if no_vbom: + args.append('--no-vbom') + child = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + timed_out = False + output = '' + try: + output, _ = child.communicate(timeout=45) + except subprocess.TimeoutExpired: + timed_out = True + child.kill() + finally: + owned_pid = pidfile.read_text().strip() + if owned_pid: + subprocess.run(['taskkill', '/PID', owned_pid, '/F'], capture_output=True) + if child.poll() is None: + child.kill() + tail, _ = child.communicate(timeout=5) + if timed_out: + output = tail + pidfile.unlink() + print(output, flush=True) + if timed_out: + raise RuntimeError('Office test timeout; partial worker output shown above') + if child.returncode: + raise RuntimeError('Office worker failed: ' + str(child.returncode)) + + +def main(): + import win32com.client + import win32process + import win32gui + existing = set() + win32gui.EnumWindows(lambda hwnd, _: existing.add(win32process.GetWindowThreadProcessId(hwnd)[1]), None) + app = win32com.client.DispatchEx('Excel.Application') + discovery_pid = win32process.GetWindowThreadProcessId(app.Hwnd)[1] + if discovery_pid in existing: + raise RuntimeError('Discovery returned an existing Office instance; refusing to use it.') + print('Discovery PID', discovery_pid, flush=True) + try: + version = app.Version + vbe = '.'.join(str(int(part)) for part in app.VBE.Version.split('.')) + book = app.Workbooks.Add() + try: + access = book.VBProject.VBComponents.Count > 0 + except Exception: + access = False + finally: + book.Close(False) + finally: + app.Quit() + del app + gc.collect() + subprocess.run(['taskkill', '/PID', str(discovery_pid), '/F'], capture_output=True) + print('Excel', version, 'VBE', vbe, 'VBProject access', access, flush=True) + policy_enables_access = False + for hive in (winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE): + try: + with winreg.OpenKey(hive, fr'Software\Policies\Microsoft\Office\{version}\Excel\Security') as key: + policy, _ = winreg.QueryValueEx(key, 'AccessVBOM') + if not policy: + raise RuntimeError('Managed policy disables Excel VBA project access.') + policy_enables_access = True + except FileNotFoundError: + pass + template_handle = tempfile.NamedTemporaryFile(prefix='stdvba86-', suffix='.xlsm', delete=False) + template = Path(template_handle.name) + template_handle.close() + changes = [] + try: + settings = [(fr'Software\Microsoft\VBA\{vbe}\Common', 'BreakOnAllErrors', 0), + (fr'Software\Microsoft\VBA\{vbe}\Common', 'BreakOnServerErrors', 0)] + if not access: + settings.append((fr'Software\Microsoft\Office\{version}\Excel\Security', 'AccessVBOM', 1)) + for path, name, value in settings: + with winreg.CreateKey(winreg.HKEY_CURRENT_USER, path) as key: + try: + old, kind = winreg.QueryValueEx(key, name) + except FileNotFoundError: + old, kind = None, None + changes.append((path, name, old, kind)) + print('SETTING', path, name, old, kind, flush=True) + winreg.SetValueEx(key, name, 0, winreg.REG_DWORD, value) + launch_worker(template) + if policy_enables_access: + print('Skipped disabled-VBOM variant: managed policy forces access on.', flush=True) + else: + path = fr'Software\Microsoft\Office\{version}\Excel\Security' + with winreg.CreateKey(winreg.HKEY_CURRENT_USER, path) as key: + try: + old, kind = winreg.QueryValueEx(key, 'AccessVBOM') + except FileNotFoundError: + old, kind = None, None + changes.append((path, 'AccessVBOM', old, kind)) + print('SETTING disabled-VBOM variant', path, 'AccessVBOM', old, kind, flush=True) + winreg.SetValueEx(key, 'AccessVBOM', 0, winreg.REG_DWORD, 0) + launch_worker(template, no_vbom=True) + finally: + for path, name, old, kind in reversed(changes): + with winreg.OpenKey(winreg.HKEY_CURRENT_USER, path, 0, winreg.KEY_SET_VALUE) as key: + if kind is None: + winreg.DeleteValue(key, name) + else: + winreg.SetValueEx(key, name, 0, kind, old) + template.unlink(missing_ok=True) + print('Original registry values restored.', flush=True) + + +if __name__ == '__main__': + worker() if '--worker' in sys.argv else main() diff --git a/tests/manifest.json b/tests/manifest.json new file mode 100644 index 00000000..bbf209f5 --- /dev/null +++ b/tests/manifest.json @@ -0,0 +1,647 @@ +{ + "version": 1, + "suites": [ + { + "id": "stdAcc", + "kind": "vba", + "files": [ + "src/stdAcc.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "src/stdLambda.cls", + "tests/stdAccTests.bas", + "tests/stdAccTestHelper.cls" + ], + "cases": { + "testAll": "stdAccTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions", + "requirements": "interactive desktop or external resources; opt in with --suite or --all" + }, + { + "id": "stdArray", + "kind": "vba", + "files": [ + "src/stdArray.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "src/stdLambda.cls", + "src/stdEnumerator.cls", + "tests/stdArrayTests.bas", + "tests/stdArrayBaseEvents.cls" + ], + "cases": { + "testAll": "stdArrayTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": true, + "result": "assertions" + }, + { + "id": "stdCallback", + "kind": "vba", + "files": [ + "src/stdICallable.cls", + "src/stdCallback.cls", + "tests/stdCallbackTests.bas" + ], + "cases": { + "ModuleCalls": "stdCallbackTests.TestModuleCalls", + "Bind": "stdCallbackTests.TestBind", + "PointerCalls": "stdCallbackTests.TestPointerCalls" + }, + "hosts": [ + "Excel" + ], + "default": true, + "result": "assertions" + }, + { + "id": "stdRegex", + "kind": "vba", + "files": [ + "src/stdRegex.cls", + "tests/stdRegexTests.bas" + ], + "cases": { + "Contract": "stdRegexTests.testContract", + "NativeEngine": "stdRegexTests.testNativeEngine", + "CaptureArraysAndLists": "stdRegexTests.testCaptureArraysAndLists", + "MatchAllCol": "stdRegexTests.TestMatchAllCol" + }, + "hosts": [ + "Excel", + "Word" + ], + "default": true, + "result": "assertions" + }, + { + "id": "stdEnumerator", + "kind": "vba", + "files": [ + "src/stdEnumerator.cls", + "src/stdArray.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "src/stdLambda.cls", + "tests/stdEnumeratorTests.bas" + ], + "cases": { + "testAll": "stdEnumeratorTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions" + }, + { + "id": "stdLambda", + "kind": "vba", + "files": [ + "src/stdICallable.cls", + "src/stdCallback.cls", + "src/stdLambda.cls", + "src/stdArray.cls", + "tests/stdLambdaTests.bas" + ], + "cases": { + "testAll": "stdLambdaTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions" + }, + { + "id": "stdHTML", + "kind": "vba", + "files": [ + "src/stdHTML.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "src/stdLambda.cls", + "tests/stdHTMLTests.bas" + ], + "cases": { + "testAll": "stdHTMLTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions" + }, + { + "id": "stdReg", + "kind": "vba", + "files": [ + "src/stdReg.cls", + "tests/stdRegTests.bas" + ], + "cases": { + "testAll": "stdRegTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": true, + "result": "assertions" + }, + { + "id": "stdPerformance", + "kind": "vba", + "files": [ + "src/stdPerformance.cls", + "tests/stdPerformanceTests.bas" + ], + "cases": { + "testAll": "stdPerformanceTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": true, + "result": "assertions" + }, + { + "id": "stdPerformanceHost", + "kind": "vba", + "files": [ + "src/stdPerformance.cls", + "tests/stdPerformanceHostTests.bas" + ], + "cases": { + "testAll": "stdPerformanceHostTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions" + }, + { + "id": "stdPerformanceAccess", + "kind": "vba", + "files": [ + "src/stdPerformance.cls", + "tests/stdPerformanceAccessTests.bas" + ], + "cases": { + "testAll": "stdPerformanceAccessTests.testAll" + }, + "hosts": [ + "Access" + ], + "default": false, + "result": "assertions", + "unsupported": "Access automation is not implemented; run this module manually in a disposable Access database." + }, + { + "id": "stdProcess", + "kind": "vba", + "files": [ + "src/stdProcess.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "src/stdLambda.cls", + "tests/stdProcessTests.bas" + ], + "cases": { + "testAll": "stdProcessTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions", + "requirements": "interactive desktop or external resources; opt in with --suite or --all" + }, + { + "id": "stdSentry", + "kind": "vba", + "files": [ + "src/stdSentry.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "src/stdLambda.cls", + "tests/stdSentryTests.bas" + ], + "cases": { + "testAll": "stdSentryTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": true, + "result": "assertions", + "display_alerts": true + }, + { + "id": "stdSentrySheetUnlocker", + "kind": "vba", + "files": [ + "src/stdSentry.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "tests/stdSentrySheetUnlockerTests.bas" + ], + "cases": { + "testAll": "stdSentrySheetUnlockerTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions" + }, + { + "id": "stdStringBuilder", + "kind": "vba", + "files": [ + "src/WIP/stdStringBuilder.cls", + "tests/stdStringBuilderTests.bas" + ], + "cases": { + "testAll": "stdStringBuilderTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions", + "requirements": "experimental WIP class" + }, + { + "id": "stdWindow", + "kind": "vba", + "files": [ + "src/stdWindow.cls", + "src/stdAcc.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "src/stdLambda.cls", + "tests/stdWindowTests.bas" + ], + "cases": { + "testAll": "stdWindowTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions", + "requirements": "interactive desktop or external resources; opt in with --suite or --all" + }, + { + "id": "stdJSON", + "kind": "vba", + "files": [ + "src/stdJSON.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "tests/stdJSONTests.bas" + ], + "cases": { + "RunTests": "stdJSONTests.RunTests" + }, + "hosts": [ + "Excel" + ], + "default": true, + "result": "pass" + }, + { + "id": "stdRefArray", + "kind": "vba", + "files": [ + "src/stdRefArray.cls", + "tests/stdRefArrayTests.bas" + ], + "cases": { + "RunTests": "stdRefArrayTests.RunTests" + }, + "hosts": [ + "Excel" + ], + "default": true, + "result": "count" + }, + { + "id": "stdTimer", + "kind": "vba", + "files": [ + "src/stdTimer.cls", + "tests/stdTimerTests.bas", + "tests/stdTimerProbe.cls" + ], + "cases": { + "testLifecycle": "stdTimerTests.testLifecycle", + "testInvalidResolution": "stdTimerTests.testInvalidResolution" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "timer" + }, + { + "id": "stdClipboard", + "kind": "vba", + "files": [ + "src/stdClipboard.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "src/stdLambda.cls", + "tests/stdClipboardTests.bas" + ], + "cases": { + "testAll": "stdClipboardTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions", + "unsupported": "Legacy suite requires the workbook-specific xlsProjectBuilder worksheet; migrate this fixture before automated execution." + }, + { + "id": "stdHTTP", + "kind": "vba", + "files": [ + "src/stdHTTP.cls", + "src/stdICallable.cls", + "tests/stdHTTPTests.bas" + ], + "cases": { + "TestMain": "stdHTTPTests.TestMain" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions", + "requirements": "interactive desktop or external resources; opt in with --suite or --all" + }, + { + "id": "stdWebSocket", + "kind": "vba", + "files": [ + "src/stdWebSocket.cls", + "src/stdICallable.cls", + "src/stdCallback.cls", + "tests/stdWebSocketTests.bas" + ], + "cases": { + "testAll": "stdWebSocketTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions", + "requirements": "interactive desktop or external resources; opt in with --suite or --all", + "unsupported": "Legacy placeholder contains no executable assertions; use stdWebSocket.local." + }, + { + "id": "stdImage", + "kind": "adapter", + "command": "tests/harnesses/stdImage-tests/run.py", + "cases": { + "regressions": "regressions" + }, + "hosts": [ + "Excel", + "Word" + ], + "default": false, + "requirements": "Pillow; clipboard; local HTTP fixtures" + }, + { + "id": "stdCallback.probes", + "kind": "adapter", + "command": "tests/harnesses/stdCallback-tests/run.py", + "cases": { + "regressions": "regressions" + }, + "hosts": [ + "Excel" + ], + "default": false, + "requirements": "characterization including deliberately invalid native calls; crashes fail the run" + }, + { + "id": "stdDate", + "kind": "vba", + "command": "tests/harnesses/stdDate-test/run-tests.ps1", + "cases": { + "testAll": "stdDateTests.testAll" + }, + "hosts": [ + "Excel" + ], + "default": false, + "requirements": "experimental class; instrumented source", + "files": [ + "src/WIP/stdDate.cls", + "tests/stdDateTests.bas", + "tests/harnesses/stdDate-test/TestRunner.bas" + ], + "result": "assertions", + "instrument": "date" + }, + { + "id": "stdHTTP.local", + "kind": "adapter", + "command": "tests/harnesses/stdHTTP-tests/run.py", + "cases": { + "regressions": "regressions" + }, + "hosts": [ + "Excel" + ], + "default": true, + "requirements": "local HTTP fixtures" + }, + { + "id": "stdWebSocket.local", + "kind": "adapter", + "command": "tests/harnesses/stdWebSocket-tests/run.py", + "cases": { + "regressions": "regressions" + }, + "hosts": [ + "Excel" + ], + "default": false, + "requirements": "local fixtures; native compiler; TLS" + }, + { + "id": "stdWindow.subclass", + "kind": "adapter", + "command": "tests/harnesses/stdWindow-subclass/run.py", + "cases": { + "regressions": "regressions" + }, + "hosts": [ + "Excel" + ], + "default": false, + "requirements": "native callbacks and UI integration" + }, + { + "id": "stdLambda.vm", + "kind": "vba", + "files": [ + "src/stdICallable.cls", + "src/stdCallback.cls", + "src/stdLambda.cls" + ], + "cases": { + "Instructions": "stdLambda.protRunVMTests" + }, + "hosts": [ + "Excel" + ], + "default": false, + "result": "assertions", + "instrument": "lambda-vm" + }, + { + "id": "tooling.regexBuild", + "kind": "python", + "command": "tests/tooling/test_regex_build.py", + "cases": { + "BuildSelectionTests.test_default_reproduces_recorded_commit_even_when_head_is_newer": "BuildSelectionTests.test_default_reproduces_recorded_commit_even_when_head_is_newer", + "BuildSelectionTests.test_ref_override_does_not_adopt_revision_implicitly": "BuildSelectionTests.test_ref_override_does_not_adopt_revision_implicitly", + "BuildSelectionTests.test_update_records_resolved_sha_and_matching_class": "BuildSelectionTests.test_update_records_resolved_sha_and_matching_class", + "BuildSelectionTests.test_update_requires_a_ref": "BuildSelectionTests.test_update_requires_a_ref", + "BuildSelectionTests.test_check_is_read_only": "BuildSelectionTests.test_check_is_read_only", + "BuildSelectionTests.test_check_cannot_update_manifest": "BuildSelectionTests.test_check_cannot_update_manifest", + "BuildSelectionTests.test_incompatible_engine_does_not_change_existing_files": "BuildSelectionTests.test_incompatible_engine_does_not_change_existing_files", + "BuildSelectionTests.test_missing_revision_does_not_change_existing_files": "BuildSelectionTests.test_missing_revision_does_not_change_existing_files" + }, + "hosts": [ + "Excel", + "Word", + "Access" + ], + "default": true + }, + { + "id": "native.windowThunk", + "kind": "python", + "command": "tests/native/test_window_thunk.py", + "cases": { + "ThunkTests.test_x86_result": "ThunkTests.test_x86_result", + "ThunkTests.test_x86_fallback": "ThunkTests.test_x86_fallback", + "ThunkTests.test_x64_result": "ThunkTests.test_x64_result", + "ThunkTests.test_x64_fallback": "ThunkTests.test_x64_fallback", + "ThunkTests.test_x86_break": "ThunkTests.test_x86_break", + "ThunkTests.test_x64_break": "ThunkTests.test_x64_break", + "ThunkTests.test_x86_design": "ThunkTests.test_x86_design", + "ThunkTests.test_x64_design": "ThunkTests.test_x64_design", + "ThunkTests.test_x86_unknown_mode": "ThunkTests.test_x86_unknown_mode", + "ThunkTests.test_x64_unknown_mode": "ThunkTests.test_x64_unknown_mode", + "ThunkTests.test_x86_break_destroy": "ThunkTests.test_x86_break_destroy", + "ThunkTests.test_x64_break_destroy": "ThunkTests.test_x64_break_destroy", + "ThunkTests.test_x86_remove_failure": "ThunkTests.test_x86_remove_failure", + "ThunkTests.test_x64_remove_failure": "ThunkTests.test_x64_remove_failure", + "ThunkTests.test_x86_opt_out_design": "ThunkTests.test_x86_opt_out_design", + "ThunkTests.test_x64_opt_out_design": "ThunkTests.test_x64_opt_out_design", + "ThunkTests.test_x86_opt_out_break": "ThunkTests.test_x86_opt_out_break", + "ThunkTests.test_x64_opt_out_break": "ThunkTests.test_x64_opt_out_break" + }, + "hosts": [ + "Excel", + "Word", + "Access" + ], + "default": false + }, + { + "id": "tooling.runner", + "kind": "python", + "command": "tests/tooling/test_runner.py", + "cases": { + "RunnerTests.test_every_root_test_module_registered": "RunnerTests.test_every_root_test_module_registered", + "RunnerTests.test_individual_case_does_not_select_siblings": "RunnerTests.test_individual_case_does_not_select_siblings", + "RunnerTests.test_unknown_selectors_fail": "RunnerTests.test_unknown_selectors_fail", + "RunnerTests.test_all_includes_opt_in_and_unavailable_suites": "RunnerTests.test_all_includes_opt_in_and_unavailable_suites", + "RunnerTests.test_bridge_checks_failures_and_zero_assertions": "RunnerTests.test_bridge_checks_failures_and_zero_assertions", + "RunnerTests.test_return_contracts_are_checked": "RunnerTests.test_return_contracts_are_checked" + }, + "hosts": [ + "Excel", + "Word", + "Access" + ], + "default": true + }, + { + "id": "stdRegex.thunkComparison", + "kind": "adapter", + "command": "tests/harnesses/stdRegex-thunk/run.py", + "cases": { + "regressions": "regressions" + }, + "hosts": [ + "Word" + ], + "default": false, + "unsupported": "Historical experimental comparison requires removed CreateVBA/CreateNative APIs; use stdRegex cases for the current distribution." + }, + { + "id": "native.regexThunk", + "kind": "command", + "command": "tests/harnesses/stdRegex-thunk/test-native.py", + "cases": { + "ABIs": "ABIs" + }, + "hosts": [ + "Excel", + "Word", + "Access" + ], + "default": false, + "compiler": true, + "requirements": "llvm-mingw clang targeting x86 and x64 Windows; select with --clang" + }, + { + "id": "tooling.officeLifecycle", + "kind": "python", + "command": "tests/tooling/test_office_runner.py", + "cases": { + "OfficeLifecycleTests.test_suppressed_assertion_failure_is_not_a_pass": "OfficeLifecycleTests.test_suppressed_assertion_failure_is_not_a_pass", + "OfficeLifecycleTests.test_unexpected_error_is_not_a_pass": "OfficeLifecycleTests.test_unexpected_error_is_not_a_pass", + "OfficeLifecycleTests.test_empty_case_is_not_a_pass": "OfficeLifecycleTests.test_empty_case_is_not_a_pass" + }, + "hosts": [ + "Excel" + ], + "default": false, + "requirements": "Windows, Excel and pywin32; deliberately fails scratch cases to verify failure reporting" + }, + { + "id": "stdEnumerator.local", + "kind": "adapter", + "command": "tests/harnesses/stdEnumerator-tests/run.py", + "cases": { + "regressions": "regressions" + }, + "hosts": [ + "Excel" + ], + "default": false, + "requirements": "Excel, pywin32; instrumented IEnumVARIANT source" + } + ] +} diff --git a/tests/native/test_window_thunk.py b/tests/native/test_window_thunk.py new file mode 100644 index 00000000..1b1fadaf --- /dev/null +++ b/tests/native/test_window_thunk.py @@ -0,0 +1,139 @@ +from pathlib import Path +import sys +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "Tools/stdWindow-subclass")) +"""Check both native ABIs without Office. Dev dependencies: keystone-engine, unicorn.""" +import struct +import unittest +from build import assemble +from unicorn import Uc, UC_ARCH_X86, UC_MODE_32, UC_MODE_64, UC_HOOK_CODE +from unicorn.x86_const import * + + +class ThunkTests(unittest.TestCase): + def check_abi(self, bits, fail=False, mode=1, message=0x8001, remove_success=True, unsafe=False): + size = bits // 8 + fmt = ' 0 Then Call Throw("Expected an even number of parameters") + + Dim size As Long: size = (UBound(opsParams) + 1) \ 2 + ReDim ops(0 To size) + + Dim i As Long + For i = 0 To size - 1 + ops(i) = testCreateOp(opsParams(i * 2), opsParams(i * 2 + 1)) + Next i + + testCreateOps = ops +End Function + +'Runs tests on the class +'@returns - An array of booleans indicating whether each test passed +Public Sub protRunVMTests() + 'iPush + Debug.Assert evaluate(testCreateOps(iPush, 10), Array()) = 10 + + 'iPop + ' Cannot directly assert by return value of evaluate unless combined with other ops + ' To make it testable: Push 10, Push 20, Pop. If pop was a no-op, result would be 20. If pop works result should be 10. + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iPop, Empty), Array()) = 10 + + ' iMerge + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iMerge, Empty), Array()) = 20 + + 'TODO: iAccess_General + + ' iAccess_Argument + Debug.Assert evaluate(testCreateOps(iAccess_Argument, 1), Array("Hello")) = "Hello" + Debug.Assert evaluate(testCreateOps(iAccess_Argument, 1), Array("Hello", "World")) = "Hello" + Debug.Assert evaluate(testCreateOps(iAccess_Argument, 2), Array("Hello", "World")) = "World" + Debug.Assert evaluate(testCreateOps(iAccess_Argument, 1), Array(Application)) Is Application + + 'TODO: iSet_General + + ' Arithmetic operations + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iArithmetic_Add, Empty), Array()) = 30 + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iArithmetic_Subtract, Empty), Array()) = -10 + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iArithmetic_Multiply, Empty), Array()) = 200 + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iArithmetic_Divide, Empty), Array()) = 0.5 + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 3, iArithmetic_Power, Empty), Array()) = 1000 + Debug.Assert evaluate(testCreateOps(iPush, 20, iPush, 10, iArithmetic_Modulo, Empty), Array()) = 0 + Debug.Assert evaluate(testCreateOps(iPush, 20, iPush, 9, iArithmetic_Modulo, Empty), Array()) = 2 + Debug.Assert evaluate(testCreateOps(iPush, -20, iPush, 10, iArithmetic_Modulo, Empty), Array()) = 0 + Debug.Assert evaluate(testCreateOps(iPush, 10, iArithmetic_Negate, Empty), Array()) = -10 + + 'iLogic_And + 'A | 1 1 0 0 + 'B | 1 0 1 0 + '----------- + 'O | 1 0 0 0 + Debug.Assert evaluate(testCreateOps(iPush, True, iPush, True, iLogic_And, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, True, iPush, False, iLogic_And, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, False, iPush, True, iLogic_And, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, False, iPush, False, iLogic_And, Empty), Array()) = False + + 'iLogic_Or + 'A | 1 1 0 0 + 'B | 1 0 1 0 + '----------- + 'O | 1 1 1 0 + Debug.Assert evaluate(testCreateOps(iPush, True, iPush, True, iLogic_Or, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, True, iPush, False, iLogic_Or, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, False, iPush, True, iLogic_Or, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, False, iPush, False, iLogic_Or, Empty), Array()) = False + + 'iLogic_Not + 'A | 1 0 + '----------- + 'O | 0 1 + Debug.Assert evaluate(testCreateOps(iPush, True, iLogic_Not, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, False, iLogic_Not, Empty), Array()) = True + + 'iLogic_Xor + 'A | 1 1 0 0 + 'B | 1 0 1 0 + '----------- + 'O | 0 1 1 0 + Debug.Assert evaluate(testCreateOps(iPush, True, iPush, True, iLogic_Xor, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, True, iPush, False, iLogic_Xor, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, False, iPush, True, iLogic_Xor, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, False, iPush, False, iLogic_Xor, Empty), Array()) = False + + 'iFunc_Call - Use `Len` as an example + Debug.Assert evaluate(testCreateOps(iPush, "Len", iPush, "TestString", iPush, 1, iFunc_Call, Empty), Array()) = Len("TestString") + + 'Test oFunctExts + Set This.FunctionExtensions = CreateObject("Scripting.Dictionary") + This.FunctionExtensions.Add "Proxy", stdLambda.Create("$1") + Debug.Assert evaluate(testCreateOps(iPush, "Proxy", iPush, "TestString", iPush, 1, iFunc_Call, Empty), Array()) = "TestString" + 'Ensure returns objects too + Debug.Assert TypeName(evaluate(testCreateOps(iPush, "Proxy", iPush, Application, iPush, 1, iFunc_Call, Empty), Array())) = "Application" + Set This.FunctionExtensions = Nothing + + 'TODO: Test other functions + + 'Comparison operations + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_Equal, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iComparison_Equal, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_NotEqual, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iComparison_NotEqual, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 20, iComparison_GreaterThan, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 9, iComparison_GreaterThan, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_GreaterThan, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 11, iComparison_GreaterThan, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 9, iComparison_GreaterThanOrEqual, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_GreaterThanOrEqual, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 11, iComparison_GreaterThanOrEqual, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 9, iComparison_LessThan, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_LessThan, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 11, iComparison_LessThan, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 9, iComparison_LessThanOrEqual, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 10, iComparison_LessThanOrEqual, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, 10, iPush, 11, iComparison_LessThanOrEqual, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, "A", iPush, "B", iComparison_Like, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, "A", iPush, "A", iComparison_Like, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, "AB", iPush, "A*", iComparison_Like, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, "ABC", iPush, "A*", iComparison_Like, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, "AB", iPush, "A?", iComparison_Like, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, "ABC", iPush, "A?", iComparison_Like, Empty), Array()) = False + + Dim o1 As Collection: Set o1 = New Collection + Dim o2 As Collection: Set o2 = New Collection + Dim o3 As Collection + Debug.Assert evaluate(testCreateOps(iPush, o1, iPush, o2, iComparison_Is, Empty), Array()) = False + Debug.Assert evaluate(testCreateOps(iPush, o1, iPush, o1, iComparison_Is, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, Nothing, iPush, Nothing, iComparison_Is, Empty), Array()) = True + Debug.Assert evaluate(testCreateOps(iPush, o3, iPush, Nothing, iComparison_Is, Empty), Array()) = True + + 'Miscellaneous operations + Debug.Assert evaluate(testCreateOps(iPush, "Hello", iPush, " World", iMisc_Concatenate, Empty), Array()) = "Hello World" + + 'Jump instructions + 'These change instruction pointer, need careful sequencing for observable results + 'iJump_Unconditional + '00: Push 10 + '01: Jump to 3 + '02: Push 20 (skipped) + '03: + 'Should return 10 + Debug.Assert evaluate(testCreateOps(iPush, 10, iJump_Unconditional, 3, iPush, 20), Array()) = 10 + + 'iJump_IfTrue + '00 Push True + '01 JumpIfTrue 4 + '02 Push 10 (skipped) + '03 JumpUnconditional 5 (skipped) + '04 Push 20 + '05 + 'Should return 20. + Debug.Assert evaluate(testCreateOps(iPush, True, iJump_IfTrue, 4, iPush, 10, iJump_Unconditional, 5, iPush, 20), Array()) = 20 + + 'Counter-example for iJump_IfTrue + '00 Push False + '01 JumpIfTrue 4 + '02 Push 10 + '03 JumpUnconditional 5 + '04 Push 20 (skipped) + '05 + 'Should return 10. + Debug.Assert evaluate(testCreateOps(iPush, False, iJump_IfTrue, 4, iPush, 10, iJump_Unconditional, 5, iPush, 20), Array()) = 10 + + 'iJump_IfFalse (inverse of the above) + Debug.Assert evaluate(testCreateOps(iPush, False, iJump_IfFalse, 4, iPush, 10, iJump_Unconditional, 5, iPush, 20), Array()) = 20 + Debug.Assert evaluate(testCreateOps(iPush, True, iJump_IfFalse, 4, iPush, 10, iJump_Unconditional, 5, iPush, 20), Array()) = 10 + + 'TODO: iReturn_WithValue - this is a utility used for returning values from functions, need to think how to test it properly + 'TODO: iReturn_NoValue - as above. + + + 'iObject_PropGet + 'TEST1 - With object return value + '00: Push Application 'Application _ + '01: Push "ThisWorkbook" ' .ThisWorkbook + '02: Object_PropGet + 'Should return `ThisWorkbook` + Debug.Assert evaluate(testCreateOps(iPush, Application, iPush, "ThisWorkbook", iObject_PropGet, Empty), Array()) Is ThisWorkbook + + 'TEST2 - With literal return value + '00: Push Application 'Application _ + '01: Push "ThisWorkbook" ' .Name + '02: Object_PropGet + 'Should return Application.Name e.g. "Microsoft Excel" + Debug.Assert evaluate(testCreateOps(iPush, Application, iPush, "Name", iObject_PropGet, Empty), Array()) = Application.Name + + 'TEST3 - With parameters + '00: Push Application 'Application _ + '01: Push "Range" ' .Range( _ + '02: Push "A1" ' "A1") + '03: Push 1 '(1 arg) + '04: Object_PropGet + 'Should return `Range("A1")` but note these won't return the same objects so `is` will still fail. + 'Instead we check the type and address. + Debug.Assert TypeName(evaluate(testCreateOps(iPush, Application, iPush, "Range", iPush, "A1", iPush, 1, iObject_PropGet, Empty), Array())) = "Range" + Debug.Assert evaluate(testCreateOps(iPush, Application, iPush, "Range", iPush, "A1", iPush, 1, iObject_PropGet, Empty), Array()).Address = "$A$1" + + 'TODO: iObject_MethodCall + 'TODO: iObject_MethodCall_WithArgs + 'TODO: iObject_FieldCall as method + 'TODO: iObject_FieldCall_WithArgs as method + 'TODO: iObject_FieldCall as property + 'TODO: iObject_FieldCall_WithArgs as property + 'TODO: iObject_PropLet + 'TODO: iObject_PropLet_WithArgs + 'TODO: iObject_PropSet + 'TODO: iObject_PropSet_WithArgs +End Sub diff --git a/tests/tooling/test_office_runner.py b/tests/tooling/test_office_runner.py new file mode 100644 index 00000000..b90f0449 --- /dev/null +++ b/tests/tooling/test_office_runner.py @@ -0,0 +1,43 @@ +"""Opt-in integration tests for the shared Office runner's failure semantics.""" +import importlib.util +from pathlib import Path +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'Tools/Testing')) +spec = importlib.util.spec_from_file_location('stdvba_runner', ROOT / 'Tools/Testing/run.py') +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +class OfficeLifecycleTests(unittest.TestCase): + def execute(self, body): + with tempfile.TemporaryDirectory(prefix='stdVBA-runner-contract-') as folder: + path = Path(folder) / 'RunnerContract.bas' + source = 'Attribute VB_Name = "RunnerContract"\nPublic Sub RunTest()\n' + body + '\nEnd Sub\n' + path.write_bytes(source.replace('\n', '\r\n').encode()) + suite = dict(id='runner.contract', kind='vba', files=[str(path)], + cases={'Contract': 'RunnerContract.RunTest'}, hosts=['Excel']) + return runner.execute(suite, 'Contract', 'Excel') + + def test_suppressed_assertion_failure_is_not_a_pass(self): + result = self.execute(' On Error Resume Next\n Test.Assert "deliberate failure", False') + self.assertEqual(result['status'], 'failed') + self.assertEqual(result['failures'], 1) + self.assertIn('deliberate failure', result['detail']) + + def test_unexpected_error_is_not_a_pass(self): + result = self.execute(' Err.Raise 5, , "deliberate runtime error"') + self.assertEqual(result['status'], 'failed') + self.assertIn('deliberate runtime error', result['detail']) + + def test_empty_case_is_not_a_pass(self): + result = self.execute(' Dim unused As Long') + self.assertEqual(result['status'], 'failed') + self.assertIn('No assertions ran', result['detail']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/tooling/test_regex_build.py b/tests/tooling/test_regex_build.py new file mode 100644 index 00000000..0b10f8db --- /dev/null +++ b/tests/tooling/test_regex_build.py @@ -0,0 +1,102 @@ +from pathlib import Path +import sys +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "Tools/stdRegex-build")) +"""Version-selection and failure-safety tests for the standalone-class builder.""" +from contextlib import redirect_stderr, redirect_stdout +import io +import json +from pathlib import Path +import subprocess +from tempfile import TemporaryDirectory +import unittest +from unittest.mock import patch + +import build + + +class BuildSelectionTests(unittest.TestCase): + def setUp(self): + self.temp = TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + root = Path(self.temp.name) + self.here = root / 'stdVBA' / 'Tools' / 'stdRegex-build' + self.here.mkdir(parents=True) + self.target = root / 'stdVBA' / 'src' / 'stdRegex.cls' + self.target.parent.mkdir() + self.target.write_bytes(b'previous class\r\n') + self.engine = root / 'engine' + self.engine.mkdir() + self.git('init', '-q') + self.git('config', 'user.name', 'Builder test') + self.git('config', 'user.email', 'test@example.invalid') + (self.engine / 'LICENSE').write_text('first engine revision') + self.git('add', 'LICENSE') + self.git('commit', '-q', '-m', 'first') + self.first = self.git('rev-parse', 'HEAD').strip() + (self.engine / 'LICENSE').write_text('second engine revision') + self.git('commit', '-qam', 'second') + self.second = self.git('rev-parse', 'HEAD').strip() + self.manifest = self.here / 'engine.json' + self.manifest.write_text(json.dumps({'repository': 'https://github.com/sihlfall/vba-regex', 'revision': self.first}) + '\n') + self.saved_manifest = self.manifest.read_bytes() + self.saved_class = self.target.read_bytes() + + def git(self, *args): + return subprocess.check_output(['git', '-C', str(self.engine), *args], text=True) + + def invoke(self, *args, fail=False): + def generate(root, selected): + if fail: + raise ValueError('incompatible engine') + return 'generated ' + selected['revision'] + '\n' + with patch.object(build, 'HERE', self.here), patch.object(build, 'generate', side_effect=generate), redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + build.main(['--engine-root', str(self.engine), *args]) + + def test_default_reproduces_recorded_commit_even_when_head_is_newer(self): + self.invoke() + self.assertEqual(self.target.read_text(), 'generated ' + self.first + '\n') + self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) + + def test_ref_override_does_not_adopt_revision_implicitly(self): + self.invoke('--engine-ref', 'HEAD') + self.assertEqual(self.target.read_text(), 'generated ' + self.second + '\n') + self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) + + def test_update_records_resolved_sha_and_matching_class(self): + self.invoke('--engine-ref', 'HEAD', '--update-engine') + self.assertEqual(json.loads(self.manifest.read_text())['revision'], self.second) + self.assertEqual(self.target.read_bytes(), ('generated ' + self.second + '\r\n').encode()) + + def test_update_requires_a_ref(self): + with self.assertRaises(SystemExit): + self.invoke('--update-engine') + self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) + self.assertEqual(self.target.read_bytes(), self.saved_class) + + def test_check_is_read_only(self): + self.target.write_text('generated ' + self.first + '\n') + before = self.target.read_bytes() + self.invoke('--check') + self.assertEqual(self.target.read_bytes(), before) + self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) + + def test_check_cannot_update_manifest(self): + with self.assertRaises(SystemExit): + self.invoke('--check', '--engine-ref', 'HEAD', '--update-engine') + self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) + + def test_incompatible_engine_does_not_change_existing_files(self): + with self.assertRaises(ValueError): + self.invoke('--engine-ref', 'HEAD', '--update-engine', fail=True) + self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) + self.assertEqual(self.target.read_bytes(), self.saved_class) + + def test_missing_revision_does_not_change_existing_files(self): + with self.assertRaises(subprocess.CalledProcessError): + self.invoke('--engine-ref', 'missing-engine-ref', '--update-engine') + self.assertEqual(self.manifest.read_bytes(), self.saved_manifest) + self.assertEqual(self.target.read_bytes(), self.saved_class) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/tooling/test_runner.py b/tests/tooling/test_runner.py new file mode 100644 index 00000000..bd5bae97 --- /dev/null +++ b/tests/tooling/test_runner.py @@ -0,0 +1,51 @@ +"""Contract checks for suite selection and generated VBA bridges; no Office needed.""" +import importlib.util +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('stdvba_runner', ROOT / 'Tools/Testing/run.py') +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +class RunnerTests(unittest.TestCase): + def setUp(self): + self.suites = runner.load_manifest() + + def test_every_root_test_module_registered(self): + registered = {f for s in self.suites for f in s.get('files', [])} + adapter_modules = {'stdImageTests', 'stdHTTPLocalTests', 'stdWebSocketLocalTests', + 'stdWebSocketAsyncTests', 'stdWindowSubclassTests'} + for path in (ROOT / 'tests').glob('*Tests.bas'): + self.assertTrue(path.relative_to(ROOT).as_posix() in registered or path.stem in adapter_modules, path) + + def test_individual_case_does_not_select_siblings(self): + selected = runner.select(self.suites, ['STDCALLBACK'], 'bind') + self.assertEqual([(s['id'], c) for s, c in selected], [('stdCallback', 'Bind')]) + + def test_unknown_selectors_fail(self): + for names, case in [(['missing'], None), (['stdCallback'], 'missing'), (None, 'Bind')]: + with self.assertRaises(ValueError): + runner.select(self.suites, names, case) + + def test_all_includes_opt_in_and_unavailable_suites(self): + selected = {s['id'] for s, c in runner.select(self.suites, all_suites=True)} + self.assertEqual(selected, {s['id'] for s in self.suites}) + + def test_bridge_checks_failures_and_zero_assertions(self): + suite, case = runner.select(self.suites, ['stdCallback'], 'Bind')[0] + source = runner.bridge(suite, case) + self.assertIn('Call stdCallbackTests.TestBind', source) + self.assertIn('Test.failures = 0', source) + self.assertIn('No assertions ran', source) + self.assertNotIn('On Error Resume Next', source) + + def test_return_contracts_are_checked(self): + for name, expected in [('stdJSON', 'Left$(CStr(result), 4)'), ('stdRefArray', 'result > 0'), ('stdTimer', 'Left$(CStr(result), 4)')]: + suite, case = runner.select(self.suites, [name])[0] + self.assertIn(expected, runner.bridge(suite, case)) + + +if __name__ == '__main__': + unittest.main() From 73788e3a94052735bf081818acce07fca59c405a Mon Sep 17 00:00:00 2001 From: Sancarn Date: Sun, 1 Nov 2026 19:16:05 +0000 Subject: [PATCH 2/2] Add stdArray ownership-transfer constructor (#122) --- changelog.md | 2 + docs.json | 71 +++++++++++++++++++++++++++++++++- src/stdArray.cls | 85 +++++++++++++++++++++++++++++++++++++++++ tests/stdArrayTests.bas | 70 +++++++++++++++++++++++++++++++++ 4 files changed, 227 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index b985af8a..648d9df1 100644 --- a/changelog.md +++ b/changelog.md @@ -484,3 +484,5 @@ The 2025-2026 entries were audited retrospectively against commits through `1bcb - 2026-11-01 `Tools/Testing` DEV - Add one manifest-driven command for aggregate, class and named-case tests, structured results, isolated Office workers, dialog/time limits and exact settings restoration. Move specialized test assets under root tests and register Python/native checks. Existing commands inside mixed build-tool folders remain compatibility shims. Split Callback and Regex into independently callable case groups; explicitly report unsupported hosts and legacy fixtures as skipped. - 2026-11-01 `stdLambda` BREAKING - Remove the production test-only protRunVMTests member and its private test helpers. Developers calling stdLambda.protRunVMTests must instead run `python Tools/Testing/run.py --suite stdLambda.vm --test Instructions`; the runner injects tests from tests/support into a disposable class copy. Normal Lambda evaluation APIs, defaults and callback contracts are unchanged. - 2026-11-01 `Tools/Testing` BREAKING - Remove the separate Tools/stdCallback-tests, stdDate-test, stdHTTP-tests, stdImage-tests stdWebSocket-tests and stdEnumerator-tests command folders. Maintainer scripts invoking those paths must migrate to `python Tools/Testing/run.py --suite `: stdCallback.probes, stdDate, stdHTTP.local, stdImage stdWebSocket.local or stdEnumerator.local respectively. Image host selection is now `--host Excel` or `--host Word` through the shared command (formerly `--hosts`). Test bodies and specialized harnesses now live under tests/harnesses. + +- 2026-11-01 `stdArray` WIP - Add experimental Windows CreateFromArrayByRef ownership transfer for non-empty dynamic one-dimensional Variant arrays. Consumes the source and uses DefaultBase; rejects fixed-size, locked, unallocated and multidimensional arrays. Existing copying APIs are unchanged; no breaking changes identified. Windows 32-bit/legacy VBA and Mac runtime support remain unverified (#122). diff --git a/docs.json b/docs.json index 980a9339..c23548eb 100644 --- a/docs.json +++ b/docs.json @@ -1147,6 +1147,35 @@ ], "requires": [], "methods": [ + { + "name": "protInitFromArrayByRef", + "description": "Take ownership of array storage without copying elements.", + "remarks": [], + "examples": [], + "params": [ + { + "name": "arr", + "type": "Variant", + "description": "Non-empty dynamic one-dimensional Variant array; consumed on success.", + "optional": false, + "defaultValue": null, + "paramArray": false, + "referenceType": "ByRef" + } + ], + "returns": null, + "deprecation": { + "status": false, + "message": "" + }, + "isDefaultMember": false, + "devNotes": [], + "todos": [], + "isProtected": true, + "throws": [], + "requires": [], + "isStatic": false + }, { "name": "protInit", "description": "Initialise array", @@ -2540,6 +2569,46 @@ "requires": [], "isStatic": false }, + { + "name": "CreateFromArrayByRef", + "description": "Create a stdArray by taking ownership of a dynamic one-dimensional Variant array.", + "remarks": [ + "Experimental Windows API. Transfers ownership, rather than sharing storage. Existing copying APIs are unchanged." + ], + "examples": [ + "```vb\r\nDim values() As Variant: ReDim values(0 To 2)\r\nvalues(0) = \"hello\"\r\nDim list As stdArray: Set list = stdArray.CreateFromArrayByRef(values)\r\n'values is now unallocated; list.Item(list.Base) is \"hello\".\r\n```" + ], + "params": [ + { + "name": "arr", + "type": "Variant", + "description": "Non-empty dynamic Variant array. Becomes unallocated on success.", + "optional": false, + "defaultValue": null, + "paramArray": false, + "referenceType": "ByRef" + } + ], + "returns": { + "type": "stdArray", + "description": "A stdArray using the transferred storage and current DefaultBase." + }, + "deprecation": { + "status": false, + "message": "" + }, + "isDefaultMember": false, + "devNotes": [], + "todos": [], + "isProtected": false, + "throws": [ + { + "errNumber": null + } + ], + "requires": [], + "isStatic": false + }, { "name": "CreateFromCollection", "description": "Create a `stdArray` object from a collection", @@ -19479,4 +19548,4 @@ "events": [], "implements": [] } -] +] \ No newline at end of file diff --git a/src/stdArray.cls b/src/stdArray.cls index 217508ad..a5e21cdc 100644 --- a/src/stdArray.cls +++ b/src/stdArray.cls @@ -207,6 +207,91 @@ Public Function CreateFromArray(ByVal arr As Variant) As stdArray RaiseEvent AfterCreate(Create) End Function +'Create a stdArray by taking ownership of a dynamic one-dimensional Variant array. +'@constructor +'@param arr - Non-empty dynamic Variant array. Becomes unallocated on success. +'@returns - A stdArray using the transferred storage and current DefaultBase. +'@remark Experimental Windows API. Transfers ownership, rather than sharing storage. Existing copying APIs are unchanged. +'@throws stdArray::CreateFromArrayByRef - Error 5 for fixed-size, locked, multidimensional, empty or unallocated arrays; source remains unchanged. +'@example ```vb +'Dim values() As Variant: ReDim values(0 To 2) +'values(0) = "hello" +'Dim list As stdArray: Set list = stdArray.CreateFromArrayByRef(values) +''values is now unallocated; list.Item(list.Base) is "hello". +'``` +Public Function CreateFromArrayByRef(ByRef arr As Variant) As stdArray + Set CreateFromArrayByRef = New stdArray + Call CreateFromArrayByRef.protInitFromArrayByRef(arr) + RaiseEvent AfterCreate(CreateFromArrayByRef) +End Function + +'Take ownership of array storage without copying elements. +'@protected +'@param arr - Non-empty dynamic one-dimensional Variant array; consumed on success. +Public Sub protInitFromArrayByRef(ByRef arr As Variant) +#If Mac Then + Err.Raise 5, "stdArray::CreateFromArrayByRef", "Windows required" +#Else + If This.Initialised Then Err.Raise 5, "stdArray::CreateFromArrayByRef", "Already initialised" + #If VBA7 Then + Dim sourceSlot As LongPtr, targetSlot As LongPtr, descriptor As LongPtr, nullPointer As LongPtr + #Else + Dim sourceSlot As Long, targetSlot As Long, descriptor As Long, nullPointer As Long + #End If + sourceSlot = ArrayPointerSlot(arr) + Call CopyMemory(VarPtr(descriptor), sourceSlot, LenB(descriptor)) + If descriptor = 0 Then Err.Raise 5, "stdArray::CreateFromArrayByRef", "Allocated array required" + Dim dimensions As Integer, flags As Integer, locks As Long + Call CopyMemory(VarPtr(dimensions), descriptor, 2) + Call CopyMemory(VarPtr(flags), descriptor + 2, 2) + Call CopyMemory(VarPtr(locks), descriptor + 8, 4) + If dimensions <> 1 Or (flags And &H17) <> 0 Or locks <> 0 Then + Err.Raise 5, "stdArray::CreateFromArrayByRef", "Dynamic unlocked one-dimensional array required" + End If + Dim count As Long, firstIndex As Long, boundsOffset As Long + #If Win64 Then + boundsOffset = 24 + #Else + boundsOffset = 16 + #End If + Call CopyMemory(VarPtr(count), descriptor + boundsOffset, 4) + If count <= 0 Then Err.Raise 5, "stdArray::CreateFromArrayByRef", "Non-empty array required" + This.Base = stdArray.DefaultBase + This.Length = count + This.ProxyLength = count + This.Chunking = 10 + targetSlot = ArrayPointerSlot(This.BaseArray) + firstIndex = 1 + 'Validation is complete; no callbacks run while ownership changes. + Call CopyMemory(descriptor + boundsOffset + 4, VarPtr(firstIndex), 4) + Call CopyMemory(sourceSlot, VarPtr(nullPointer), LenB(nullPointer)) + Call CopyMemory(targetSlot, VarPtr(descriptor), LenB(descriptor)) + This.Initialised = True +#End If +End Sub + +'Locate the SAFEARRAY pointer slot of a typed array passed through a Variant. +#If VBA7 Then +Private Function ArrayPointerSlot(ByRef value As Variant) As LongPtr +#Else +Private Function ArrayPointerSlot(ByRef value As Variant) As Long +#End If +#If Mac Then + Err.Raise 5, "stdArray::ArrayPointerSlot", "Windows required" +#Else + Dim vt As Integer + Call CopyMemory(VarPtr(vt), VarPtr(value), 2) + If vt = (vbArray Or vbVariant) Then + ArrayPointerSlot = VarPtr(value) + 8 + Exit Function + End If + If vt <> (vbArray Or vbVariant Or &H4000) Then + Err.Raise 5, "stdArray::ArrayPointerSlot", "Typed Variant array required" + End If + Call CopyMemory(VarPtr(ArrayPointerSlot), VarPtr(value) + 8, LenB(ArrayPointerSlot)) +#End If +End Function + 'Create a `stdArray` object from a collection '@constructor '@param col - Collection to create a `stdArray` object from. diff --git a/tests/stdArrayTests.bas b/tests/stdArrayTests.bas index 31b4e710..505e9027 100644 --- a/tests/stdArrayTests.bas +++ b/tests/stdArrayTests.bas @@ -53,6 +53,7 @@ Sub testAll() Test.Assert "Property Arr 4 item 3 equal", vArr(3) = arr.item(3) end if + Call testCreateFromArrayByRef Call testDefaultBase Call testArrLet @@ -354,3 +355,72 @@ Failed: stdArray.DefaultBase = originalBase Err.Raise failure, "stdArrayTests.testDefaultBase", description End Sub + +Public Sub testCreateFromArrayByRef() + Test.Topic "stdArray ownership transfer" + Dim values() As Variant: ReDim values(-2 To 0) + values(-2) = "hello" + Dim obj As Collection: Set obj = New Collection + obj.Add "alive" + Set values(-1) = obj + values(0) = Array(3, 4) + Dim list As stdArray: Set list = stdArray.CreateFromArrayByRef(values) + Test.Assert "Moved length", list.Length = 3 + Test.Assert "Moved string", list.Item(list.Base) = "hello" + Test.Assert "Moved object identity", list.Item(list.Base + 1) Is obj + Dim nested As Variant: nested = list.Item(list.Base + 2) + Test.Assert "Moved nested array", nested(1) = 4 + Dim n As Long, failure As Long + On Error Resume Next + n = LBound(values) + failure = Err.Number + Err.Clear + On Error GoTo 0 + Test.Assert "Source unallocated", failure = 9 + ReDim values(0 To 1) + values(0) = "independent" + Test.Assert "Source reusable", list.Item(list.Base) = "hello" + list.Push 5 + Test.Assert "Moved array grows", list.Length = 4 + Test.Assert "Moved array clones", list.Clone.Length = 4 + Set list = Nothing + Test.Assert "Object survives cleanup", obj.Item(1) = "alive" + Dim fixed(0 To 2) As Variant + fixed(0) = "retained" + On Error Resume Next + Set list = stdArray.CreateFromArrayByRef(fixed) + failure = Err.Number + Err.Clear + On Error GoTo 0 + Test.Assert "Reject fixed", failure = 5 + Test.Assert "Rejected source unchanged", fixed(0) = "retained" + Dim matrix() As Variant: ReDim matrix(0 To 1, 0 To 1) + On Error Resume Next + Set list = stdArray.CreateFromArrayByRef(matrix) + failure = Err.Number + Err.Clear + On Error GoTo 0 + Test.Assert "Reject multidimensional", failure = 5 + Dim unallocated() As Variant + On Error Resume Next + Set list = stdArray.CreateFromArrayByRef(unallocated) + failure = Err.Number + Err.Clear + On Error GoTo 0 + Test.Assert "Reject unallocated", failure = 5 + Dim v As Variant: v = Array(8, 9) + Set list = stdArray.CreateFromArrayByRef(v) + Test.Assert "Variant containing array", list.Join = "8,9" + v = Array(10) + Test.Assert "Variant source reusable", list.Join = "8,9" + Dim locked() As Variant: ReDim locked(0 To 1) + Dim element As Variant + For Each element In locked + On Error Resume Next + Set list = stdArray.CreateFromArrayByRef(locked) + failure = Err.Number + Err.Clear + On Error GoTo 0 + Test.Assert "Reject locked array", failure = 5 + Next +End Sub