From 5529b8d22c8643f96669016a2f15126c3aa634e6 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Thu, 2 Apr 2026 14:04:14 -0700 Subject: [PATCH 01/17] fix: Fix a flaky test that occurs due to a race condition. --- lib/fact/index_checkpoint_file.ex | 1 + test/test_helper.exs | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/fact/index_checkpoint_file.ex b/lib/fact/index_checkpoint_file.ex index 4562f7b..3bccd46 100644 --- a/lib/fact/index_checkpoint_file.ex +++ b/lib/fact/index_checkpoint_file.ex @@ -98,6 +98,7 @@ defmodule Fact.IndexCheckpointFile do decoded else {:error, :enoent} -> 0 + {:error, {:decode, _}} -> 0 end end diff --git a/test/test_helper.exs b/test/test_helper.exs index bb3eefa..6c72452 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,9 +1,6 @@ defmodule TestHelper do require Logger - @doc """ - Test helper method to create a database with a random name. - """ @doc """ Creates a CAS-mode database (hash@1 record file name) for MMR testing. """ @@ -29,6 +26,9 @@ defmodule TestHelper do path end + @doc """ + Creates a database with a random name. + """ def create_db(name_prefix) do path = :uuid.get_v4() From d0ffa1e045b2f975fee6882b2d8f8fd80fe3ce4d Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Wed, 1 Apr 2026 23:54:58 -0700 Subject: [PATCH 02/17] feat: Adds encryption at rest --- guides/components/encryption.md | 224 ++++++++++++++ lib/fact.ex | 19 ++ lib/fact/bootstrap_file.ex | 30 +- lib/fact/bootstrapper.ex | 110 ++++++- lib/fact/database_supervisor.ex | 82 ++++-- lib/fact/encryption/kek_provider.ex | 36 +++ lib/fact/encryption/kek_provider/explicit.ex | 46 +++ lib/fact/genesis/decider.ex | 39 ++- lib/fact/genesis/the_creator.ex | 56 +++- lib/fact/index_checkpoint_file.ex | 1 + lib/fact/index_file.ex | 1 + lib/fact/key_ring.ex | 111 +++++++ lib/fact/ledger_file.ex | 1 + lib/fact/lock_file.ex | 1 + lib/fact/record_file.ex | 13 +- lib/fact/seam/file_reader/encrypted/v1.ex | 89 ++++++ lib/fact/seam/file_reader/registry.ex | 1 + lib/fact/seam/file_writer/encrypted/v1.ex | 137 +++++++++ lib/fact/seam/file_writer/registry.ex | 2 +- lib/fact/supervisor.ex | 24 ++ lib/mix/tasks/fact/create.ex | 104 +++++-- lib/mix/tasks/fact/recover.ex | 196 +++++++++++++ mix.exs | 2 + test/fact/encryption_test.exs | 294 +++++++++++++++++++ 24 files changed, 1531 insertions(+), 88 deletions(-) create mode 100644 guides/components/encryption.md create mode 100644 lib/fact/encryption/kek_provider.ex create mode 100644 lib/fact/encryption/kek_provider/explicit.ex create mode 100644 lib/fact/key_ring.ex create mode 100644 lib/fact/seam/file_reader/encrypted/v1.ex create mode 100644 lib/fact/seam/file_writer/encrypted/v1.ex create mode 100644 lib/mix/tasks/fact/recover.ex create mode 100644 test/fact/encryption_test.exs diff --git a/guides/components/encryption.md b/guides/components/encryption.md new file mode 100644 index 0000000..744549c --- /dev/null +++ b/guides/components/encryption.md @@ -0,0 +1,224 @@ +# Encryption at Rest + +This guide explains Fact's encryption-at-rest feature, which protects event record files on disk using +authenticated encryption. It covers the motivation, how to create and open an encrypted database, key +management, and recovery procedures. + +At a high level: + +- Event record files are encrypted using **AES-256-GCM** (authenticated encryption) +- Encryption is configured at **database creation time** and backed by the seam architecture +- Keys use **envelope encryption** — a Data Encryption Key (DEK) encrypts records, and a Key Encryption + Key (KEK) wraps the DEK +- The DEK never appears in plaintext on disk +- A **recovery KEK** is required at creation time to prevent permanent data loss + +## Motivation + +Event stores accumulate history that is never deleted. This makes them high-value targets for data +exfiltration — a single disk image or backup copy exposes every event ever written. Encryption at rest +ensures that even if the underlying storage is compromised, the event data remains unreadable without +the encryption key. + +Fact encrypts at the **record file level**. Each event record file on disk is individually encrypted with +its own random nonce, providing strong confidentiality guarantees even if an attacker obtains partial +access to the storage directory. + +Ledger files, index files, checkpoint files, and lock files are **not encrypted**. These contain only +record IDs (opaque identifiers) and operational metadata, not event payloads. + +## Envelope Encryption + +Fact uses envelope encryption with two layers of keys: + +- **Data Encryption Key (DEK)** — A 256-bit key generated at database creation time. Used to encrypt + and decrypt every event record file. One DEK per database. +- **Key Encryption Key (KEK)** — Supplied by the operator at runtime. Wraps (encrypts) the DEK so it + can be stored safely on disk. Never touches disk in plaintext. +- **Recovery KEK** — A second KEK that can also unwrap the DEK. Required at creation time. Stored + separately by the operator for disaster recovery. + +The wrapped DEK (encrypted with both the primary and recovery KEKs) is stored in the `.bootstrap` file +alongside the database configuration. The KEKs themselves are never written to disk. + +``` + ┌─────────────┐ + │ Primary KEK │ (runtime only, never on disk) + └──────┬──────┘ + │ wraps + ▼ + ┌────────────────┐ + │ DEK │ (wrapped form stored in .bootstrap) + └────────┬───────┘ + │ encrypts + ▼ + ┌────────────────┐ + │ Record Files │ (encrypted on disk) + └────────────────┘ +``` + +## Creating an Encrypted Database + +Use `mix fact.create` with the `--encrypted` flag: + +```sh +$ mix fact.create -p data/secure_turtles --encrypted +``` + +The primary key and recovery key are **generated automatically** and displayed in the output: + +``` + ================================================================ + 🔐 ENCRYPTION KEYS — SAVE THESE NOW + ================================================================ + + KEY: dGhpcyBpcyBhIHNhbXBsZSBrZXkgZm9yIGRvY3M= + RECOVERY: cmVjb3Zlcnkga2V5IGZvciBkb2NzIHNhbXBsZQ== + + ================================================================ + + Store the key securely (secrets manager, environment variable). + Store the recovery key separately in a safe, offline location. + + These keys will NOT be shown again. If you lose both keys, + your data is unrecoverable. + ================================================================ +``` + +You can also provide your own keys via `--key` and `--recovery-key` (base64-encoded, 16, 24, or +32 bytes decoded): + +```sh +$ mix fact.create -p data/secure_turtles --encrypted \ + --key=$MY_KEY \ + --recovery-key=$MY_RECOVERY_KEY +``` + +Store the primary key securely (e.g., in a secrets manager or vault). Store the recovery key +in a separate, offline location (e.g., a printed key in a safe, a hardware security module). + +## Opening an Encrypted Database + +Supply the base64-encoded key via the `:encryption_key` option. The bootstrapper reads the +wrapped DEK from the `.bootstrap` file automatically — you only need to provide the key itself: + +```elixir +{:ok, db} = Fact.open("data/secure_turtles", + encryption_key: System.fetch_env!("FACT_KEY") +) +``` + +### In a Supervision Tree + +```elixir +# application.ex +def start(_type, _args) do + children = [ + {Fact.Supervisor, + databases: [ + {"data/secure_turtles", + encryption_key: System.fetch_env!("FACT_KEY")} + ]} + ] + + Supervisor.start_link(children, strategy: :one_for_one) +end +``` + +## KEK Providers + +For advanced use cases (vault integrations, per-database key resolution), Fact supports a provider +behaviour (`Fact.Encryption.KEKProvider`). The `:encryption_key` option uses the built-in +`Fact.Encryption.KEKProvider.Explicit` provider under the hood. + +Custom providers can integrate with external key management systems: + +```elixir +defmodule MyApp.VaultKEKProvider do + @behaviour Fact.Encryption.KEKProvider + + @impl true + def fetch_kek(opts) do + database_id = Keyword.fetch!(opts, :database_id) + {:ok, MyApp.Vault.get_key("fact/#{database_id}/kek")} + end +end +``` + +```elixir +Fact.open("data/secure_turtles", + encryption: [ + kek_provider: MyApp.VaultKEKProvider, + database_id: "my_db" + ] +) +``` + +## Recovery + +If the primary key is lost or compromised, use the `mix fact.recover` task with the recovery key. +This task generates a new DEK, a new primary key, and a new recovery key, then re-encrypts all +record files in place. + +```sh +$ mix fact.recover -p data/secure_turtles --recovery-key=$RECOVERY_KEY +``` + +The recovery task will: + +1. Unwrap the existing DEK using the recovery key +2. Decrypt all record files with the old DEK +3. Generate a new DEK, primary key, and recovery key +4. Re-encrypt all record files with the new DEK +5. Update the `.bootstrap` file with the new wrapped DEKs +6. Display the new keys + +The recovery key **cannot** be used directly with `Fact.open`. Recovery is an offline operation +that must be performed while the database is not running. This ensures the key rotation is atomic +and no reads or writes occur with stale keys. + +## File Format + +Each encrypted record file has the following binary layout: + +``` +┌──────────┬──────────────┬─────────────────┐ +│ Nonce │ Auth Tag │ Ciphertext │ +│ 12 bytes │ 16 bytes │ variable │ +└──────────┴──────────────┴─────────────────┘ +``` + +- **Nonce** (12 bytes) — A random initialization vector, unique per file. Generated using + `:crypto.strong_rand_bytes/1`. +- **Auth Tag** (16 bytes) — The GCM authentication tag. Verifies that the ciphertext has not been + tampered with. +- **Ciphertext** (variable) — The encrypted event record payload. + +Since record files are write-once (WORM), each file gets a unique nonce at write time. Nonce reuse +is impossible under normal operation. + +## What is Not Encrypted + +The following files remain in plaintext: + +| File Type | Contents | Why Not Encrypted | +|-----------|----------|-------------------| +| `.bootstrap` | Database config + wrapped DEK | Must be readable to bootstrap | +| `.ledger` | Record IDs (opaque identifiers) | Performance; no event payloads | +| Index files | Record IDs grouped by dimension | Performance; no event payloads | +| Checkpoint files | Integer positions | Operational metadata only | +| Lock files | Process metadata | Operational metadata only | + +Record IDs are opaque strings (UUIDs or content hashes) that do not reveal event content. The ledger +and index files enable efficient reads without decrypting every record on disk. + +## Security Considerations + +- **Key storage**: Never commit KEKs to source control or store them alongside the database. Use a + secrets manager, environment variables injected at deploy time, or a hardware security module. +- **Key rotation**: The DEK is permanent for the lifetime of the database. To rotate the KEK, unwrap + the DEK with the old KEK, re-wrap with a new KEK, and update the bootstrap file. +- **Backup encryption**: Database backups created with `mix fact.backup` include encrypted record files + as-is. The backup is only useful to someone who also has the KEK. +- **Memory**: The DEK is held in memory by the `Fact.KeyRing` process for the lifetime of the database. + It is not written to disk or logged. diff --git a/lib/fact.ex b/lib/fact.ex index 25da3c0..6fa7c60 100644 --- a/lib/fact.ex +++ b/lib/fact.ex @@ -628,6 +628,25 @@ defmodule Fact do Fact.Supervisor.start_database(path, opts) end + @doc """ + Closes a running database. + + Shuts down the database's supervision tree and releases its lock. After closing, + the database can be reopened with `open/2` or opened by another process. + + ## Examples + + iex> {:ok, db} = Fact.open("data/turtles") + iex> Fact.close(db) + :ok + + """ + @doc since: "0.3.1" + @spec close(database_id()) :: :ok | {:error, :not_found} + def close(database_id) when is_binary(database_id) do + Fact.Supervisor.stop_database(database_id) + end + @doc """ Read from an event source. diff --git a/lib/fact/bootstrap_file.ex b/lib/fact/bootstrap_file.ex index 6505245..8f568f3 100644 --- a/lib/fact/bootstrap_file.ex +++ b/lib/fact/bootstrap_file.ex @@ -186,19 +186,27 @@ defmodule Fact.BootstrapFile do It writes just enough configuration, so that the genesis record (`Fact.Genesis.Event.DatabaseCreated.V1`) can be read and a full `Fact.Context` can be loaded. """ - @doc since: "0.2.0" - @spec write(Path.t(), genesis_record :: Fact.record()) :: :ok | {:error, term()} - def write(path, {record_id, event}) do + @doc since: "0.3.1" + @spec write(Path.t(), genesis_record :: Fact.record(), keyword()) :: :ok | {:error, term()} + def write(path, {record_id, event}, opts \\ []) do this = get_bootstrap_write_context() - {:ok, bootstrap_record} = - Encoder.encode(this, %{ - record_id: record_id, - event_schema: event.event_schema, - record_file_decoder: event.record_file_decoder, - record_file_reader: event.record_file_reader, - storage: event.storage - }) + record = %{ + record_id: record_id, + event_schema: event.event_schema, + record_file_decoder: event.record_file_decoder, + record_file_reader: event.record_file_reader, + record_file_writer: event.record_file_writer, + storage: event.storage + } + + record = + case Keyword.get(opts, :encryption) do + nil -> record + encryption -> Map.put(record, :encryption, encryption) + end + + {:ok, bootstrap_record} = Encoder.encode(this, record) Writer.write(this, Path.join(path, Name.get(this)), bootstrap_record) end diff --git a/lib/fact/bootstrapper.ex b/lib/fact/bootstrapper.ex index d7597de..c29c443 100644 --- a/lib/fact/bootstrapper.ex +++ b/lib/fact/bootstrapper.ex @@ -71,14 +71,16 @@ defmodule Fact.Bootstrapper do @impl true @doc false def handle_continue(:bootstrap, %{path: path, caller: caller, opts: opts} = state) do - with {:ok, context} <- load_context(path) do + with {:ok, context, encryption_runtime_opts} <- load_context(path, opts) do + merged_opts = Keyword.merge(opts, encryption_runtime_opts) + case Fact.Registry.get_context(context.database_id) do {:ok, _} -> maybe_send(caller, {:database_started, context.database_id}) {:stop, :normal, state} {:error, _} -> - case start_database(context, opts) do + case start_database(context, merged_opts) do {:ok, _pid} -> maybe_send(caller, {:database_started, context.database_id}) {:stop, :normal, state} @@ -111,31 +113,117 @@ defmodule Fact.Bootstrapper do end end - defp load_context_from_bootstrap_file(path) do - with {:ok, bootstrap} <- Fact.BootstrapFile.read(path) do + defp load_context_from_bootstrap_file(path, opts) do + with {:ok, bootstrap} <- Fact.BootstrapFile.read(path), + {:ok, dek, encryption_runtime_opts} <- resolve_encryption(bootstrap, opts) do bootstrap_context = %Fact.Context{ event_schema: Fact.Event.Schema.from_config(bootstrap["event_schema"]), record_file_decoder: Fact.RecordFile.Decoder.from_config(bootstrap["record_file_decoder"]), - record_file_reader: Fact.RecordFile.Reader.from_config(bootstrap["record_file_reader"]), + record_file_reader: + Fact.RecordFile.Reader.from_config( + bootstrap["record_file_reader"] || + %{"family" => "full", "version" => 1, "options" => %{}} + ), storage: Fact.Storage.from_config(bootstrap["storage"]) } - {_, event} = Fact.RecordFile.read(bootstrap_context, bootstrap["record_id"]) + {_, event} = read_genesis_record(bootstrap_context, bootstrap["record_id"], dek) event_data = Map.get(event, Fact.Event.Schema.get(bootstrap_context).event_data) + context = Fact.Context.from_genesis_event_data(event_data) + + {:ok, context, encryption_runtime_opts} + end + end + + defp resolve_encryption(bootstrap, opts) do + case bootstrap["encryption"] do + nil -> + {:ok, nil, []} + + encryption_config -> + with {:ok, kek} <- fetch_kek(opts), + {:ok, wrapped_dek} <- get_wrapped_dek(encryption_config), + {:ok, dek} <- unwrap_dek(wrapped_dek, kek) do + {:ok, dek, [encryption: [wrapped_dek: wrapped_dek, kek: kek]]} + end + end + end + + # When encrypted, read raw file with Full.V1 and decrypt before decoding. + defp read_genesis_record(context, record_id, dek) when is_binary(dek) do + {:ok, record_path} = {:ok, Fact.Storage.records_path(context, record_id)} + + {:ok, stream} = + Fact.Seam.FileReader.Full.V1.read(%Fact.Seam.FileReader.Full.V1{}, record_path, []) + + encrypted_binary = Enum.at(stream, 0) + {:ok, plaintext} = Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(dek, encrypted_binary) + {:ok, record} = Fact.RecordFile.Decoder.decode(context, plaintext) + {record_id, record} + end + + defp read_genesis_record(context, record_id, nil) do + Fact.RecordFile.read(context, record_id) + end + + defp fetch_kek(opts) do + case Keyword.get(opts, :encryption_key) do + value when is_binary(value) -> + decode_encryption_key(value) - {:ok, Fact.Context.from_genesis_event_data(event_data)} + nil -> + encryption_opts = Keyword.get(opts, :encryption, []) + + case Keyword.get(encryption_opts, :kek_provider) do + nil -> + {:error, :encryption_key_required} + + provider -> + provider.fetch_kek(encryption_opts) + end + end + end + + defp decode_encryption_key(value) do + case Base.decode64(value) do + {:ok, key} when byte_size(key) in [16, 24, 32] -> + {:ok, key} + + {:ok, key} -> + {:error, {:invalid_encryption_key_size, byte_size(key)}} + + :error -> + {:error, {:invalid_encryption_key, "must be valid base64"}} + end + end + + defp unwrap_dek(wrapped_dek, kek) do + case Fact.KeyRing.unwrap_dek(wrapped_dek, kek) do + {:ok, dek} -> {:ok, dek} + {:error, :dek_unwrap_failed} -> {:error, :invalid_encryption_key} + error -> error end end - defp load_context(path) do - case load_context_from_bootstrap_file(path) do + defp get_wrapped_dek(encryption_config) do + {:ok, Base.decode64!(encryption_config["wrapped_dek_primary"])} + end + + defp load_context(path, opts) do + case load_context_from_bootstrap_file(path, opts) do {:error, :bootstraps_not_found} -> - load_context_v1(path) + case load_context_v1(path) do + {:ok, context} -> {:ok, context, []} + error -> error + end - {:ok, _context} = result -> + {:ok, _context, _encryption_opts} = result -> result + + {:error, _reason} = error -> + error end end diff --git a/lib/fact/database_supervisor.ex b/lib/fact/database_supervisor.ex index 731fec0..7083fa9 100644 --- a/lib/fact/database_supervisor.ex +++ b/lib/fact/database_supervisor.ex @@ -98,35 +98,41 @@ defmodule Fact.DatabaseSupervisor do Keyword.get(opts, :wal, []) cache_opts = Keyword.get(opts, :cache, []) + key_ring_children = key_ring_children(database_id, opts) - children = [ - {Registry, keys: :unique, name: Fact.Registry.registry(database_id)}, - {Phoenix.PubSub, name: Fact.Registry.pubsub(database_id)}, - {Fact.EventPublisher, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventPublisher)}, - {Fact.Database, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.Database)}, - {Fact.WriteAheadLog, wal_opts}, - {Fact.EventLedger, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventLedger)}, - {Fact.EventStreamIndexer, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventStreamIndexer)}, - {Fact.EventTagsIndexer, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventTagsIndexer)}, - {Fact.EventTypeIndexer, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventTypeIndexer)}, - {Fact.EventStreamCategoryIndexer, - database_id: database_id, - name: Fact.Registry.via(database_id, Fact.EventStreamCategoryIndexer)}, - {Fact.EventStreamsIndexer, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventStreamsIndexer)}, - {Fact.EventStreamsByCategoryIndexer, - database_id: database_id, - name: Fact.Registry.via(database_id, Fact.EventStreamsByCategoryIndexer)}, - {DynamicSupervisor, - strategy: :one_for_one, - name: Fact.Registry.via(database_id, Fact.EventStreamWriterSupervisor)} - ] + children = + [ + {Registry, keys: :unique, name: Fact.Registry.registry(database_id)} + ] ++ + key_ring_children ++ + [ + {Phoenix.PubSub, name: Fact.Registry.pubsub(database_id)}, + {Fact.EventPublisher, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventPublisher)}, + {Fact.Database, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.Database)}, + {Fact.WriteAheadLog, wal_opts}, + {Fact.EventLedger, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventLedger)}, + {Fact.EventStreamIndexer, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventStreamIndexer)}, + {Fact.EventTagsIndexer, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventTagsIndexer)}, + {Fact.EventTypeIndexer, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventTypeIndexer)}, + {Fact.EventStreamCategoryIndexer, + database_id: database_id, + name: Fact.Registry.via(database_id, Fact.EventStreamCategoryIndexer)}, + {Fact.EventStreamsIndexer, + database_id: database_id, + name: Fact.Registry.via(database_id, Fact.EventStreamsIndexer)}, + {Fact.EventStreamsByCategoryIndexer, + database_id: database_id, + name: Fact.Registry.via(database_id, Fact.EventStreamsByCategoryIndexer)}, + {DynamicSupervisor, + strategy: :one_for_one, + name: Fact.Registry.via(database_id, Fact.EventStreamWriterSupervisor)} + ] cache_children = case Keyword.get(cache_opts, :max_size) do @@ -149,6 +155,26 @@ defmodule Fact.DatabaseSupervisor do Supervisor.init(children ++ cache_children ++ merkle_children, strategy: :one_for_one) end + defp key_ring_children(database_id, opts) do + encryption_opts = Keyword.get(opts, :encryption, []) + + case Keyword.get(encryption_opts, :wrapped_dek) do + nil -> + [] + + wrapped_dek -> + kek = Keyword.fetch!(encryption_opts, :kek) + + [ + {Fact.KeyRing, + database_id: database_id, + wrapped_dek: wrapped_dek, + kek: kek, + name: Fact.Registry.via(database_id, Fact.KeyRing)} + ] + end + end + defp merkle_children(context, database_id, opts) do merkle_opts = merkle_opts(opts) diff --git a/lib/fact/encryption/kek_provider.ex b/lib/fact/encryption/kek_provider.ex new file mode 100644 index 0000000..7d04a64 --- /dev/null +++ b/lib/fact/encryption/kek_provider.ex @@ -0,0 +1,36 @@ +defmodule Fact.Encryption.KEKProvider do + @moduledoc """ + Behaviour for providing Key Encryption Keys (KEKs) at runtime. + + A KEK provider is responsible for supplying the key used to unwrap the + Data Encryption Key (DEK) when a database is opened. The DEK is stored + in the `.bootstrap` file in wrapped (encrypted) form and must be unwrapped + before the database can read or write encrypted record files. + + ## Implementations + + * `Fact.Encryption.KEKProvider.Explicit` — takes the KEK directly from options. + """ + @moduledoc since: "0.4.0" + + @typedoc """ + A module implementing the `Fact.Encryption.KEKProvider` behaviour. + """ + @typedoc since: "0.4.0" + @type t :: module() + + @typedoc """ + Options passed to `c:fetch_kek/1`. The shape depends on the provider implementation. + """ + @typedoc since: "0.4.0" + @type opts :: keyword() + + @doc """ + Fetches the Key Encryption Key. + + Returns `{:ok, kek}` where `kek` is a binary encryption key, or + `{:error, reason}` if the key cannot be retrieved. + """ + @doc since: "0.4.0" + @callback fetch_kek(opts()) :: {:ok, binary()} | {:error, term()} +end diff --git a/lib/fact/encryption/kek_provider/explicit.ex b/lib/fact/encryption/kek_provider/explicit.ex new file mode 100644 index 0000000..2e72ba8 --- /dev/null +++ b/lib/fact/encryption/kek_provider/explicit.ex @@ -0,0 +1,46 @@ +defmodule Fact.Encryption.KEKProvider.Explicit do + @moduledoc """ + A `Fact.Encryption.KEKProvider` that takes the KEK directly from the options. + + This is the simplest provider — the caller supplies the key material when opening + the database. Key sourcing (environment variables, vault clients, config files) is + the caller's responsibility. + + ## Examples + + Fact.open("data/turtles", + encryption: [ + kek_provider: Fact.Encryption.KEKProvider.Explicit, + kek: System.fetch_env!("FACT_KEK") |> Base.decode64!() + ] + ) + """ + @moduledoc since: "0.4.0" + + @behaviour Fact.Encryption.KEKProvider + + @doc """ + Fetches the KEK from the `:kek` option. + + Expects `opts` to contain a `:kek` key with a binary value of 16, 24, or 32 bytes + (corresponding to AES-128, AES-192, or AES-256). + """ + @doc since: "0.4.0" + @impl true + @spec fetch_kek(keyword()) :: {:ok, binary()} | {:error, term()} + def fetch_kek(opts) do + case Keyword.fetch(opts, :kek) do + {:ok, kek} when is_binary(kek) and byte_size(kek) in [16, 24, 32] -> + {:ok, kek} + + {:ok, kek} when is_binary(kek) -> + {:error, {:invalid_kek_size, byte_size(kek)}} + + {:ok, _} -> + {:error, :kek_must_be_binary} + + :error -> + {:error, :kek_not_provided} + end + end +end diff --git a/lib/fact/genesis/decider.ex b/lib/fact/genesis/decider.ex index 0a66062..e7a7385 100644 --- a/lib/fact/genesis/decider.ex +++ b/lib/fact/genesis/decider.ex @@ -25,7 +25,8 @@ defmodule Fact.Genesis.Decider do def decide(:initial_state, %CreateDatabase.V1{args: args} = _command) do with args <- resolve_name_and_path(args), :ok <- verify_path(Keyword.get(args, :path)), - {:ok, configuration} <- build_configuration(args) do + {:ok, configuration} <- build_configuration(args), + {:ok, encryption_meta} <- resolve_encryption(args, configuration) do info = %{ database_id: generate_database_id(), database_name: Keyword.get(args, :name), @@ -36,7 +37,8 @@ defmodule Fact.Genesis.Decider do otp_version: otp_version() } - {:ok, [struct(DatabaseCreated.V1, Map.merge(info, configuration))]} + event = struct(DatabaseCreated.V1, Map.merge(info, configuration)) + {:ok, [event], encryption_meta} end end @@ -271,6 +273,39 @@ defmodule Fact.Genesis.Decider do end end + defp resolve_encryption(args, %{record_file_writer: %{family: :encrypted}} = _config) do + with {:ok, kek} <- resolve_key(Keyword.get(args, :key)), + {:ok, recovery_kek} <- resolve_key(Keyword.get(args, :recovery_key)) do + dek = Fact.KeyRing.generate_dek() + + {:ok, + %{ + dek: dek, + kek: kek, + recovery_kek: recovery_kek, + wrapped_dek_primary: Fact.KeyRing.wrap_dek(dek, kek), + wrapped_dek_recovery: Fact.KeyRing.wrap_dek(dek, recovery_kek) + }} + end + end + + defp resolve_encryption(_args, _config), do: {:ok, nil} + + defp resolve_key(nil), do: {:ok, :crypto.strong_rand_bytes(32)} + + defp resolve_key(value) when is_binary(value) do + case Base.decode64(value) do + {:ok, key} when byte_size(key) in [16, 24, 32] -> + {:ok, key} + + {:ok, key} -> + {:error, {:invalid_key_size, byte_size(key)}} + + :error -> + {:error, {:invalid_key, "must be valid base64"}} + end + end + @hash_algorithm_bit_length %{ md5: 128, sha: 160, diff --git a/lib/fact/genesis/the_creator.ex b/lib/fact/genesis/the_creator.ex index a5f9b89..17b7600 100644 --- a/lib/fact/genesis/the_creator.ex +++ b/lib/fact/genesis/the_creator.ex @@ -21,10 +21,14 @@ defmodule Fact.Genesis.TheCreator do Creates a Fact database from the `Fact.Genesis.Event.DatabaseCreated.V1` event. It initializes storage, and appends the event as the first record of every database. + + When `encryption_meta` is provided, the genesis record is encrypted manually + (since the KeyRing process is not yet running) and the bootstrap file includes + the wrapped DEK for later decryption. """ - @doc since: "0.1.0" - @spec let_there_be_light(%DatabaseCreated.V1{}) :: :ok - def let_there_be_light(%DatabaseCreated.V1{} = event) do + @doc since: "0.3.1" + @spec let_there_be_light(%DatabaseCreated.V1{}, map() | nil) :: :ok + def let_there_be_light(%DatabaseCreated.V1{} = event, encryption_meta \\ nil) do with context <- DatabaseCreated.V1.to_context(event), {:ok, path} <- Storage.initialize_storage(context) do schema = Event.Schema.get(context) @@ -40,14 +44,54 @@ defmodule Fact.Genesis.TheCreator do schema.event_tags => ["__fact__:#{event.database_id}"] } - {:ok, record_id} = Fact.RecordFile.write(context, genesis) + {:ok, record_id} = write_genesis_record(context, genesis, encryption_meta) {:ok, ^record_id} = Fact.LedgerFile.write(context, record_id) + bootstrap_opts = build_bootstrap_opts(encryption_meta) + # writes the bare minimum into a special .bootstrap file - # with just enough to read the genesis event and start the system. - :ok = Fact.BootstrapFile.write(path, {record_id, event}) + # with just enough to read the genesis event and start the system. + :ok = Fact.BootstrapFile.write(path, {record_id, event}, bootstrap_opts) :ok end end + + # When encrypted, we encode the genesis record normally, then encrypt and write + # it manually. The KeyRing process doesn't exist yet during creation, so we + # can't use the encrypted FileWriter seam. + defp write_genesis_record(context, genesis, %{dek: dek} = _meta) do + with {:ok, encoded} <- Fact.RecordFile.Encoder.encode(context, genesis), + {:ok, record_id} <- Fact.RecordFile.Name.get(context, {genesis, encoded}), + {:ok, record_path} <- {:ok, Storage.records_path(context, record_id)} do + nonce = :crypto.strong_rand_bytes(12) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, encoded, "", true) + + encrypted = <> + + :ok = File.write!(record_path, encrypted) + File.chmod!(record_path, 0o444) + + {:ok, record_id} + end + end + + defp write_genesis_record(context, genesis, nil) do + Fact.RecordFile.write(context, genesis) + end + + defp build_bootstrap_opts(nil), do: [] + + defp build_bootstrap_opts(%{wrapped_dek_primary: primary, wrapped_dek_recovery: recovery}) do + [ + encryption: %{ + family: :aes_gcm, + version: 1, + wrapped_dek_primary: Base.encode64(primary), + wrapped_dek_recovery: Base.encode64(recovery) + } + ] + end end diff --git a/lib/fact/index_checkpoint_file.ex b/lib/fact/index_checkpoint_file.ex index 4562f7b..42b5abd 100644 --- a/lib/fact/index_checkpoint_file.ex +++ b/lib/fact/index_checkpoint_file.ex @@ -68,6 +68,7 @@ defmodule Fact.IndexCheckpointFile do """ use Fact.Seam.FileWriter.Adapter, context: :index_checkpoint_file_writer, + allowed_impls: [{:standard, 1}], fixed_options: %{ {:standard, 1} => %{ access: :write, diff --git a/lib/fact/index_file.ex b/lib/fact/index_file.ex index eb2c483..482c8e9 100644 --- a/lib/fact/index_file.ex +++ b/lib/fact/index_file.ex @@ -71,6 +71,7 @@ defmodule Fact.IndexFile do """ use Fact.Seam.FileWriter.Adapter, context: :index_file_writer, + allowed_impls: [{:standard, 1}], fixed_options: %{ {:standard, 1} => %{ access: :append, diff --git a/lib/fact/key_ring.ex b/lib/fact/key_ring.ex new file mode 100644 index 0000000..4bab4c2 --- /dev/null +++ b/lib/fact/key_ring.ex @@ -0,0 +1,111 @@ +defmodule Fact.KeyRing do + @moduledoc """ + Holds the Data Encryption Key (DEK) in memory for a database instance. + + The `Fact.KeyRing` is started as part of the `Fact.DatabaseSupervisor` when the database + is configured with encryption. At startup, it unwraps the DEK using the configured + `Fact.Encryption.KEKProvider` and holds it in memory for the lifetime of the database. + + The encrypted `Fact.Seam.FileWriter.Encrypted.V1` and `Fact.Seam.FileReader.Encrypted.V1` + retrieve the DEK via `get_dek/1`. + """ + @moduledoc since: "0.4.0" + + use GenServer + + @nonce_size 12 + @auth_tag_size 16 + + @typedoc since: "0.4.0" + @type option :: + {:database_id, Fact.database_id()} + | {:wrapped_dek, binary()} + | {:kek, binary()} + | {:name, GenServer.name()} + + @doc """ + Retrieves the DEK for the given database. + """ + @doc since: "0.4.0" + @spec get_dek(Fact.database_id()) :: {:ok, binary()} | {:error, term()} + def get_dek(database_id) do + GenServer.call(Fact.Registry.via(database_id, __MODULE__), :get_dek) + end + + @doc false + @spec child_spec([option()]) :: Supervisor.child_spec() + def child_spec(opts) do + database_id = Keyword.fetch!(opts, :database_id) + + %{ + id: {__MODULE__, database_id}, + start: {__MODULE__, :start_link, [opts]} + } + end + + @doc false + @spec start_link([option()]) :: GenServer.on_start() + def start_link(opts) do + {start_opts, init_opts} = Keyword.split(opts, [:name]) + GenServer.start_link(__MODULE__, init_opts, start_opts) + end + + @impl true + def init(opts) do + wrapped_dek = Keyword.fetch!(opts, :wrapped_dek) + kek = Keyword.fetch!(opts, :kek) + + case unwrap_dek(wrapped_dek, kek) do + {:ok, dek} -> + {:ok, %{dek: dek}} + + {:error, reason} -> + {:stop, reason} + end + end + + @impl true + def handle_call(:get_dek, _from, %{dek: dek} = state) do + {:reply, {:ok, dek}, state} + end + + @doc """ + Wraps a DEK with a KEK using AES-256-GCM. + + Returns the wrapped DEK as `<>`. + """ + @doc since: "0.4.0" + @spec wrap_dek(dek :: binary(), kek :: binary()) :: binary() + def wrap_dek(dek, kek) do + nonce = :crypto.strong_rand_bytes(@nonce_size) + {ciphertext, auth_tag} = :crypto.crypto_one_time_aead(:aes_256_gcm, kek, nonce, dek, "", true) + <> + end + + @doc """ + Unwraps a DEK using a KEK. + + Expects the wrapped DEK format: `<>`. + """ + @doc since: "0.4.0" + @spec unwrap_dek(wrapped_dek :: binary(), kek :: binary()) :: {:ok, binary()} | {:error, term()} + def unwrap_dek( + <>, + kek + ) do + case :crypto.crypto_one_time_aead(:aes_256_gcm, kek, nonce, ciphertext, "", auth_tag, false) do + dek when is_binary(dek) -> {:ok, dek} + :error -> {:error, :dek_unwrap_failed} + end + end + + def unwrap_dek(_wrapped_dek, _kek), do: {:error, :invalid_wrapped_dek} + + @doc """ + Generates a random 256-bit DEK. + """ + @doc since: "0.4.0" + @spec generate_dek() :: binary() + def generate_dek, do: :crypto.strong_rand_bytes(32) +end diff --git a/lib/fact/ledger_file.ex b/lib/fact/ledger_file.ex index 54c868a..8f31b7e 100644 --- a/lib/fact/ledger_file.ex +++ b/lib/fact/ledger_file.ex @@ -69,6 +69,7 @@ defmodule Fact.LedgerFile do """ use Fact.Seam.FileWriter.Adapter, context: :ledger_file_writer, + allowed_impls: [{:standard, 1}], fixed_options: %{ {:standard, 1} => %{ access: :append, diff --git a/lib/fact/lock_file.ex b/lib/fact/lock_file.ex index 71fc485..e8db44d 100644 --- a/lib/fact/lock_file.ex +++ b/lib/fact/lock_file.ex @@ -70,6 +70,7 @@ defmodule Fact.LockFile do """ use Fact.Seam.FileWriter.Adapter, context: :lock_file_writer, + allowed_impls: [{:standard, 1}], fixed_options: %{ {:standard, 1} => %{ access: :write, diff --git a/lib/fact/record_file.ex b/lib/fact/record_file.ex index 170405f..039657c 100644 --- a/lib/fact/record_file.ex +++ b/lib/fact/record_file.ex @@ -76,7 +76,8 @@ defmodule Fact.RecordFile do """ use Fact.Seam.FileReader.Adapter, context: :record_file_reader, - allowed_impls: [{:full, 1}] + allowed_impls: [{:encrypted, 1}, {:full, 1}], + default_impl: {:full, 1} end defmodule Writer do @@ -91,6 +92,8 @@ defmodule Fact.RecordFile do """ use Fact.Seam.FileWriter.Adapter, context: :record_file_writer, + allowed_impls: [{:encrypted, 1}, {:standard, 1}], + default_impl: {:standard, 1}, fixed_options: %{ {:standard, 1} => %{ access: :write, @@ -99,6 +102,14 @@ defmodule Fact.RecordFile do raw: true, sync: false, worm: true + }, + {:encrypted, 1} => %{ + access: :write, + binary: true, + exclusive: true, + raw: false, + sync: false, + worm: true } } end diff --git a/lib/fact/seam/file_reader/encrypted/v1.ex b/lib/fact/seam/file_reader/encrypted/v1.ex new file mode 100644 index 0000000..8611ba8 --- /dev/null +++ b/lib/fact/seam/file_reader/encrypted/v1.ex @@ -0,0 +1,89 @@ +defmodule Fact.Seam.FileReader.Encrypted.V1 do + @moduledoc """ + Encrypted V1 implementation of `Fact.Seam.FileReader`. + + Reads a file written by `Fact.Seam.FileWriter.Encrypted.V1`, parsing the header + to extract the nonce and authentication tag, then decrypting the ciphertext. + + The expected file format is: + + <> + + The Data Encryption Key (DEK) is retrieved from `Fact.KeyRing` at read time. + + ## Options + + * `:cipher` - The cipher algorithm. Currently only `:aes_gcm` is supported. + Defaults to `:aes_gcm`. + """ + @moduledoc since: "0.4.0" + + use Fact.Seam.FileReader, + family: :encrypted, + version: 1 + + import Fact.Seam.Parsers, only: [parse_existing_atom: 1] + + @nonce_size 12 + @auth_tag_size 16 + @header_size @nonce_size + @auth_tag_size + + @enforce_keys [:cipher] + defstruct [:cipher] + + @impl true + def default_options(), do: %{cipher: :aes_gcm} + + @impl true + def option_specs() do + %{ + cipher: %{ + allowed: [:aes_gcm], + parse: &parse_existing_atom/1, + error: :invalid_cipher_option + } + } + end + + @impl true + def prepare_options(options), do: Map.take(options, [:cipher]) + + @impl true + def read(%__MODULE__{cipher: :aes_gcm}, path, options) do + context = Keyword.get(options, :__context__) + database_id = context.database_id + + with {:ok, dek} <- Fact.KeyRing.get_dek(database_id), + {:ok, binary} <- File.read(path), + {:ok, plaintext} <- decrypt_binary(dek, binary) do + {:ok, Stream.map([plaintext], & &1)} + end + end + + @doc """ + Decrypts a binary that was written by `Fact.Seam.FileWriter.Encrypted.V1`. + + This is used by the bootstrapper to decrypt the genesis record before the + `Fact.KeyRing` process is started. + """ + @doc since: "0.4.0" + @spec decrypt_binary(dek :: binary(), encrypted :: binary()) :: + {:ok, binary()} | {:error, term()} + def decrypt_binary( + dek, + <> + ) do + case :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, ciphertext, "", auth_tag, false) do + plaintext when is_binary(plaintext) -> + {:ok, plaintext} + + :error -> + {:error, :decryption_failed} + end + end + + def decrypt_binary(_dek, binary) when byte_size(binary) < @header_size do + {:error, :invalid_encrypted_file} + end +end diff --git a/lib/fact/seam/file_reader/registry.ex b/lib/fact/seam/file_reader/registry.ex index 325739e..8ae70c0 100644 --- a/lib/fact/seam/file_reader/registry.ex +++ b/lib/fact/seam/file_reader/registry.ex @@ -6,6 +6,7 @@ defmodule Fact.Seam.FileReader.Registry do """ use Fact.Seam.Registry, impls: [ + Fact.Seam.FileReader.Encrypted.V1, Fact.Seam.FileReader.Full.V1, Fact.Seam.FileReader.FixedLength.V1 ] diff --git a/lib/fact/seam/file_writer/encrypted/v1.ex b/lib/fact/seam/file_writer/encrypted/v1.ex new file mode 100644 index 0000000..fe1c09c --- /dev/null +++ b/lib/fact/seam/file_writer/encrypted/v1.ex @@ -0,0 +1,137 @@ +defmodule Fact.Seam.FileWriter.Encrypted.V1 do + @moduledoc """ + Encrypted V1 implementation of `Fact.Seam.FileWriter`. + + Encrypts the value before writing it to disk. Each file is written with a header + containing a random nonce and authentication tag, followed by the ciphertext. + + The file format is: + + <> + + The Data Encryption Key (DEK) is retrieved from `Fact.KeyRing` at write time. + + ## Options + + * `:cipher` - The cipher algorithm. Currently only `:aes_gcm` is supported. + Defaults to `:aes_gcm`. + * `:access` - File access mode (`:write` or `:append`). Defaults to `:write`. + * `:binary` - Whether to open in binary mode. Defaults to `true`. + * `:exclusive` - Fail if the file already exists. Defaults to `true`. + * `:raw` - Use raw file descriptors. Defaults to `false`. + * `:sync` - Fsync after write. Defaults to `false`. + * `:worm` - Mark file read-only after write. Defaults to `false`. + """ + @moduledoc since: "0.4.0" + + use Fact.Seam.FileWriter, + family: :encrypted, + version: 1 + + import Fact.Seam.Parsers, only: [parse_existing_atom: 1] + + @nonce_size 12 + @auth_tag_size 16 + + @enforce_keys [:cipher, :modes, :sync, :worm] + defstruct [:cipher, :modes, :sync, :worm] + + @impl true + def default_options(), + do: %{ + cipher: :aes_gcm, + access: :write, + binary: true, + exclusive: true, + raw: false, + sync: false, + worm: false + } + + @impl true + def option_specs() do + %{ + cipher: %{ + allowed: [:aes_gcm], + parse: &parse_existing_atom/1, + error: :invalid_cipher_option + }, + access: %{ + allowed: [:write, :append], + parse: &parse_existing_atom/1, + error: :invalid_access_option + }, + binary: %{ + allowed: [true, false], + parse: &parse_existing_atom/1, + error: :invalid_binary_option + }, + exclusive: %{ + allowed: [true, false], + parse: &parse_existing_atom/1, + error: :invalid_exclusive_option + }, + raw: %{ + allowed: [true, false], + parse: &parse_existing_atom/1, + error: :invalid_raw_option + }, + sync: %{ + allowed: [true, false], + parse: &parse_existing_atom/1, + error: :invalid_sync_option + }, + worm: %{ + allowed: [true, false], + parse: &parse_existing_atom/1, + error: :invalid_worm_option + } + } + end + + @impl true + def prepare_options(options) do + modes = + [ + options.access, + if(options.binary, do: :binary, else: nil), + if(options.exclusive, do: :exclusive, else: nil), + if(options.raw, do: :raw, else: nil) + ] + |> Enum.reject(&is_nil/1) + + options + |> Map.take([:cipher, :sync, :worm]) + |> Map.put(:modes, modes) + end + + @impl true + def write( + %__MODULE__{cipher: :aes_gcm, modes: modes, sync: sync, worm: worm}, + path, + value, + options + ) do + context = Keyword.get(options, :__context__) + database_id = context.database_id + + with {:ok, dek} <- Fact.KeyRing.get_dek(database_id) do + nonce = :crypto.strong_rand_bytes(@nonce_size) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, value, "", true) + + encrypted_value = + <> + + with {:ok, fd} <- File.open(path, modes), + :ok <- IO.binwrite(fd, encrypted_value), + :ok <- if(sync, do: :file.sync(fd), else: :ok), + :ok <- File.close(fd), + :ok <- if(worm, do: File.chmod(path, 0o444), else: :ok) do + :ok + end + end + end +end diff --git a/lib/fact/seam/file_writer/registry.ex b/lib/fact/seam/file_writer/registry.ex index a5cac36..0a90416 100644 --- a/lib/fact/seam/file_writer/registry.ex +++ b/lib/fact/seam/file_writer/registry.ex @@ -5,5 +5,5 @@ defmodule Fact.Seam.FileWriter.Registry do Provides a lookup for allowed implementations, their versions, and the latest default implementation. """ use Fact.Seam.Registry, - impls: [Fact.Seam.FileWriter.Standard.V1] + impls: [Fact.Seam.FileWriter.Encrypted.V1, Fact.Seam.FileWriter.Standard.V1] end diff --git a/lib/fact/supervisor.ex b/lib/fact/supervisor.ex index 6d94f6c..d6ec15d 100644 --- a/lib/fact/supervisor.ex +++ b/lib/fact/supervisor.ex @@ -86,6 +86,30 @@ defmodule Fact.Supervisor do end end + @doc """ + Stops a running database. + + This terminates the `Fact.DatabaseSupervisor` for the given database and removes it from the + supervision tree. The database lock is released as part of the shutdown. + + Returns `:ok` on success, or `{:error, :not_found}` if no database with the given identifier + is running. + """ + @doc since: "0.3.1" + @spec stop_database(Fact.database_id()) :: :ok | {:error, :not_found} + def stop_database(database_id) when is_binary(database_id) do + child_id = {Fact.DatabaseSupervisor, database_id} + + case Supervisor.terminate_child(__MODULE__, child_id) do + :ok -> + Supervisor.delete_child(__MODULE__, child_id) + :ok + + {:error, :not_found} -> + {:error, :not_found} + end + end + @doc """ Starts the `Fact.Supervisor`. diff --git a/lib/mix/tasks/fact/create.ex b/lib/mix/tasks/fact/create.ex index d895dd1..0877aed 100644 --- a/lib/mix/tasks/fact/create.ex +++ b/lib/mix/tasks/fact/create.ex @@ -3,15 +3,15 @@ defmodule Mix.Tasks.Fact.Create do Creates a new database. ### Usage - + ### Options - + There are a lot of options to control how the database stores information. However many of these options currently only support a single value. Each represents a **seam** within the system, and each can be supplied with a set of implementation specific options. Each seam implementation is configured using `{family}@{version}` format, while options are specified using a comma-delimited list of key value pairs `{key1}={value1},{key2}={value2}`. - + Maybe this all just a big old YAGNI...but I think it will prove its worth in time. #### Event options @@ -22,30 +22,30 @@ defmodule Mix.Tasks.Fact.Create do * `--event-schema-options` - Configuration options for the selected event schema. (default: `""`) #### Index file options - - * `--index-file-decoder` - Controls how record are decoded after reading. (default: `raw@1`) + + * `--index-file-decoder` - Controls how record are decoded after reading. (default: `raw@1`) * `--index-file-decoder-options` - Configuration options for the selected record file decoder. (default: `""`) * `--index-file-encoder` - Controls how record are encoded for writing. (default: `delimited@1`) * `--index-file-encoder-options` - Configuration options for the selected record file encoder. (default: `""`) * `--index-file-name` - Controls how record files are named. (default: `raw@1`) - * `--index-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) + * `--index-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) * `--index-file-reader` - Controls how record files are read. (default: `fixed_length@1`) * `--index-file-reader-options` - Configuration options for the selected record file reader. (default: `""`) * `--index-file-writer` - Controls how record files are written. (default: `standard@1`) * `--index-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) - - #### Index checkpoint file options + + #### Index checkpoint file options * `--index-checkpoint-file-decoder` - Controls how record are decoded after reading. (default: `integer@1`) * `--index-checkpoint-file-decoder-options` - Configuration options for the selected record file decoder. (default: `""`) * `--index-checkpoint-file-encoder` - Controls how record are encoded for writing. (default: `integer@1`) * `--index-checkpoint-file-encoder-options` - Configuration options for the selected record file encoder. (default: `""`) * `--index-checkpoint-file-name` - Controls how record files are named. (default: `fixed@1`) - * `--index-checkpoint-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) + * `--index-checkpoint-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) * `--index-checkpoint-file-reader` - Controls how record files are read. (default: `full@1`) * `--index-checkpoint-file-reader-options` - Configuration options for the selected record file reader. (default: `""`) * `--index-checkpoint-file-writer` - Controls how record files are written. (default: `standard@1`) - * `--index-checkpoint-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) + * `--index-checkpoint-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) #### Ledger file options @@ -54,7 +54,7 @@ defmodule Mix.Tasks.Fact.Create do * `--ledger-file-encoder` - Controls how record are encoded for writing. (default: `delimited@1`) * `--ledger-file-encoder-options` - Configuration options for the selected record file encoder. (default: `""`) * `--ledger-file-name` - Controls how record files are named. (default: `fixed@1`) - * `--ledger-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) + * `--ledger-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) * `--ledger-file-reader` - Controls how record files are read. (default: `fixed_length@1`) * `--ledger-file-reader-options` - Configuration options for the selected record file reader. (default: `""`) * `--ledger-file-writer` - Controls how record files are written. (default: `standard@1`) @@ -67,30 +67,37 @@ defmodule Mix.Tasks.Fact.Create do * `--lock-file-encoder` - Controls how record are encoded for writing. (default: `json@1`) * `--lock-file-encoder-options` - Configuration options for the selected record file encoder. (default: `""`) * `--lock-file-name` - Controls how record files are named. (default: `fixed@1`) - * `--lock-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) + * `--lock-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) * `--lock-file-reader` - Controls how record files are read. (default: `full@1`) * `--lock-file-reader-options` - Configuration options for the selected record file reader. (default: `""`) * `--lock-file-writer` - Controls how record files are written. (default: `standard@1`) - * `--lock-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) + * `--lock-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) #### Record file options - + * `--record-file-decoder` - Controls how record are decoded after reading. (default: `json@1`) * `--record-file-decoder-options` - Configuration options for the selected record file decoder. (default: `""`) * `--record-file-encoder` - Controls how record are encoded for writing. (default: `json@1`) * `--record-file-encoder-options` - Configuration options for the selected record file encoder. (default: `""`) * `--record-file-name` - Controls how record files are named. (default: `event_id@1`) - * `--record-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) + * `--record-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) * `--record-file-reader` - Controls how record files are read. (default: `full@1`) * `--record-file-reader-options` - Configuration options for the selected record file reader. (default: `""`) * `--record-file-writer` - Controls how record files are written. (default: `standard@1`) * `--record-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) - + #### Storage options - + * `--storage` - Controls how the files are organized within the file system. (default: `standard@1`) * `--storage-options` - Configuration options for the selected storage. (default: `""`) + #### Encryption options + + * `--encrypted` - Enables encryption at rest for event record files. This is a shorthand that + sets the record file writer and reader to `encrypted@1` with `cipher=aes_gcm`. + * `--key` - Base64-encoded primary encryption key (16, 24, or 32 bytes decoded). Generated if not provided. + * `--recovery-key` - Base64-encoded recovery encryption key (16, 24, or 32 bytes decoded). Generated if not provided. + """ use Mix.Task @@ -160,6 +167,9 @@ defmodule Mix.Tasks.Fact.Create do record_file_writer_options: :string, storage: :string, storage_options: :string, + encrypted: :boolean, + key: :string, + recovery_key: :string, quiet: :boolean ] @@ -207,15 +217,17 @@ defmodule Mix.Tasks.Fact.Create do Mix.raise("Unexpected arguments: #{Enum.join(argv, " ")}") end + parsed = expand_encrypted_flag(parsed) command = %CreateDatabase.V1{args: parsed} state = Decider.initial_state() - with {:ok, [genesis_event]} <- Decider.decide(state, command) do - TheCreator.let_there_be_light(genesis_event) + with {:ok, [genesis_event], encryption_meta} <- Decider.decide(state, command) do + TheCreator.let_there_be_light(genesis_event, encryption_meta) unless Keyword.get(parsed, :quiet, false) do display_banner() display_results(genesis_event, Keyword.get(command.args, :path)) + display_encryption_keys(encryption_meta) display_next_steps() end @@ -228,6 +240,19 @@ defmodule Mix.Tasks.Fact.Create do end end + defp expand_encrypted_flag(parsed) do + if Keyword.get(parsed, :encrypted, false) do + parsed + |> Keyword.delete(:encrypted) + |> Keyword.put_new(:record_file_writer, "encrypted@1") + |> Keyword.put_new(:record_file_writer_options, "cipher=aes_gcm") + |> Keyword.put_new(:record_file_reader, "encrypted@1") + |> Keyword.put_new(:record_file_reader_options, "cipher=aes_gcm") + else + parsed + end + end + defp display_banner() do # ANSI Shadow Mix.shell().info(""" @@ -235,25 +260,25 @@ defmodule Mix.Tasks.Fact.Create do ███████╗ █████╗ ██████╗ ████████╗ ██╔════╝██╔══██╗██╔════╝ ╚══██╔══╝ - █████╗ ███████║██║ ██║ - ██╔══╝ ██╔══██║██║ ██║ - ██║ ██║ ██║╚██████╗ ██║ - ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ v#{Fact.BuildInfo.version()} (#{Fact.BuildInfo.codename()}) + █████╗ ███████║██║ ██║ + ██╔══╝ ██╔══██║██║ ██║ + ██║ ██║ ██║╚██████╗ ██║ + ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ v#{Fact.BuildInfo.version()} (#{Fact.BuildInfo.codename()}) """) end defp display_results(event, path) do Mix.shell().info(""" - + 🐢 "#{Enum.random(@quotes)}" - - ================================================================ + + ================================================================ ID: #{event.database_id} NAME: #{event.database_name} PATH: #{Path.absname(path)} ================================================================ - + Try it out... $ iex -S mix @@ -264,10 +289,33 @@ defmodule Mix.Tasks.Fact.Create do """) end + defp display_encryption_keys(nil), do: :ok + + defp display_encryption_keys(%{kek: kek, recovery_kek: recovery_kek}) do + Mix.shell().info(""" + + ================================================================ + 🔐 ENCRYPTION KEYS — SAVE THESE NOW + ================================================================ + + KEY: #{Base.encode64(kek)} + RECOVERY: #{Base.encode64(recovery_kek)} + + ================================================================ + + Store the key securely (secrets manager, environment variable). + Store the recovery key separately in a safe, offline location. + + These keys will NOT be shown again. If you lose both keys, + your data is unrecoverable. + ================================================================ + """) + end + defp display_next_steps() do Mix.shell().info(""" Next Steps: - + 📖 \e]8;;#{Fact.BuildInfo.docs_url()}\e\\Read the documentation\e]8;;\e\\ at #{Fact.BuildInfo.docs_url()} 🤓 \e]8;;https://leanpub.com/eventmodeling-and-eventsourcing\e\\Learn to understand event sourcing\e]8;;\e\\ 🫆 \e]8;;https://eventmodeling.org\e\\Design and deliver better systems with Event Modeling\e]8;;\e\\ diff --git a/lib/mix/tasks/fact/recover.ex b/lib/mix/tasks/fact/recover.ex new file mode 100644 index 0000000..0243f6a --- /dev/null +++ b/lib/mix/tasks/fact/recover.ex @@ -0,0 +1,196 @@ +defmodule Mix.Tasks.Fact.Recover do + @moduledoc """ + Recovers an encrypted database using the recovery key. + + This task performs a full key rotation: it decrypts all record files with the + old Data Encryption Key (DEK), generates a new DEK along with new primary and + recovery keys, re-encrypts every record, and updates the `.bootstrap` file. + + The database **must not be running** during recovery. + + ## Usage + + mix fact.recover --path data/my_db --recovery-key + + ## Options + + * `--path` / `-p` — (required) Path to the database directory. + * `--recovery-key` — (required) Base64-encoded recovery key. + """ + + use Mix.Task + + @shortdoc "Recovers an encrypted database with a recovery key" + + @switches [ + path: :string, + recovery_key: :string + ] + + @aliases [p: :path] + + @impl true + def run(args) do + {parsed, _argv, _invalid} = OptionParser.parse(args, strict: @switches, aliases: @aliases) + + path = fetch_required!(parsed, :path) + recovery_key_b64 = fetch_required!(parsed, :recovery_key) + + unless File.dir?(path) do + Mix.raise("Path does not exist or is not a directory: #{path}") + end + + with {:ok, recovery_key} <- decode_key(recovery_key_b64), + {:ok, bootstrap} <- Fact.BootstrapFile.read(path), + {:ok, encryption_config} <- get_encryption_config(bootstrap), + {:ok, old_dek} <- unwrap_with_recovery(encryption_config, recovery_key), + {:ok, context} <- load_context(bootstrap, old_dek) do + # Collect all record IDs from the ledger + record_ids = Fact.LedgerFile.read(context) |> Enum.to_list() + total = length(record_ids) + + Mix.shell().info("Recovering #{total} record(s)...\n") + + # Generate new keys + new_dek = Fact.KeyRing.generate_dek() + new_key = :crypto.strong_rand_bytes(32) + new_recovery_key = :crypto.strong_rand_bytes(32) + + # Re-encrypt each record + record_ids + |> Enum.with_index(1) + |> Enum.each(fn {record_id, index} -> + record_path = Fact.Storage.records_path(context, record_id) + + # Make writable (records are WORM) + File.chmod!(record_path, 0o644) + + # Decrypt with old DEK + {:ok, encrypted} = File.read(record_path) + {:ok, plaintext} = Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(old_dek, encrypted) + + # Re-encrypt with new DEK + nonce = :crypto.strong_rand_bytes(12) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, new_dek, nonce, plaintext, "", true) + + new_encrypted = <> + + File.write!(record_path, new_encrypted) + File.chmod!(record_path, 0o444) + + percent = div(index * 100, total) + Mix.shell().info("\r [#{percent}%] #{index}/#{total}") + end) + + # Update bootstrap with new wrapped DEKs + new_wrapped_primary = Fact.KeyRing.wrap_dek(new_dek, new_key) + new_wrapped_recovery = Fact.KeyRing.wrap_dek(new_dek, new_recovery_key) + + updated_bootstrap = %{ + bootstrap + | "encryption" => %{ + bootstrap["encryption"] + | "wrapped_dek_primary" => Base.encode64(new_wrapped_primary), + "wrapped_dek_recovery" => Base.encode64(new_wrapped_recovery) + } + } + + write_bootstrap(path, updated_bootstrap) + + Mix.shell().info(""" + + ================================================================ + Recovery complete. #{total} record(s) re-encrypted. + ================================================================ + 🔐 NEW ENCRYPTION KEYS — SAVE THESE NOW + ================================================================ + + KEY: #{Base.encode64(new_key)} + RECOVERY: #{Base.encode64(new_recovery_key)} + + ================================================================ + + The old keys are no longer valid. + Store these keys securely — they will NOT be shown again. + ================================================================ + """) + else + {:error, reason} -> + Mix.raise("Recovery failed: #{inspect(reason)}") + end + end + + defp fetch_required!(opts, key) do + case Keyword.get(opts, key) do + nil -> + Mix.raise("Missing required option: --#{key |> to_string() |> String.replace("_", "-")}") + + value -> + value + end + end + + defp decode_key(b64) do + case Base.decode64(b64) do + {:ok, key} when byte_size(key) in [16, 24, 32] -> {:ok, key} + {:ok, key} -> {:error, {:invalid_key_size, byte_size(key)}} + :error -> {:error, :invalid_base64} + end + end + + defp get_encryption_config(%{"encryption" => config}) when is_map(config), do: {:ok, config} + defp get_encryption_config(_), do: {:error, :database_not_encrypted} + + defp unwrap_with_recovery(encryption_config, recovery_key) do + wrapped = Base.decode64!(encryption_config["wrapped_dek_recovery"]) + Fact.KeyRing.unwrap_dek(wrapped, recovery_key) + end + + defp load_context(bootstrap, dek) do + bootstrap_context = %Fact.Context{ + event_schema: Fact.Event.Schema.from_config(bootstrap["event_schema"]), + record_file_decoder: Fact.RecordFile.Decoder.from_config(bootstrap["record_file_decoder"]), + record_file_reader: + Fact.RecordFile.Reader.from_config( + bootstrap["record_file_reader"] || + %{"family" => "full", "version" => 1, "options" => %{}} + ), + storage: Fact.Storage.from_config(bootstrap["storage"]) + } + + # Read the genesis record to build the full context + record_id = bootstrap["record_id"] + {:ok, record_path} = {:ok, Fact.Storage.records_path(bootstrap_context, record_id)} + + {:ok, stream} = + Fact.Seam.FileReader.Full.V1.read(%Fact.Seam.FileReader.Full.V1{}, record_path, []) + + encrypted_binary = Enum.at(stream, 0) + {:ok, plaintext} = Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(dek, encrypted_binary) + {:ok, event} = Fact.RecordFile.Decoder.decode(bootstrap_context, plaintext) + + event_data = Map.get(event, Fact.Event.Schema.get(bootstrap_context).event_data) + {:ok, Fact.Context.from_genesis_event_data(event_data)} + end + + defp write_bootstrap(path, bootstrap_data) do + bootstrap_context = %Fact.BootstrapFile.Context{ + encoder: Fact.BootstrapFile.Encoder.from_config(%{family: :json, version: 1, options: %{}}), + name: Fact.BootstrapFile.Name.from_config(%{family: :fixed, version: 1, options: %{}}), + writer: + Fact.BootstrapFile.Writer.from_config(%{family: :standard, version: 1, options: %{}}) + } + + bootstrap_path = Path.join(path, Fact.BootstrapFile.Name.get(bootstrap_context)) + + # Bootstrap is WORM, make writable before overwriting + File.chmod!(bootstrap_path, 0o644) + + {:ok, encoded} = Fact.BootstrapFile.Encoder.encode(bootstrap_context, bootstrap_data) + Fact.BootstrapFile.Writer.write(bootstrap_context, bootstrap_path, encoded) + + File.chmod!(bootstrap_path, 0o444) + end +end diff --git a/mix.exs b/mix.exs index 5bfe7ef..cd1c504 100644 --- a/mix.exs +++ b/mix.exs @@ -70,6 +70,7 @@ defmodule Fact.MixProject do extra_section: "GUIDES", extras: [ "LICENSE", + "guides/components/encryption.md", "guides/components/merkle-mountain-range.md", "guides/components/queries-and-indexes.md", "guides/components/record-cache.md", @@ -87,6 +88,7 @@ defmodule Fact.MixProject do "guides/introduction/process-model.md" ], Components: [ + "guides/components/encryption.md", "guides/components/merkle-mountain-range.md", "guides/components/queries-and-indexes.md", "guides/components/record-cache.md", diff --git a/test/fact/encryption_test.exs b/test/fact/encryption_test.exs new file mode 100644 index 0000000..082ff22 --- /dev/null +++ b/test/fact/encryption_test.exs @@ -0,0 +1,294 @@ +defmodule Fact.EncryptionTest do + @moduledoc false + + use ExUnit.Case + + @moduletag :capture_log + + describe "KEKProvider.Explicit" do + alias Fact.Encryption.KEKProvider.Explicit + + test "returns KEK when valid 32-byte key provided" do + kek = :crypto.strong_rand_bytes(32) + assert {:ok, ^kek} = Explicit.fetch_kek(kek: kek) + end + + test "accepts 16-byte key" do + kek = :crypto.strong_rand_bytes(16) + assert {:ok, ^kek} = Explicit.fetch_kek(kek: kek) + end + + test "accepts 24-byte key" do + kek = :crypto.strong_rand_bytes(24) + assert {:ok, ^kek} = Explicit.fetch_kek(kek: kek) + end + + test "returns error for invalid key size" do + assert {:error, {:invalid_kek_size, 10}} = + Explicit.fetch_kek(kek: :crypto.strong_rand_bytes(10)) + end + + test "returns error when kek not provided" do + assert {:error, :kek_not_provided} = Explicit.fetch_kek([]) + end + + test "returns error when kek is not binary" do + assert {:error, :kek_must_be_binary} = Explicit.fetch_kek(kek: 12345) + end + end + + describe "KeyRing" do + test "wrap and unwrap DEK round-trips" do + dek = Fact.KeyRing.generate_dek() + kek = :crypto.strong_rand_bytes(32) + + wrapped = Fact.KeyRing.wrap_dek(dek, kek) + assert {:ok, ^dek} = Fact.KeyRing.unwrap_dek(wrapped, kek) + end + + test "unwrap fails with wrong KEK" do + dek = Fact.KeyRing.generate_dek() + kek = :crypto.strong_rand_bytes(32) + wrong_kek = :crypto.strong_rand_bytes(32) + + wrapped = Fact.KeyRing.wrap_dek(dek, kek) + assert {:error, :dek_unwrap_failed} = Fact.KeyRing.unwrap_dek(wrapped, wrong_kek) + end + + test "unwrap fails with invalid wrapped DEK" do + kek = :crypto.strong_rand_bytes(32) + assert {:error, :invalid_wrapped_dek} = Fact.KeyRing.unwrap_dek(<<1, 2, 3>>, kek) + end + + test "generate_dek returns 32 bytes" do + dek = Fact.KeyRing.generate_dek() + assert byte_size(dek) == 32 + end + end + + describe "Encrypted FileWriter and FileReader round-trip" do + setup do + dir = Path.join("tmp", "encryption_rw_test_#{:rand.uniform(1_000_000)}") + File.mkdir_p!(dir) + on_exit(fn -> File.rm_rf(dir) end) + + {:ok, dir: dir} + end + + test "encrypts and decrypts a value", %{dir: _dir} do + dek = Fact.KeyRing.generate_dek() + plaintext = ~s({"event_type":"turtle_hatched","event_data":{"name":"Shelldon"}}) + + nonce = :crypto.strong_rand_bytes(12) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, plaintext, "", true) + + encrypted = <> + + {:ok, decrypted} = Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(dek, encrypted) + assert decrypted == plaintext + end + + test "decrypt_binary fails with wrong key", %{dir: _dir} do + dek = Fact.KeyRing.generate_dek() + wrong_dek = Fact.KeyRing.generate_dek() + plaintext = "secret data" + + nonce = :crypto.strong_rand_bytes(12) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, plaintext, "", true) + + encrypted = <> + + assert {:error, :decryption_failed} = + Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(wrong_dek, encrypted) + end + + test "decrypt_binary fails with truncated file" do + dek = Fact.KeyRing.generate_dek() + + assert {:error, :invalid_encrypted_file} = + Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(dek, <<1, 2, 3>>) + end + end + + describe "end-to-end encrypted database" do + setup do + path = + :uuid.get_v4() + |> :uuid.uuid_to_string(:nodash) + |> to_string() + |> then(fn uuid -> "e2e_enc_" <> uuid end) + |> String.downcase() + |> String.split_at(63) + |> then(fn {name, _} -> Path.join("tmp", name) end) + + # Create encrypted database — keys are auto-generated + args = [ + path: path, + record_file_writer: "encrypted@1", + record_file_writer_options: "cipher=aes_gcm", + record_file_reader: "encrypted@1", + record_file_reader_options: "cipher=aes_gcm" + ] + + command = %Fact.Genesis.Command.CreateDatabase.V1{args: args} + + {:ok, [genesis_event], encryption_meta} = + Fact.Genesis.Decider.decide(Fact.Genesis.Decider.initial_state(), command) + + Fact.Genesis.TheCreator.let_there_be_light(genesis_event, encryption_meta) + + on_exit(fn -> TestHelper.rm_rf(path) end) + + {:ok, path: path, encryption_meta: encryption_meta} + end + + test "keys are auto-generated when not provided", %{encryption_meta: meta} do + assert byte_size(meta.kek) == 32 + assert byte_size(meta.recovery_kek) == 32 + assert byte_size(meta.dek) == 32 + assert meta.kek != meta.recovery_kek + end + + test "genesis record file is not plaintext on disk", %{path: path} do + record_file = + Path.join(path, "events/**/*") + |> Path.wildcard() + |> Enum.reject(&File.dir?/1) + |> hd() + + {:ok, bytes} = File.read(record_file) + + refute String.starts_with?(bytes, "{") + assert byte_size(bytes) > 28 + end + + test "bootstrap file contains encryption metadata", %{path: path} do + {:ok, bootstrap} = Fact.BootstrapFile.read(path) + + assert bootstrap["encryption"] + assert bootstrap["encryption"]["wrapped_dek_primary"] + assert bootstrap["encryption"]["wrapped_dek_recovery"] + assert bootstrap["encryption"]["family"] == "aes_gcm" + end + + test "open and read with encryption_key", %{path: path, encryption_meta: meta} do + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + + events = Fact.read(db, :all) + assert length(events) == 1 + assert hd(events)["event_type"] == "Elixir.Fact.Genesis.Event.DatabaseCreated.V1" + end + + test "append and read encrypted events", %{path: path, encryption_meta: meta} do + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + + event = %{type: "turtle_hatched", data: %{name: "Shelldon"}} + {:ok, position} = Fact.append(db, event) + + TestHelper.subscribe_and_wait(db, position) + + [record] = Fact.read(db, :all, position: position - 1, count: 1) + assert record["event_type"] == "turtle_hatched" + assert record["event_data"]["name"] == "Shelldon" + end + + test "wrong encryption key returns clear error", %{path: path} do + wrong_key = :crypto.strong_rand_bytes(32) |> Base.encode64() + assert {:error, :invalid_encryption_key} = Fact.open(path, encryption_key: wrong_key) + end + + test "missing encryption key returns clear error", %{path: path} do + assert {:error, :encryption_key_required} = Fact.open(path) + end + + test "mix fact.recover rotates keys and re-encrypts records", %{ + path: path, + encryption_meta: meta + } do + # Open, append an event, then close the database + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + {:ok, position} = Fact.append(db, %{type: "turtle_hatched", data: %{name: "Shelldon"}}) + TestHelper.subscribe_and_wait(db, position) + # subscribe_and_wait links us to PubSub; trap exits during close + Process.flag(:trap_exit, true) + :ok = Fact.close(db) + Process.flag(:trap_exit, false) + + # Run recovery with the recovery key + Mix.Tasks.Fact.Recover.run([ + "--path", + path, + "--recovery-key", + Base.encode64(meta.recovery_kek) + ]) + + # Old key should no longer work — bootstrap has new wrapped DEKs + {:ok, bootstrap} = Fact.BootstrapFile.read(path) + old_wrapped = Base.decode64!(bootstrap["encryption"]["wrapped_dek_primary"]) + assert {:error, :dek_unwrap_failed} = Fact.KeyRing.unwrap_dek(old_wrapped, meta.kek) + end + + test "--encrypted flag via mix task creates encrypted database" do + path = + :uuid.get_v4() + |> :uuid.uuid_to_string(:nodash) + |> to_string() + |> then(fn uuid -> "e2e_enc_flag_" <> uuid end) + |> String.downcase() + |> String.split_at(63) + |> then(fn {name, _} -> Path.join("tmp", name) end) + + Mix.Tasks.Fact.Create.run(["--path", path, "--encrypted"]) + + {:ok, bootstrap} = Fact.BootstrapFile.read(path) + assert bootstrap["encryption"] + assert bootstrap["record_file_writer"]["family"] == "encrypted" + + TestHelper.rm_rf(path) + end + + test "user-provided keys are used when specified" do + path = + :uuid.get_v4() + |> :uuid.uuid_to_string(:nodash) + |> to_string() + |> then(fn uuid -> "e2e_enc_custom_" <> uuid end) + |> String.downcase() + |> String.split_at(63) + |> then(fn {name, _} -> Path.join("tmp", name) end) + + my_key = :crypto.strong_rand_bytes(32) + my_recovery = :crypto.strong_rand_bytes(32) + + args = [ + path: path, + record_file_writer: "encrypted@1", + record_file_writer_options: "cipher=aes_gcm", + record_file_reader: "encrypted@1", + record_file_reader_options: "cipher=aes_gcm", + key: Base.encode64(my_key), + recovery_key: Base.encode64(my_recovery) + ] + + command = %Fact.Genesis.Command.CreateDatabase.V1{args: args} + + {:ok, [genesis_event], encryption_meta} = + Fact.Genesis.Decider.decide(Fact.Genesis.Decider.initial_state(), command) + + assert encryption_meta.kek == my_key + assert encryption_meta.recovery_kek == my_recovery + + Fact.Genesis.TheCreator.let_there_be_light(genesis_event, encryption_meta) + + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(my_key)) + events = Fact.read(db, :all) + assert length(events) == 1 + + TestHelper.rm_rf(path) + end + end +end From ed34d41dcfa923bef59a0504eeb3a015a7e5eb83 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Wed, 1 Apr 2026 23:59:21 -0700 Subject: [PATCH 03/17] fix: prevent DEK exposure in crash dumps, :sys.get_state, observer, and trace outputs. --- lib/fact/key_ring.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/fact/key_ring.ex b/lib/fact/key_ring.ex index 4bab4c2..fb95c07 100644 --- a/lib/fact/key_ring.ex +++ b/lib/fact/key_ring.ex @@ -55,6 +55,10 @@ defmodule Fact.KeyRing do wrapped_dek = Keyword.fetch!(opts, :wrapped_dek) kek = Keyword.fetch!(opts, :kek) + # Prevent the DEK from appearing in crash dumps, :sys.get_state, + # observer, and trace output. + Process.flag(:sensitive, true) + case unwrap_dek(wrapped_dek, kek) do {:ok, dek} -> {:ok, %{dek: dek}} From 772543334383c33eb4f56bcccc77ce19aca62df2 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Thu, 2 Apr 2026 00:02:53 -0700 Subject: [PATCH 04/17] fix: add decryption for verification --- lib/fact/merkle_mountain_range.ex | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/fact/merkle_mountain_range.ex b/lib/fact/merkle_mountain_range.ex index 248dbbf..b14e5e4 100644 --- a/lib/fact/merkle_mountain_range.ex +++ b/lib/fact/merkle_mountain_range.ex @@ -456,12 +456,13 @@ defmodule Fact.MerkleMountainRange do defp do_verify(state) do {:ok, context} = Fact.Registry.get_context(state.database_id) prev_leaf_hash = :binary.copy(<<0>>, state.hash_size) + dek = resolve_dek(context) tampered = Fact.LedgerFile.read(state.database_id) |> Enum.with_index() |> Enum.reduce({[], prev_leaf_hash}, fn {record_id, index}, {tampered, prev_hash} -> - case verify_record(context, state, record_id, index, prev_hash) do + case verify_record(context, state, record_id, index, prev_hash, dek) do {:ok, _store_position, leaf_hash} -> {tampered, leaf_hash} @@ -478,8 +479,15 @@ defmodule Fact.MerkleMountainRange do end end - defp verify_record(context, state, record_id, index, prev_hash) do - content_ok? = verify_content_hash(context, record_id) + defp resolve_dek(%{record_file_reader: %{module: mod}} = context) do + if mod.family() == :encrypted do + {:ok, dek} = Fact.KeyRing.get_dek(context.database_id) + dek + end + end + + defp verify_record(context, state, record_id, index, prev_hash, dek) do + content_ok? = verify_content_hash(context, record_id, dek) # Read event and extract position; if the file was corrupted beyond decoding, # treat it as tampered but continue verification with a poison hash. @@ -504,17 +512,23 @@ defmodule Fact.MerkleMountainRange do end end - defp verify_content_hash(context, record_id) do + defp verify_content_hash(context, record_id, dek) do record_path = Fact.Storage.records_path(context, record_id) with {:ok, raw_bytes} <- File.read(record_path), - {:ok, expected_id} <- Fact.RecordFile.Name.get(context, raw_bytes, []) do + {:ok, content} <- decrypt_if_needed(dek, raw_bytes), + {:ok, expected_id} <- Fact.RecordFile.Name.get(context, content, []) do expected_id == record_id else _ -> false end end + defp decrypt_if_needed(nil, raw_bytes), do: {:ok, raw_bytes} + + defp decrypt_if_needed(dek, raw_bytes), + do: Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(dek, raw_bytes) + # ----------------------------- # MMR Position Arithmetic # ----------------------------- From 7f3bb9b7a447dfdb2dd2698f215d538bf51a7aa1 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Thu, 2 Apr 2026 00:10:35 -0700 Subject: [PATCH 05/17] chore: update doc attributes --- lib/fact.ex | 2 +- lib/fact/bootstrap_file.ex | 2 +- lib/fact/genesis/the_creator.ex | 2 +- lib/fact/supervisor.ex | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/fact.ex b/lib/fact.ex index 6fa7c60..6aab53e 100644 --- a/lib/fact.ex +++ b/lib/fact.ex @@ -641,7 +641,7 @@ defmodule Fact do :ok """ - @doc since: "0.3.1" + @doc since: "0.4.0" @spec close(database_id()) :: :ok | {:error, :not_found} def close(database_id) when is_binary(database_id) do Fact.Supervisor.stop_database(database_id) diff --git a/lib/fact/bootstrap_file.ex b/lib/fact/bootstrap_file.ex index 8f568f3..08981da 100644 --- a/lib/fact/bootstrap_file.ex +++ b/lib/fact/bootstrap_file.ex @@ -186,7 +186,7 @@ defmodule Fact.BootstrapFile do It writes just enough configuration, so that the genesis record (`Fact.Genesis.Event.DatabaseCreated.V1`) can be read and a full `Fact.Context` can be loaded. """ - @doc since: "0.3.1" + @doc since: "0.4.0" @spec write(Path.t(), genesis_record :: Fact.record(), keyword()) :: :ok | {:error, term()} def write(path, {record_id, event}, opts \\ []) do this = get_bootstrap_write_context() diff --git a/lib/fact/genesis/the_creator.ex b/lib/fact/genesis/the_creator.ex index 17b7600..791234c 100644 --- a/lib/fact/genesis/the_creator.ex +++ b/lib/fact/genesis/the_creator.ex @@ -26,7 +26,7 @@ defmodule Fact.Genesis.TheCreator do (since the KeyRing process is not yet running) and the bootstrap file includes the wrapped DEK for later decryption. """ - @doc since: "0.3.1" + @doc since: "0.4.0" @spec let_there_be_light(%DatabaseCreated.V1{}, map() | nil) :: :ok def let_there_be_light(%DatabaseCreated.V1{} = event, encryption_meta \\ nil) do with context <- DatabaseCreated.V1.to_context(event), diff --git a/lib/fact/supervisor.ex b/lib/fact/supervisor.ex index d6ec15d..0bb47da 100644 --- a/lib/fact/supervisor.ex +++ b/lib/fact/supervisor.ex @@ -95,7 +95,7 @@ defmodule Fact.Supervisor do Returns `:ok` on success, or `{:error, :not_found}` if no database with the given identifier is running. """ - @doc since: "0.3.1" + @doc since: "0.4.0" @spec stop_database(Fact.database_id()) :: :ok | {:error, :not_found} def stop_database(database_id) when is_binary(database_id) do child_id = {Fact.DatabaseSupervisor, database_id} From 95eccc389e01a5a7d98426b77d73dea1151a584f Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Wed, 1 Apr 2026 18:32:55 -0700 Subject: [PATCH 06/17] feat: Adds support for custom indexers --- guides/components/indexers.md | 334 +++++++++++++++++++++++++++++ lib/fact.ex | 263 ++++++++++++----------- lib/fact/database_supervisor.ex | 79 ++++++- lib/fact/event_indexer.ex | 46 +++- mix.exs | 2 + test/fact/custom_indexers_test.exs | 70 ++++++ 6 files changed, 654 insertions(+), 140 deletions(-) create mode 100644 guides/components/indexers.md create mode 100644 test/fact/custom_indexers_test.exs diff --git a/guides/components/indexers.md b/guides/components/indexers.md new file mode 100644 index 0000000..383651a --- /dev/null +++ b/guides/components/indexers.md @@ -0,0 +1,334 @@ +# Indexers + +This guide explains how **event indexers** work in Fact, describes each of the built-in indexers, and walks through +building and configuring **custom indexers** for your application. + +At a high level: + +- Indexers are **event projections** that filter the event ledger and produce keyed, ordered sets of event record IDs +- Each indexer runs as a **supervised GenServer** that subscribes to all events and maintains its own index files +- Indexes are **append-only** and **safe to delete** — they rebuild automatically on next startup +- Custom indexers implement a single callback and are configured via `Fact.open/2` + +## How Indexers Work + +Every indexer follows the same lifecycle: + +1. **Startup** — the indexer subscribes to all events via `Fact.EventPublisher` and reads its checkpoint (the last + store position it processed) +2. **Rebuild** — any events after the checkpoint are replayed from the ledger, and the index is brought up to date +3. **Live** — as new events are appended, the indexer receives `{:appended, record}` messages and updates its index + in real time + +For each event, the indexer calls its `index_event/3` callback. The callback returns: + +- `nil` — the event is skipped +- A string value — a single index entry is written +- A list of string values — an entry is written to each corresponding index file + +Index files are stored under `/indices///`. Each distinct value returned by the callback +produces its own file containing the event record IDs that matched, ordered by store position. + +After processing each event, the indexer writes its checkpoint so it can resume from the correct position after a +restart. + +## Built-in Indexers + +Fact starts six indexers automatically with every database. These power the core read APIs and query system. + +### Event Stream Indexer + +**Module:** `Fact.EventStreamIndexer` + +Indexes every event by its stream ID. Creates one index file per stream. + +``` +/indices/event_stream/turtle_nest-1234 +/indices/event_stream/__fact +``` + +### Event Type Indexer + +**Module:** `Fact.EventTypeIndexer` + +Indexes every event by its type. Creates one index file per event type. + +``` +/indices/event_type/ClutchLaid +/indices/event_type/EggHatched +``` + +### Event Tags Indexer + +**Module:** `Fact.EventTagsIndexer` + +Indexes events by each of their tags. A single event with multiple tags produces entries in multiple index files. + +``` +/indices/event_tags/turtle:t1 +/indices/event_tags/clutch:c1 +``` + +### Event Stream Category Indexer + +**Module:** `Fact.EventStreamCategoryIndexer` + +Indexes events by the **category** portion of the stream ID — the first segment when split on the separator +(default: `"-"`). For example, stream `turtle_nest-1234` produces category `turtle_nest`. + +``` +/indices/event_stream_category/turtle_nest +/indices/event_stream_category/__fact +``` + +### Event Streams Indexer + +**Module:** `Fact.EventStreamsIndexer` + +Indexes only the **first event** of each stream. Produces a single index file called `index` containing the record +IDs for all stream-opening events. Useful for discovering all aggregate instances in the system. + +``` +/indices/event_streams/index +``` + +### Event Streams by Category Indexer + +**Module:** `Fact.EventStreamsByCategoryIndexer` + +Indexes the **first event** of each stream, grouped by stream category. Produces one index file per category, each +containing the first event of every stream in that category. Useful for queries like "get all orders" or "get all +customers." + +``` +/indices/event_streams_by_category/turtle_nest +/indices/event_streams_by_category/order +``` + +### Event Data Indexer + +**Module:** `Fact.EventDataIndexer` + +A parameterized indexer that extracts the value of a specific field from the event data payload. Each instance +targets a single key and is identified by that key. + +``` +/indices/event_data/user_id/user_123 +/indices/event_data/user_id/user_456 +/indices/event_data/tenant_id/acme +``` + +## Custom Indexers + +Custom indexers allow you to define application-specific indexes that are maintained automatically as events are +appended. A custom indexer is a module that uses `Fact.EventIndexer` and implements the `index_event/3` callback. + +### Defining a Custom Indexer + +```elixir +defmodule MyApp.UserIndexer do + use Fact.EventIndexer + + @impl true + def index_event(schema, event, _opts) do + unless is_nil(user_id = Map.get(event[schema.event_data], "user_id")), + do: to_string(user_id) + end +end +``` + +The `schema` argument provides the field names for the configured event schema (e.g., `schema.event_data`, +`schema.event_type`, `schema.event_stream_id`). This ensures your indexer works regardless of which event schema +the database is configured with. + +The callback must return `nil`, a string, or a list of strings. + +### Returning Multiple Values + +An indexer can create entries in multiple index files for a single event by returning a list. This is how the +built-in `Fact.EventTagsIndexer` works — it returns the full list of tags, and each tag gets its own index entry. + +```elixir +defmodule MyApp.MentionIndexer do + use Fact.EventIndexer + + @impl true + def index_event(schema, event, _opts) do + case Map.get(event[schema.event_data], "mentions") do + nil -> nil + mentions -> Enum.map(mentions, &to_string/1) + end + end +end +``` + +### Using Options + +The third argument to `index_event/3` is a keyword list of options passed through from the configuration. This +enables a single module to serve multiple purposes. + +```elixir +defmodule MyApp.MetadataIndexer do + use Fact.EventIndexer + + @impl true + def index_event(schema, event, opts) do + field = Keyword.fetch!(opts, :indexer_key) + + unless is_nil(value = Map.get(event[schema.event_metadata], field)), + do: to_string(value) + end +end +``` + +### Using Keys for Parameterized Instances + +A `:key` distinguishes multiple instances of the same indexer module. Each instance gets its own checkpoint file and +index directory, so they operate independently. + +```elixir +Fact.open("data/my_app", + indexers: [ + {MyApp.MetadataIndexer, key: "correlation_id", options: [indexer_key: "correlation_id"]}, + {MyApp.MetadataIndexer, key: "causation_id", options: [indexer_key: "causation_id"]} + ] +) +``` + +This produces: + +``` +/indices/metadata/correlation_id/ +/indices/metadata/causation_id/ +``` + +## Configuration + +Custom indexers are passed to `Fact.open/2` via the `:indexers` option. Each entry can be a bare module or a +`{module, opts}` tuple. + +```elixir +{:ok, db} = Fact.open("data/my_app", + indexers: [ + MyApp.UserIndexer, + MyApp.MentionIndexer, + {Fact.EventDataIndexer, key: "tenant_id"}, + {Fact.EventDataIndexer, key: "order_id"} + ] +) +``` + +The same configuration works when starting Fact under a supervision tree: + +```elixir +children = [ + {Fact.Supervisor, + databases: [ + {"data/my_app", + indexers: [ + MyApp.UserIndexer, + {Fact.EventDataIndexer, key: "tenant_id"} + ]} + ]} +] +``` + +Or via application configuration: + +```elixir +# config/config.exs +config :my_app, Fact, + databases: [ + {"data/my_app", + indexers: [ + MyApp.UserIndexer, + {Fact.EventDataIndexer, key: "tenant_id"} + ]} + ] + +# application.ex +children = [ + {Fact.Supervisor, Application.get_env(:my_app, Fact)} +] +``` + +Custom indexers are **additive** — they run alongside the six built-in indexers, which are always started. + +## Reading from Custom Indexes + +Once a custom indexer is running, you can read from its index using `Fact.read/3` with the `:index` source: + +```elixir +# Read all events for a specific user +Fact.read(db, {:index, {MyApp.UserIndexer, nil}, "user_123"}) + +# Read all events for a tenant (via EventDataIndexer) +Fact.read(db, {:index, {Fact.EventDataIndexer, "tenant_id"}, "acme"}) +``` + +Custom indexes also work with subscriptions: + +```elixir +# Subscribe to a custom index +Fact.subscribe(db, {:index, {MyApp.UserIndexer, nil}, "user_123"}) +``` + +In practice, it is common to wrap index reads in a helper function that hides the indexer tuple and forwards +read options to `Fact.read/3`: + +```elixir +defmodule MyApp.Users do + @indexer_id {MyApp.UserIndexer, nil} + + def read(db, user_id, read_opts \\ []) do + Fact.read(db, {:index, @indexer_id, user_id}, read_opts) + end +end +``` + +```elixir +# Get the last 10 events for a user, most recent first +MyApp.Users.read(db, "user_123", direction: :backward, position: :end, count: 10) + +# Get all events for a user as record tuples +MyApp.Users.read(db, "user_123", result: :record) +``` + +## Index File Layout + +All indexers write to the same directory structure under the database's indices path: + +``` +indices/ +├── event_stream/ # EventStreamIndexer +│ └── +├── event_type/ # EventTypeIndexer +│ └── +├── event_tags/ # EventTagsIndexer +│ └── +├── event_stream_category/ # EventStreamCategoryIndexer +│ └── +├── event_streams/ # EventStreamsIndexer +│ └── index +├── event_streams_by_category/ # EventStreamsByCategoryIndexer +│ └── +├── event_data/ # EventDataIndexer +│ └── / +│ └── +└── user/ # MyApp.UserIndexer (custom) + └── +``` + +The directory name for a custom indexer is derived from its module name. `MyApp.UserIndexer` becomes `user`, +`MyApp.OrderFulfillmentIndexer` becomes `order_fulfillment`. This can be overridden by passing `:name` to the +`use Fact.EventIndexer` macro: + +```elixir +defmodule MyApp.UserIndexer do + use Fact.EventIndexer, name: "app_user" + + # ... +end +``` + +Each indexer directory also contains a `.checkpoint` file that tracks the last processed store position. diff --git a/lib/fact.ex b/lib/fact.ex index 25da3c0..4441dd2 100644 --- a/lib/fact.ex +++ b/lib/fact.ex @@ -12,11 +12,11 @@ defmodule Fact do Fact is built around a few intentional concepts: - * **Events are facts** - they describe something that happened in the domain - * **The ledger is append-only** - state is derived, never mutated in place - * **Streams define static consistency boundaries** - typically aligned with DDD aggregates + * **Events are facts** - they describe something that happened in the domain + * **The ledger is append-only** - state is derived, never mutated in place + * **Streams define static consistency boundaries** - typically aligned with DDD aggregates * **Queries define dynamic consistency boundaries** - enabling emergent boundaries based - on event types, tags, and data-level conditions + on event types, tags, and data-level conditions * **Storage is transparent** - files on disk, deterministic layouts, no black boxes; easy to inspect and manipulate (but never change) with standard OS tooling such as `grep`, `sed`, `awk`, `jq` and plenty of other command-line utilities. @@ -41,10 +41,10 @@ defmodule Fact do Fact provides two complementary consistency mechanisms: - * **Stream expectations** - optimistic concurrency within a single stream - * **Append conditions** - optional duplicate / conflict detection at the ledger level + * **Stream expectations** - optimistic concurrency within a single stream + * **Append conditions** - optional duplicate / conflict detection at the ledger level - These tools allow you to model invariants where they belong **in the domain**, + These tools allow you to model invariants where they belong **in the domain**, rather than inside storage mechanics. ## Durability, Ordering, and Guarantees @@ -61,13 +61,13 @@ defmodule Fact do The structure of an event record is defined by `Fact.Event.Schema`, including: - * field names for type, data, metadata, and tags - * storage keys for positions, timestamps, and stream attributes + * field names for type, data, metadata, and tags + * storage keys for positions, timestamps, and stream attributes * identifier and encoding strategies Events are represented as plain maps before being persisted and enriched with system metadata at commit-time. - + ## When to Use Fact Fact works best when: @@ -89,30 +89,30 @@ defmodule Fact do 4. Build projections and workflows from subscriptions See the documentation for `append/4`, `append_stream/5`, `read/3`, - and `subscribe/3` for operational details. + and `subscribe/3` for operational details. - > #### Here there be 🐉🐉 {: .warning} - > - > Elixir's type system isn't as strict as say F#. So I've done my best to describe the types, their - > format, and encoding. Many of these are not enforced, and supplying other types may compile but - > produce errors or unexpected behavior. + > #### Here there be 🐉🐉 {: .warning} + > + > Elixir's type system isn't as strict as say F#. So I've done my best to describe the types, their + > format, and encoding. Many of these are not enforced, and supplying other types may compile but + > produce errors or unexpected behavior. > > Just use the system as I intended, and it'll just work 😉. """ @typedoc """ A condition that **must not** be satisfied in order for an append operation to succeed. - + An `t:Fact.append_condition/0` allows you to express causal or state-dependent constraints - using queries against the event ledger. If the condition evaluates as a match, the append + using queries against the event ledger. If the condition evaluates as a match, the append is rejected. - + When `nil`, no conditional check is performed. The events are always appended. - - When a `t:fail_if_match/0` value is provided, the append function will fail if the + + When a `t:fail_if_match/0` value is provided, the append function will fail if the query matches any events already present in the ledger. - - When a `{fail_if_match, after_position}` tuple is provided, the append function will + + When a `{fail_if_match, after_position}` tuple is provided, the append function will fail if the query matches any events present in the ledger found after the specified position. """ @type append_condition :: @@ -120,7 +120,7 @@ defmodule Fact do @typedoc """ A condition that fails the append operation when matching events are found. - + May be expressed as a `t:Fact.Query.t/0`, a single `t:Fact.QueryItem.t/0`, or a list of `t:Fact.QueryItem.t/0`. All forms represent a predicate function evaluated against events committed to the ledger. @@ -129,7 +129,7 @@ defmodule Fact do @typedoc """ Options for `append/5`. - + * `:timeout` **(default: 5000)** - the maximum time (in milliseconds) to wait for the append operation to compile. """ @type append_options :: [timeout: timeout()] @@ -141,29 +141,29 @@ defmodule Fact do @typedoc """ Options for `append_stream/5`. - + * `:timeout` **(default: 5000)** - the maximum time (in milliseconds) to wait for the append operation to compile. """ @type append_stream_options :: [timeout: timeout()] @typedoc """ A unique identifier for a Fact database. - - It is used as the primary handle for all database operations. Many - Fact subsystems use this identifier to retrieve the database context + + It is used as the primary handle for all database operations. Many + Fact subsystems use this identifier to retrieve the database context in order to perform file and storage operations. """ @type database_id :: uuid_v4_base32_uppercase_sans_padding() @typedoc """ - The user-friendly name of the Fact database. + The user-friendly name of the Fact database. """ @type database_name :: String.t() @typedoc """ Represents an event before being written to the event store. - At minimum, it must define a `:type` key. + At minimum, it must define a `:type` key. It may also include: * `:data` - a map of custom data @@ -198,7 +198,7 @@ defmodule Fact do @type event_metadata :: map() @typedoc """ - A number indicating the location of the event within the ledger or an event stream. + A number indicating the location of the event within the ledger or an event stream. """ @type event_position :: pos_integer() @@ -209,7 +209,7 @@ defmodule Fact do @typedoc """ A schema definition describing the field names used in a `t:Fact.event/0`. - + Each key in this map represents a logical event attribute, and its value is the string key under which that attribute is stored in the event map. """ @@ -229,7 +229,7 @@ defmodule Fact do A consumer defined, domain specific id for a stream of events. An event stream represents a logical partition within the ledger which - is used to relate events for downstream system capabilities. The default + is used to relate events for downstream system capabilities. The default consistency boundary for persisting Domain-Driven Design (DDD) Aggregate Roots. """ @type event_stream_id :: no_whitespace_string() @@ -244,22 +244,22 @@ defmodule Fact do @type event_tags :: list(event_tag()) @typedoc """ - A consumer defined, domain-specific name for an event. + A consumer defined, domain-specific name for an event. - It is recommended they are named in the past-tense, and describe a fact that is important to capture for the domain. + It is recommended they are named in the past-tense, and describe a fact that is important to capture for the domain. """ @type event_type :: no_whitespace_string() @typedoc """ - A string that contains no whitespace characters of any kind, including + A string that contains no whitespace characters of any kind, including spaces, tabs, newlines, and non-displayable control characters. """ @type no_whitespace_string :: String.t() @typedoc """ A string whose internal structure is opaque to consumers. - - Opaque strings should be treated as identifiers or tokens whose format + + Opaque strings should be treated as identifiers or tokens whose format is not meaningful outside the system. Do not make assumptions about their contents or structure. """ @@ -349,7 +349,7 @@ defmodule Fact do @type subscribe_options :: list(subscribe_option) @typedoc """ - Represents the event sources that a process may subscribe to for notifications. + Represents the event sources that a process may subscribe to for notifications. """ @type subscribe_source :: :all @@ -358,8 +358,8 @@ defmodule Fact do | {:query, Fact.QueryItem.t() | [Fact.QueryItem.t()]} @typedoc """ - An RFC-4122 UUID v4 encoded in Base32, using only uppercase characters. - The encoding contains no padding characters (`=`). This defines the + An RFC-4122 UUID v4 encoded in Base32, using only uppercase characters. + The encoding contains no padding characters (`=`). This defines the expected format; it does not perform validation at runtime. """ @type uuid_v4_base32_uppercase_sans_padding :: String.t() @@ -383,15 +383,15 @@ defmodule Fact do * `{fail_if_match, after_position}` — the append will be rejected if the query matches any events whose position is strictly greater than - `after_position`. + `after_position`. If the condition is violated, the append is rejected and an error tuple including - a `Fact.ConcurrencyError` is returned. + a `Fact.ConcurrencyError` is returned. - On success, each appended event record is enriched with the `:event_id`, - `:event_store_position`, `:event_store_timestamp` fields defined by the configured - event schema. In addition the field keys used to define events (`type`, `data`, - `metatadata` and `tags`) are renamed according to the configured schema. + On success, each appended event record is enriched with the `:event_id`, + `:event_store_position`, `:event_store_timestamp` fields defined by the configured + event schema. In addition the field keys used to define events (`type`, `data`, + `metatadata` and `tags`) are renamed according to the configured schema. The function returns `{:ok, last_position}`, where `last_position` is the store position of the last appended event. @@ -402,9 +402,9 @@ defmodule Fact do iex> {:ok, pos} = Fact.append(db, %{type: "user_registered", data: %{id: 123}}) {:ok, 42} - + Append a duplicate: - + iex> {:ok, pos} = Fact.append(db, %{type: "user_registered", data: %{id: 123}}) {:ok, 43} @@ -416,15 +416,15 @@ defmodule Fact do {:error, %Fact.ConcurrencyError{source: :all, expected: 0, actual: 42}} Allow the append only if no matching events exist **after** a given position: - + iex> Fact.append(db, %{type: "user_registered"}, {fail_if_match, last_pos}) {:ok, 44} - + The final example was intentionally contrived; in practice, append conditions are - best applied to model explicit business invariants. For deeper guidance and + best applied to model explicit business invariants. For deeper guidance and real-world usage patterns, see the [Dynamic Consistency Boundary](https://dcb.events) - website. + website. """ @spec append( Fact.database_id(), @@ -443,78 +443,78 @@ defmodule Fact do @doc """ Appends one or more events to a stream. - - Event streams define a consistency boundary for a set of related events. To + + Event streams define a consistency boundary for a set of related events. To preserve this boundary, Fact, like many event stores uses stream position expectations to provide optimistic concurrency control. - + When appending to a stream, you may provide an expected position value: - + * A non negative integer - verifies that the stream is currently at that exact position before appending * `:any` **(default)** - no check is performed, the events are just appended * `:none` - ensures the stream does not yet exist (equivalent to `0`) * `:exists` - ensures the stream already exists (i.e. position ≥ 1) - - If the expectation is not met, the append is rejected and a + + If the expectation is not met, the append is rejected and a `{:error, %Fact.ConcurrencyError{}}` is returned. - On success, each appended event record is enriched with the stream fields as defined - by `:event_stream_id` and `:event_stream_position` of the configured schema + On success, each appended event record is enriched with the stream fields as defined + by `:event_stream_id` and `:event_stream_position` of the configured schema (see `Fact.Event.Schema`). The function returns `{:ok, last_stream_position}`, where the last_stream_position refers to the stream position of the final event written to the stream. - + It is strongly recommnded that you persis this returned position in your application state and reuse it in subsequent calls to `append_stream/5` to ensure consistency. ## Examples - + Append a single event. - + iex> {:ok, db} = Fact.open("data/turtle") {:ok, "TURTLE4F7Q6Y2X3VQKBJ5M7P4Z"} - + iex> Fact.append_stream(db, %{type: "egg_hatched", data: %{name: "Turts"}}, "turtle-1") {:ok, 1} - + iex> Fact.read(db, {:stream, "turtle-1"}) |> Enum.to_list() [ %{ "event_data" => %{"name" => "Turts"}, - "event_id" => "3bb4808303c847fd9ceb0a1251ef95da", - "event_metadata" => %{}, - "event_tags" => [], - "event_type" => "egg_hatched", - "store_position" => 2, - "store_timestamp" => 1765039106962264, - "stream_id" => "turtle-1", - "stream_position" => 1 - } - ] + "event_id" => "3bb4808303c847fd9ceb0a1251ef95da", + "event_metadata" => %{}, + "event_tags" => [], + "event_type" => "egg_hatched", + "store_position" => 2, + "store_timestamp" => 1765039106962264, + "stream_id" => "turtle-1", + "stream_position" => 1 + } + ] Append another event, expecting correct stream position. - + iex> Fact.append_stream(db, %{type: "MySecondEvent"}, "myteststream", 1) {:ok, 2} - + Append a third event, but provide an invalid expected position. - + iex> Fact.append_stream(:mydb, %{type: "MyThirdEvent"}, "myteststream", 1) {:error, %Fact.ConcurrencyError{source: "myteststream", actual: 2, expected: 1}} Append multiple events to a **new** stream. - + iex> Fact.append_stream(db, [%{type: "foo"}, %{type: "bar"}, %{type: "baz"}], "foobarbaz-1", :none) {:ok, 3} - + Expect a stream to not exist, yet it does. iex> Fact.append_stream(db, %{type: "foo"}, "foobarbaz-1", :none) {:ok, %Fact.ConcurrencyError{source: "foobarbaz-1", expected: :none, actual: 3}} Expect a stream to exist, when it does not exist. - + iex> Fact.append_stream(db, %{type: "foo"}, "foo-1", :exists) {:ok, %Fact.ConcurrencyError{source: "foo-1", expected: :exists, actual: 0}} @@ -549,12 +549,15 @@ defmodule Fact do * `:cache` - A keyword list of record cache options. See `Fact.RecordCache` for details. * `:max_size` - Maximum cache size in bytes. The cache is disabled when not set. * `:decay_interval` - Milliseconds between frequency decay sweeps. Defaults to `600_000` (10 minutes). - + * `:merkle` - A keyword list of Merkle Mountain Range options. See `Fact.MerkleMountainRange`. Requires CAS mode (`record_file_name` configured with `hash@1`). * `:batch_size` - Maximum events to buffer before flushing. Defaults to `1`. * `:flush_interval` - Maximum milliseconds before flushing a partial batch. Defaults to `1_000`. + * `:indexers` - A list of custom indexer specifications to start with the database. + See `t:Fact.DatabaseSupervisor.indexer_spec/0`. Each entry is a module or `{module, opts}` tuple. + * `:wal` - A keyword list of write-ahead log options. See `Fact.WriteAheadLog` for details. * `:enable_fsync` - Whether to call fsync after writes. Defaults to `true`. * `:max_file_size` - Maximum size in bytes of a WAL segment file before rotation. Defaults to `16_777_216` (16 MB). @@ -572,17 +575,27 @@ defmodule Fact do iex> {:ok, db} = Fact.open("data/turtles", merkle: [batch_size: 10, flush_interval: 1_000]) {:ok, "EF73AQJ6S5HHZE5PMX7ZP254QQ"} - + Opens a database with an in-memory record cache. iex> {:ok, db} = Fact.open("data/turtles", cache: [max_size: 512 * 1024 * 1024]) {:ok, "EF73AQJ6S5HHZE5PMX7ZP254QQ"} - + Opens a database with custom write-ahead log options. iex> {:ok, db} = Fact.open("data/turtles", wal: [max_file_size: 64 * 1024 * 1024, sync_interval: 500]) {:ok, "EF73AQJ6S5HHZE5PMX7ZP254QQ"} + Opens a database with custom indexers. + + iex> {:ok, db} = Fact.open("data/turtles", + ...> indexers: [ + ...> MyApp.UserIndexer, + ...> {Fact.EventDataIndexer, key: "tenant_id"} + ...> ] + ...> ) + {:ok, "EF73AQJ6S5HHZE5PMX7ZP254QQ"} + Subsequent calls to the same path return the same database id...with the same BEAM. iex> {:ok, db2} = Fact.open("data/turtles") @@ -633,26 +646,26 @@ defmodule Fact do ## Event Sources - Like most event stores, you can read from the global stream, or an individual event stream. + Like most event stores, you can read from the global stream, or an individual event stream. Fact provides a few more options... - * `:none` - the empty stream + * `:none` - the empty stream * `:all` - the all stream (a.k.a. the global stream; a.k.a. the ledger) * `{:stream, stream_id}` - an individual event stream - * `{:index, indexer_id, index}` - an event index + * `{:index, indexer_id, index}` - an event index * `{:query, query_items}` - an event query ## Options - + You may provide a keyword list with the following options to craft the results to fit your need. If any of the options are not specified, sensible defaults will be provided. - + ### direction: - + * `:forward` **(default)** - events are read in increasing position order (e.g. 1, 2, 3, ...) * `:backward` - events are read in decreasing position order (e.g. 100, 99, 98, ...) @@ -661,7 +674,7 @@ defmodule Fact do > Positions will not always increase or decrease by 1, it totally depends on the event source. ### position: - + Set the position to begin reading the event source. * `:start` **(default)** - the position immediately **before the first item** in the event source @@ -671,66 +684,66 @@ defmodule Fact do > #### A note on specific positions {: .info} > > The meaning of the integer position is specific to the event source. - > + > > * For streams, it refers to the stream position. > * Other event sources store position (i.e. the global stream position) > #### A reminder when reading from the end {: .tip} > - > Starting reads from the end is typically only used when **reading backwards**, or subscribing to a live source. + > Starting reads from the end is typically only used when **reading backwards**, or subscribing to a live source. > So if you're unexpectedly getting no results, double-check your direction and position options. > > I've made these mistake many a time...🤦 - > + > > * reading forward from the end - > * reading backward from the start + > * reading backward from the start ### count: - + Control the maximum size of the result set. - + * `:all` **(default)** - reads everything in the event source - * Or a non-negative integer. + * Or a non-negative integer. > #### You've been warned 🤠 🧑‍🚒 {: .warning} > > This option is super useful if you don't want to return a bazillion events to a consumer, - > spiking I/O ops, slowing response times, and generally clogging up the pipes. - > + > spiking I/O ops, slowing response times, and generally clogging up the pipes. + > > But this is one area where the default value will happily allow you to shoot yourself in the foot. ### result: - + Control the shape of the elements in result set. - - * `:event` **(default)** - each element is a map containing the event details. + + * `:event` **(default)** - each element is a map containing the event details. * `:record` - each element is a 2-tuple `{record_id, event}` containing both the record id and the event. * `:record_id` - each element is the record id of the event. - > #### Event schemas are ...kind of... configurable {: .info} - > + > #### Event schemas are ...kind of... configurable {: .info} + > > When using `:event` or `record`, the exact "shape" of the event depends on the configured schema. > > See the `Fact.Seam.EventSchema.Registry` for all the available schemas. > #### What's a record id??? {: .info} > - > Fact separates the concepts of an **event id** and a **record id**. The record id is the actual - > name of file on disk where the event is stored. This could be the same as the event id, and by + > Fact separates the concepts of an **event id** and a **record id**. The record id is the actual + > name of file on disk where the event is stored. This could be the same as the event id, and by > default it is... but it really depends on how the `Fact.RecordFile.Name` has been configured. ### eager: Controls if the result set will be enumerated or lazy. - + * `true` **(default)** - The internal `Stream` is enumerated and a `List` is returned. * `false` - The result is returned as `Stream` - > #### Fool me once, shame on, shame on you. Fool me ... you can't get fooled again {: .warning} - > + > #### Fool me once, shame on, shame on you. Fool me ... you can't get fooled again {: .warning} + > > If the event source being read has many events, and `:count` is `:all`, it would be - > quite wise 🧙‍♂️ to set this to **false**. Nearly all the internal components of Fact - > use lazy reads, this is really just meant as a convenience so you don't have to + > quite wise 🧙‍♂️ to set this to **false**. Nearly all the internal components of Fact + > use lazy reads, this is really just meant as a convenience so you don't have to > remember to `|> Enum.to_list()`. """ @spec read(database_id(), read_source(), read_options()) :: read_result() @@ -808,38 +821,38 @@ defmodule Fact do @doc """ Subscribe a process to an event source. - + A subscription streams new events to the subscriber process as they are appended to the event store. The subscriber process receives one message per event in the form: - + * `{:record, record}` - where record is a 2-tuple `{record_id, event_record}` (see `t:Fact.record()`) The subscription begins by replaying events from the specified source, starting at the configured position. Once all historical events have been delivered, the processes receives a `:caught_up` message and the subscription shifts into **live mode**, where it waits for and delivers new events as they arrive. - + ## Event Sources - Subscriptions support most of the same sources as `read/3`, with a few exceptions. + Subscriptions support most of the same sources as `read/3`, with a few exceptions. Subscribing to the empty stream (i.e. `:none`) would create a very lonely `Fact.CatchUpSubscription` process that never delivers any messages, and I don't believe that would be useful. If you have a legitimate use case, please make your case. It would be straightforward to implement and support. - Subscribing to query sources requires `t:Fact.QueryItem.t/0`; a `t:Fact.Query.t/0` won't work here. - A `t:Fact.Query.t/0` is just a function, and that function depends on data produced by a combination - of the `Fact.EventTypeIndexer`, `Fact.EventTagsIndexer`, and any number of `Fact.EventDataIndexer` processes. + Subscribing to query sources requires `t:Fact.QueryItem.t/0`; a `t:Fact.Query.t/0` won't work here. + A `t:Fact.Query.t/0` is just a function, and that function depends on data produced by a combination + of the `Fact.EventTypeIndexer`, `Fact.EventTagsIndexer`, and any number of `Fact.EventDataIndexer` processes. A `t:Fact.QueryItem.t/0` contains the metadata needed to subscribe to the correct indexers so the subscription and coordinate when events become "visible" to the subscriber. - + Could a `t:Fact.Query.t/0` be decompiled back into an AST so we could reconstruct that information? Probably! But for now, I'd rather spend my time building other things. If you're like my good buddy Tim and love spelunking through - abstract syntax trees, I'll happily take your pull request. + abstract syntax trees, I'll happily take your pull request. ## Options ### position: - + Set the position to begin reading the event source. * `:start` **(default)** - the position immediately **before the first item** in the event source @@ -852,7 +865,7 @@ defmodule Fact do ### subscriber: - + Specifies the PID of the process that will receive subscription message. Defaults to `self()`. """ @spec subscribe(database_id(), subscribe_source(), subscribe_options()) :: {:ok, pid()} diff --git a/lib/fact/database_supervisor.ex b/lib/fact/database_supervisor.ex index 731fec0..a4d2d8e 100644 --- a/lib/fact/database_supervisor.ex +++ b/lib/fact/database_supervisor.ex @@ -12,22 +12,65 @@ defmodule Fact.DatabaseSupervisor do Each child process is registered under a database-specific name via `Fact.Registry`, ensuring isolation between multiple database instances. - This supervisor is automatically started by `Fact.Supervisor` when a database is initialized, and consumers + This supervisor is automatically started by `Fact.Supervisor` when a database is initialized, and consumers typically interact with the database through higher-level APIs rather than directly starting this supervisor. """ use Supervisor @typedoc """ - Runtime options for database subsystems, passed via the `:opts` key. + Specifies a custom indexer to start with the database. + Can be a bare module name or a `{module, opts}` tuple. When given as a bare module, + the indexer is started with no key and no options. + + ## Options + + * `:key` - An `t:Fact.EventIndexer.indexer_key/0` for parameterized indexers. Allows + multiple instances of the same module to run, each indexing a different dimension. + * `:options` - A list of `t:Fact.EventIndexer.indexer_option/0` values passed to the + `c:Fact.EventIndexer.index_event/3` callback. + + ## Examples + + # Bare module + MyApp.UserIndexer + + # With a key (e.g. EventDataIndexer for a specific field) + {Fact.EventDataIndexer, key: "tenant_id"} + + # With key and options + {MyApp.RegionIndexer, key: "us-east", options: [separator: "-"]} + """ + @typedoc since: "0.3.1" + @type indexer_spec :: + module() + | {module(), [indexer_spec_option()]} + + @typedoc """ + Options for an `t:indexer_spec/0`. + + * `:key` - An `t:Fact.EventIndexer.indexer_key/0` to distinguish parameterized instances. + * `:options` - Options forwarded to `c:Fact.EventIndexer.index_event/3`. + """ + @typedoc since: "0.3.1" + @type indexer_spec_option :: + {:key, Fact.EventIndexer.indexer_key()} + | {:options, Fact.EventIndexer.indexer_options()} + + @typedoc """ + Options used when starting a `Fact.DatabaseSupervisor`. + + * `:context` - (required) The `Fact.Context` providing database identity and configuration. + * `:opts` - (optional) Runtime options for database subsystems. Supports: * `:cache` - Record cache options. See `t:Fact.RecordCache.cache_option/0`. + * `:indexers` - Custom indexer specifications. See `t:indexer_spec/0`. * `:merkle` - Merkle Mountain Range options. See `t:Fact.MerkleMountainRange.merkle_option/0`. - Requires CAS mode. May also be the bare atom `:merkle` to enable with defaults. * `:wal` - Write-ahead log options. See `t:Fact.WriteAheadLog.wal_option/0`. """ - @typedoc since: "0.3.0" + @typedoc since: "0.3.1" @type subsystem_option :: {:cache, [Fact.RecordCache.cache_option()]} + | {:indexers, indexer_spec()} | {:merkle, [Fact.MerkleMountainRange.merkle_option()]} | {:wal, [Fact.WriteAheadLog.wal_option()]} @@ -37,7 +80,7 @@ defmodule Fact.DatabaseSupervisor do * `:context` - (required) The `Fact.Context` providing database identity and configuration. * `:opts` - (optional) Runtime options for database subsystems. See `t:subsystem_option/0`. """ - @typedoc since: "0.3.0" + @typedoc since: "0.3.1" @type option :: {:context, Fact.Context.t()} | {:opts, [subsystem_option()]} @@ -145,8 +188,32 @@ defmodule Fact.DatabaseSupervisor do end merkle_children = merkle_children(context, database_id, opts) + indexer_children = indexer_children(database_id, opts) + + Supervisor.init(children ++ cache_children ++ merkle_children ++ indexer_children, + strategy: :one_for_one + ) + end + + defp indexer_children(database_id, opts) do + opts + |> Keyword.get(:indexers, []) + |> Enum.map(&indexer_child_spec(database_id, &1)) + end + + defp indexer_child_spec(database_id, module) when is_atom(module) do + indexer_child_spec(database_id, {module, []}) + end + + defp indexer_child_spec(database_id, {module, opts}) do + key = Keyword.get(opts, :key) + options = Keyword.get(opts, :options, []) - Supervisor.init(children ++ cache_children ++ merkle_children, strategy: :one_for_one) + {module, + database_id: database_id, + key: key, + options: options, + name: Fact.Registry.via(database_id, {module, key})} end defp merkle_children(context, database_id, opts) do diff --git a/lib/fact/event_indexer.ex b/lib/fact/event_indexer.ex index aa1e525..c8e569f 100644 --- a/lib/fact/event_indexer.ex +++ b/lib/fact/event_indexer.ex @@ -26,10 +26,11 @@ defmodule Fact.EventIndexer do ## Custom Indexers - Custom indexers only need to implement the callback. + Custom indexers only need to implement the callback, then include them in the `:indexers` + option when opening the database. ### Examples - + This would produce an index for every user, including all the events which define a `user_id` in the event data. defmodule YourApp.UserIndexer do @@ -37,26 +38,53 @@ defmodule Fact.EventIndexer do @impl true def index_event(schema, event, _opts) do - unless is_nil(user_id = Map.get(event[schema.event_data], "user_id")), + unless is_nil(user_id = Map.get(event[schema.event_data], "user_id")), do: to_string(user_id) - end + end end - - This would produce an index for every tenant, including all the events which define a `tenant_id` in the event + + This would produce an index for every tenant, including all the events which define a `tenant_id` in the event metadata. - + defmodule YourApp.TenantIndexer do use Fact.EventIndexer @impl true def index_event(schema, event, _opts) do unless is_nil(tenant_id = Map.get(event[schema.event_metadata], "tenant_id")), - do: to_string(tenant_id) + do: to_string(tenant_id) end end + ### Configuration + + Pass custom indexers to `Fact.open/2` via the `:indexers` option. Bare modules, `{module, opts}` + tuples, and `Fact.EventDataIndexer` instances can all be mixed freely. + + {:ok, db} = Fact.open("data/my_app", + indexers: [ + YourApp.UserIndexer, + YourApp.TenantIndexer, + {Fact.EventDataIndexer, key: "order_id"} + ] + ) + + The same configuration works in application supervision trees: + + # In your Application.start/2 + children = [ + {Fact.Supervisor, + databases: [ + {"data/my_app", + indexers: [ + YourApp.UserIndexer, + {Fact.EventDataIndexer, key: "order_id"} + ]} + ]} + ] + """ - @moduledoc since: "0.1.0" + @moduledoc since: "0.3.1" @typedoc """ The values that can be return by a `c:Fact.EventIndexer.index_event` callback function. diff --git a/mix.exs b/mix.exs index 5bfe7ef..be02eb7 100644 --- a/mix.exs +++ b/mix.exs @@ -70,6 +70,7 @@ defmodule Fact.MixProject do extra_section: "GUIDES", extras: [ "LICENSE", + "guides/components/indexers.md", "guides/components/merkle-mountain-range.md", "guides/components/queries-and-indexes.md", "guides/components/record-cache.md", @@ -87,6 +88,7 @@ defmodule Fact.MixProject do "guides/introduction/process-model.md" ], Components: [ + "guides/components/indexers.md", "guides/components/merkle-mountain-range.md", "guides/components/queries-and-indexes.md", "guides/components/record-cache.md", diff --git a/test/fact/custom_indexers_test.exs b/test/fact/custom_indexers_test.exs new file mode 100644 index 0000000..80a398e --- /dev/null +++ b/test/fact/custom_indexers_test.exs @@ -0,0 +1,70 @@ +defmodule Fact.CustomIndexersTest do + @moduledoc false + + use ExUnit.Case + + @moduletag :capture_log + + defmodule BreedIndexer do + use Fact.EventIndexer + + @impl true + def index_event(schema, event, _opts) do + unless is_nil(breed = Map.get(event[schema.event_data], "breed")), + do: to_string(breed) + end + end + + describe "custom indexers via Fact.open/2" do + setup do + path = TestHelper.create_db("custom_indexer_") + on_exit(fn -> TestHelper.rm_rf(path) end) + + {:ok, db} = + Fact.open(path, + indexers: [ + BreedIndexer, + {Fact.EventDataIndexer, key: "name"} + ] + ) + + {:ok, db: db} + end + + test "custom indexer processes are running", %{db: db} do + assert [{pid, _}] = + Registry.lookup(Fact.Registry.registry(db), {BreedIndexer, nil}) + + assert Process.alive?(pid) + end + + test "EventDataIndexer with key is running", %{db: db} do + assert [{pid, _}] = + Registry.lookup(Fact.Registry.registry(db), {Fact.EventDataIndexer, "name"}) + + assert Process.alive?(pid) + end + + test "custom indexer indexes events", %{db: db} do + event = %{type: "pet_adopted", data: %{name: "Bandit", breed: "Lab"}} + {:ok, position} = Fact.append(db, event) + + TestHelper.subscribe_and_wait(db, position) + + records = Fact.read(db, {:index, {BreedIndexer, nil}, "Lab"}) + assert length(records) == 1 + assert hd(records)["event_data"]["breed"] == "Lab" + end + + test "EventDataIndexer via config indexes events", %{db: db} do + event = %{type: "pet_adopted", data: %{name: "Snicker", breed: "Mutt"}} + {:ok, position} = Fact.append(db, event) + + TestHelper.subscribe_and_wait(db, position) + + records = Fact.read(db, {:index, {Fact.EventDataIndexer, "name"}, "Snicker"}) + assert length(records) == 1 + assert hd(records)["event_data"]["name"] == "Snicker" + end + end +end From fedcc1d12305190c5fc17e31b61b7cb478bf7916 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Wed, 1 Apr 2026 22:07:35 -0700 Subject: [PATCH 07/17] chore: update guide with new metadata indexer. --- guides/components/indexers.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/guides/components/indexers.md b/guides/components/indexers.md index 383651a..72263f1 100644 --- a/guides/components/indexers.md +++ b/guides/components/indexers.md @@ -34,7 +34,9 @@ restart. ## Built-in Indexers -Fact starts six indexers automatically with every database. These power the core read APIs and query system. +Fact ships with eight indexers. Six start automatically with every database and power the core read APIs and query +system. The remaining two, `Fact.EventDataIndexer` and `Fact.EventMetadataIndexer`, are parameterized and started +via configuration. ### Event Stream Indexer @@ -118,6 +120,20 @@ targets a single key and is identified by that key. /indices/event_data/tenant_id/acme ``` +### Event Metadata Indexer + +**Module:** `Fact.EventMetadataIndexer` + +A parameterized indexer that extracts the value of a specific field from the event metadata payload. Works the same +way as `Fact.EventDataIndexer` but reads from metadata instead of data. Useful for indexing operational concerns +like correlation IDs, causation IDs, or tenant identifiers carried in metadata. + +``` +/indices/event_metadata/correlation_id/abc-123 +/indices/event_metadata/correlation_id/def-456 +/indices/event_metadata/causation_id/xyz-789 +``` + ## Custom Indexers Custom indexers allow you to define application-specific indexes that are maintained automatically as events are @@ -252,7 +268,7 @@ children = [ ] ``` -Custom indexers are **additive** — they run alongside the six built-in indexers, which are always started. +Custom indexers are **additive** — they run alongside the built-in indexers that are always started. ## Reading from Custom Indexes @@ -315,6 +331,9 @@ indices/ ├── event_data/ # EventDataIndexer │ └── / │ └── +├── event_metadata/ # EventMetadataIndexer +│ └── / +│ └── └── user/ # MyApp.UserIndexer (custom) └── ``` From d385e9b5c76671a58c4b4310f38fc95d8f386602 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Thu, 2 Apr 2026 00:16:27 -0700 Subject: [PATCH 08/17] chore: update doc attributes --- lib/fact/database_supervisor.ex | 6 +++--- lib/fact/event_indexer.ex | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/fact/database_supervisor.ex b/lib/fact/database_supervisor.ex index a4d2d8e..9f315ab 100644 --- a/lib/fact/database_supervisor.ex +++ b/lib/fact/database_supervisor.ex @@ -41,7 +41,7 @@ defmodule Fact.DatabaseSupervisor do # With key and options {MyApp.RegionIndexer, key: "us-east", options: [separator: "-"]} """ - @typedoc since: "0.3.1" + @typedoc since: "0.4.0" @type indexer_spec :: module() | {module(), [indexer_spec_option()]} @@ -52,7 +52,7 @@ defmodule Fact.DatabaseSupervisor do * `:key` - An `t:Fact.EventIndexer.indexer_key/0` to distinguish parameterized instances. * `:options` - Options forwarded to `c:Fact.EventIndexer.index_event/3`. """ - @typedoc since: "0.3.1" + @typedoc since: "0.4.0" @type indexer_spec_option :: {:key, Fact.EventIndexer.indexer_key()} | {:options, Fact.EventIndexer.indexer_options()} @@ -80,7 +80,7 @@ defmodule Fact.DatabaseSupervisor do * `:context` - (required) The `Fact.Context` providing database identity and configuration. * `:opts` - (optional) Runtime options for database subsystems. See `t:subsystem_option/0`. """ - @typedoc since: "0.3.1" + @typedoc since: "0.4.0" @type option :: {:context, Fact.Context.t()} | {:opts, [subsystem_option()]} diff --git a/lib/fact/event_indexer.ex b/lib/fact/event_indexer.ex index c8e569f..88428ea 100644 --- a/lib/fact/event_indexer.ex +++ b/lib/fact/event_indexer.ex @@ -84,7 +84,7 @@ defmodule Fact.EventIndexer do ] """ - @moduledoc since: "0.3.1" + @moduledoc since: "0.4.0" @typedoc """ The values that can be return by a `c:Fact.EventIndexer.index_event` callback function. From afdc6288edd0d09e44ad2ec3776a454e7d1eff7b Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Wed, 1 Apr 2026 21:51:13 -0700 Subject: [PATCH 09/17] feat: Adds Fact.EventMetadataIndexer for indexing by correlation_id, causation_id, tenant_id, or other metadata you may add to your events. --- lib/fact/event_indexer.ex | 4 +- lib/fact/event_metadata_indexer.ex | 55 +++++++++++++++++++++++ mix.exs | 1 + test/fact/event_metadata_indexer_test.exs | 52 +++++++++++++++++++++ 4 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 lib/fact/event_metadata_indexer.ex create mode 100644 test/fact/event_metadata_indexer_test.exs diff --git a/lib/fact/event_indexer.ex b/lib/fact/event_indexer.ex index aa1e525..b0b774a 100644 --- a/lib/fact/event_indexer.ex +++ b/lib/fact/event_indexer.ex @@ -108,6 +108,7 @@ defmodule Fact.EventIndexer do ### Built-in Indexers * `Fact.EventDataIndexer` - requires an `t:Fact.EventIndexer.indexer_key/0` + * `Fact.EventMetadataIndexer` - requires an `t:Fact.EventIndexer.indexer_key/0` * `Fact.EventStreamCategoryIndexer` * `Fact.EventStreamIndexer` * `Fact.EventStreamsByCategoryIndexer` @@ -115,9 +116,10 @@ defmodule Fact.EventIndexer do * `Fact.EventTagsIndexer` * `Fact.EventTypeIndexer` """ - @typedoc since: "0.1.2" + @typedoc since: "0.3.1" @type indexer_id() :: Fact.EventDataIndexer.id() + | Fact.EventMetadataIndexer.id() | Fact.EventStreamCategoryIndexer.id() | Fact.EventStreamIndexer.id() | Fact.EventStreamsByCategoryIndexer.id() diff --git a/lib/fact/event_metadata_indexer.ex b/lib/fact/event_metadata_indexer.ex new file mode 100644 index 0000000..4a1bc66 --- /dev/null +++ b/lib/fact/event_metadata_indexer.ex @@ -0,0 +1,55 @@ +defmodule Fact.EventMetadataIndexer do + @moduledoc """ + Indexes events by the values of a specified field within event metadata. + """ + @moduledoc since: "0.3.1" + use Fact.EventIndexer + + @typedoc """ + The id for a Fact.EventMetadataIndexer. + """ + @typedoc since: "0.3.1" + @type id :: {Fact.EventMetadataIndexer, Fact.EventIndexer.indexer_key()} + + @typedoc """ + Custom option values passed to the `c:Fact.EventIndexer.index_event/3` callback function. + """ + @typedoc since: "0.3.1" + @type option :: {:indexer_key, String.t()} | Fact.EventIndexer.indexer_option() + + @typedoc """ + Custom options passed to the `c:Fact.EventIndexer.index_event/3` callback function. + """ + @typedoc since: "0.3.1" + @type options :: [option()] + + @doc """ + Retrieves the value for the configured `:indexer_key` from the event's metadata payload. + + ### Options + + * `:indexer_key` — required, specifies the field to lookup within the event metadata + """ + @doc since: "0.3.1" + @impl true + @spec index_event(Fact.event_record_schema(), Fact.event_record(), options()) :: + String.t() | nil + def index_event(schema, event, opts) do + event_metadata = event[schema.event_metadata] + indexer_key = Keyword.get(opts, :indexer_key) + + unless is_nil(value = get(event_metadata, indexer_key)) do + to_string(value) + end + end + + defp get(map, key) when is_binary(key) do + case Map.fetch(map, key) do + {:ok, value} -> value + :error -> Map.get(map, String.to_existing_atom(key)) + end + rescue + ArgumentError -> + nil + end +end diff --git a/mix.exs b/mix.exs index 5bfe7ef..41f5f11 100644 --- a/mix.exs +++ b/mix.exs @@ -162,6 +162,7 @@ defmodule Fact.MixProject do Indexing: [ Fact.EventDataIndexer, Fact.EventIndexer, + Fact.EventMetadataIndexer, Fact.EventStreamCategoryIndexer, Fact.EventStreamIndexer, Fact.EventStreamsByCategoryIndexer, diff --git a/test/fact/event_metadata_indexer_test.exs b/test/fact/event_metadata_indexer_test.exs new file mode 100644 index 0000000..1cc0f02 --- /dev/null +++ b/test/fact/event_metadata_indexer_test.exs @@ -0,0 +1,52 @@ +defmodule Fact.EventMetadataIndexerTest do + @moduledoc false + + use ExUnit.Case + + alias Fact.EventMetadataIndexer + + setup_all do + seam = Fact.Seam.EventSchema.Standard.V1.init() + schema = Fact.Seam.EventSchema.Standard.V1.get(seam, []) + + {:ok, schema: schema} + end + + test "indexes the specified indexer_key for event metadata with string key", %{schema: schema} do + event = %{ + schema.event_type => "OrderPlaced", + schema.event_data => %{"order_id" => "ord_1"}, + schema.event_metadata => %{"correlation_id" => "abc-123", "causation_id" => "def-456"} + } + + assert "abc-123" = + EventMetadataIndexer.index_event(schema, event, indexer_key: "correlation_id") + + assert "def-456" == + EventMetadataIndexer.index_event(schema, event, indexer_key: "causation_id") + end + + test "indexes the specified indexer_key for event metadata with atom key", %{schema: schema} do + event = %{ + schema.event_type => "OrderPlaced", + schema.event_data => %{"order_id" => "ord_1"}, + schema.event_metadata => %{correlation_id: "abc-123", causation_id: "def-456"} + } + + assert "abc-123" = + EventMetadataIndexer.index_event(schema, event, indexer_key: "correlation_id") + + assert "def-456" == + EventMetadataIndexer.index_event(schema, event, indexer_key: "causation_id") + end + + test "does not index event when event metadata does not define the key", %{schema: schema} do + event = %{ + schema.event_type => "OrderPlaced", + schema.event_data => %{"order_id" => "ord_1"}, + schema.event_metadata => %{"correlation_id" => "abc-123"} + } + + assert nil === EventMetadataIndexer.index_event(schema, event, indexer_key: "tenant_id") + end +end From 1aeaf5ce580c0825a42227875d0459d354f446fe Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Thu, 2 Apr 2026 00:24:37 -0700 Subject: [PATCH 10/17] chore: update doc attributes --- lib/fact/event_indexer.ex | 2 +- lib/fact/event_metadata_indexer.ex | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/fact/event_indexer.ex b/lib/fact/event_indexer.ex index b0b774a..efbcc97 100644 --- a/lib/fact/event_indexer.ex +++ b/lib/fact/event_indexer.ex @@ -116,7 +116,7 @@ defmodule Fact.EventIndexer do * `Fact.EventTagsIndexer` * `Fact.EventTypeIndexer` """ - @typedoc since: "0.3.1" + @typedoc since: "0.4.0" @type indexer_id() :: Fact.EventDataIndexer.id() | Fact.EventMetadataIndexer.id() diff --git a/lib/fact/event_metadata_indexer.ex b/lib/fact/event_metadata_indexer.ex index 4a1bc66..13e6116 100644 --- a/lib/fact/event_metadata_indexer.ex +++ b/lib/fact/event_metadata_indexer.ex @@ -2,25 +2,25 @@ defmodule Fact.EventMetadataIndexer do @moduledoc """ Indexes events by the values of a specified field within event metadata. """ - @moduledoc since: "0.3.1" + @moduledoc since: "0.4.0" use Fact.EventIndexer @typedoc """ The id for a Fact.EventMetadataIndexer. """ - @typedoc since: "0.3.1" + @typedoc since: "0.4.0" @type id :: {Fact.EventMetadataIndexer, Fact.EventIndexer.indexer_key()} @typedoc """ Custom option values passed to the `c:Fact.EventIndexer.index_event/3` callback function. """ - @typedoc since: "0.3.1" + @typedoc since: "0.4.0" @type option :: {:indexer_key, String.t()} | Fact.EventIndexer.indexer_option() @typedoc """ Custom options passed to the `c:Fact.EventIndexer.index_event/3` callback function. """ - @typedoc since: "0.3.1" + @typedoc since: "0.4.0" @type options :: [option()] @doc """ @@ -30,7 +30,7 @@ defmodule Fact.EventMetadataIndexer do * `:indexer_key` — required, specifies the field to lookup within the event metadata """ - @doc since: "0.3.1" + @doc since: "0.4.0" @impl true @spec index_event(Fact.event_record_schema(), Fact.event_record(), options()) :: String.t() | nil From 1ca81c8f7d32a7a27a2ffa6a95d3c53559268e4d Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Tue, 7 Apr 2026 23:33:29 -0700 Subject: [PATCH 11/17] feat: Adds encryption to the write ahead log entries. --- lib/fact/database_supervisor.ex | 9 ++++- lib/fact/write_ahead_log.ex | 60 +++++++++++++++++++++++++++------ test/fact/encryption_test.exs | 40 ++++++++++++++++++++++ 3 files changed, 98 insertions(+), 11 deletions(-) diff --git a/lib/fact/database_supervisor.ex b/lib/fact/database_supervisor.ex index 7083fa9..2a5295a 100644 --- a/lib/fact/database_supervisor.ex +++ b/lib/fact/database_supervisor.ex @@ -93,8 +93,15 @@ defmodule Fact.DatabaseSupervisor do def init({%Fact.Context{database_id: database_id} = context, opts}) do Fact.Registry.register(context) + encryption_opts = Keyword.get(opts, :encryption, []) + encrypted = Keyword.get(encryption_opts, :wrapped_dek) != nil + wal_opts = - [database_id: database_id, name: Fact.Registry.via(database_id, Fact.WriteAheadLog)] ++ + [ + database_id: database_id, + name: Fact.Registry.via(database_id, Fact.WriteAheadLog), + encrypted: encrypted + ] ++ Keyword.get(opts, :wal, []) cache_opts = Keyword.get(opts, :cache, []) diff --git a/lib/fact/write_ahead_log.ex b/lib/fact/write_ahead_log.ex index 802d46f..c4e94f8 100644 --- a/lib/fact/write_ahead_log.ex +++ b/lib/fact/write_ahead_log.ex @@ -163,7 +163,8 @@ defmodule Fact.WriteAheadLog do max_file_size: nil, max_segments: nil, should_fsync: false, - sync_interval: 0 + sync_interval: 0, + encrypted: false # ----------------------------- # Init @@ -195,6 +196,8 @@ defmodule Fact.WriteAheadLog do file = segment_path(database_id, segment_index) {:ok, buffer} = :file.open(file, [:append, :binary, :read, :write]) + encrypted = Keyword.get(opts, :encrypted, false) + state = %__MODULE__{ database_id: database_id, file: file, @@ -203,7 +206,8 @@ defmodule Fact.WriteAheadLog do max_file_size: max_file_size, max_segments: max_segments, should_fsync: enable_fsync, - sync_interval: sync_interval + sync_interval: sync_interval, + encrypted: encrypted } last_seq = get_last_sequence_no(state) @@ -224,6 +228,7 @@ defmodule Fact.WriteAheadLog do {:ok, state} -> seq = state.last_seq + 1 + data = maybe_encrypt(data, state) entry = Entry.create(seq, data, checkpoint?) {:ok, state} = write_entry_to_buffer(entry, %{state | last_seq: seq}) @@ -252,7 +257,7 @@ defmodule Fact.WriteAheadLog do def handle_call({:read_all, from_checkpoint?}, _from, state) do files = segment_files(state.database_id) |> Enum.sort() - {:reply, read_segments(files, from_checkpoint?), state} + {:reply, read_segments(files, from_checkpoint?, state), state} end def handle_call({:read_all_from_offset, offset, from_checkpoint?}, _from, state) do @@ -261,7 +266,7 @@ defmodule Fact.WriteAheadLog do |> Enum.filter(fn {idx, _} -> idx >= offset end) |> Enum.sort() - {:reply, read_segments(files, from_checkpoint?), state} + {:reply, read_segments(files, from_checkpoint?, state), state} end def handle_call(:repair, _from, state) do @@ -286,6 +291,39 @@ defmodule Fact.WriteAheadLog do {:noreply, state} end + # ----------------------------- + # Encryption + # ----------------------------- + + @nonce_size 12 + @auth_tag_size 16 + + defp maybe_encrypt(data, %{encrypted: false}), do: data + + defp maybe_encrypt(data, %{encrypted: true, database_id: database_id}) do + {:ok, dek} = Fact.KeyRing.get_dek(database_id) + nonce = :crypto.strong_rand_bytes(@nonce_size) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, data, "", true) + + <> + end + + defp maybe_decrypt_entry(entry, %{encrypted: false}), do: entry + + defp maybe_decrypt_entry(entry, %{encrypted: true, database_id: database_id}) do + {:ok, dek} = Fact.KeyRing.get_dek(database_id) + + <> = entry.data + + plaintext = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, ciphertext, "", auth_tag, false) + + %{entry | data: plaintext} + end + # ----------------------------- # Helpers # ----------------------------- @@ -368,10 +406,10 @@ defmodule Fact.WriteAheadLog do # Reading # --------------------- - defp read_segments(files, from_checkpoint?) do + defp read_segments(files, from_checkpoint?, state) do Enum.reduce(files, {[], 0}, fn {_idx, path}, {acc, last_cp} -> {:ok, fd} = File.open(path, [:read, :binary]) - {entries, cp} = do_read_entries(fd, from_checkpoint?) + {entries, cp} = do_read_entries(fd, from_checkpoint?, state) File.close(fd) if from_checkpoint? and cp > last_cp do @@ -383,17 +421,19 @@ defmodule Fact.WriteAheadLog do |> elem(0) end - defp do_read_entries(fd, from_checkpoint?) do - read_loop(fd, [], 0, from_checkpoint?) + defp do_read_entries(fd, from_checkpoint?, state) do + read_loop(fd, [], 0, from_checkpoint?, state) end - defp read_loop(fd, acc, last_cp, from_checkpoint?) do + defp read_loop(fd, acc, last_cp, from_checkpoint?, state) do case :file.read(fd, 4) do {:ok, <>} -> case :file.read(fd, size) do {:ok, bin} -> case Entry.deserialize(bin) do {:ok, entry} -> + entry = maybe_decrypt_entry(entry, state) + {new_acc, new_cp} = if entry.is_checkpoint and from_checkpoint? do {[], entry.lsn} @@ -401,7 +441,7 @@ defmodule Fact.WriteAheadLog do {[entry | acc], last_cp} end - read_loop(fd, new_acc, new_cp, from_checkpoint?) + read_loop(fd, new_acc, new_cp, from_checkpoint?, state) {:error, _} -> {Enum.reverse(acc), last_cp} diff --git a/test/fact/encryption_test.exs b/test/fact/encryption_test.exs index 082ff22..758e644 100644 --- a/test/fact/encryption_test.exs +++ b/test/fact/encryption_test.exs @@ -196,6 +196,46 @@ defmodule Fact.EncryptionTest do assert record["event_data"]["name"] == "Shelldon" end + test "WAL segment files do not contain plaintext event data", %{ + path: path, + encryption_meta: meta + } do + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + + {:ok, position} = Fact.append(db, %{type: "turtle_hatched", data: %{name: "Shelldon"}}) + TestHelper.subscribe_and_wait(db, position) + + Fact.WriteAheadLog.sync(db) + + wal_dir = Path.join(path, "wal") + + wal_bytes = + Path.wildcard(Path.join(wal_dir, "*")) + |> Enum.map(&File.read!/1) + |> Enum.join() + + refute String.contains?(wal_bytes, "turtle_hatched") + refute String.contains?(wal_bytes, "Shelldon") + end + + test "WAL recovery works with encrypted entries", %{path: path, encryption_meta: meta} do + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + + {:ok, position} = Fact.append(db, %{type: "turtle_hatched", data: %{name: "Shelldon"}}) + TestHelper.subscribe_and_wait(db, position) + + # Close and reopen to trigger WAL recovery + Process.flag(:trap_exit, true) + :ok = Fact.close(db) + Process.flag(:trap_exit, false) + + {:ok, db2} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + + events = Fact.read(db2, :all) + event_types = Enum.map(events, & &1["event_type"]) + assert "turtle_hatched" in event_types + end + test "wrong encryption key returns clear error", %{path: path} do wrong_key = :crypto.strong_rand_bytes(32) |> Base.encode64() assert {:error, :invalid_encryption_key} = Fact.open(path, encryption_key: wrong_key) From b676c14819aef3049476fdea5bb9dfa44982fb3f Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Wed, 8 Apr 2026 07:36:05 -0700 Subject: [PATCH 12/17] chore: format code --- lib/fact/write_ahead_log.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fact/write_ahead_log.ex b/lib/fact/write_ahead_log.ex index c4e94f8..7f5a5ac 100644 --- a/lib/fact/write_ahead_log.ex +++ b/lib/fact/write_ahead_log.ex @@ -315,8 +315,8 @@ defmodule Fact.WriteAheadLog do defp maybe_decrypt_entry(entry, %{encrypted: true, database_id: database_id}) do {:ok, dek} = Fact.KeyRing.get_dek(database_id) - <> = entry.data + <> = + entry.data plaintext = :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, ciphertext, "", auth_tag, false) From 255f6fd361eb6fefc5511328fd9624ba82ee496d Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Fri, 17 Apr 2026 22:04:55 -0700 Subject: [PATCH 13/17] fix: deduplicate event tags before appending. --- lib/fact/event_ledger.ex | 13 +++++++++---- test/fact_test.exs | 11 +++++++++++ test/test_helper.exs | 10 +++++----- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/lib/fact/event_ledger.ex b/lib/fact/event_ledger.ex index ebaacb6..7e2dbf3 100644 --- a/lib/fact/event_ledger.ex +++ b/lib/fact/event_ledger.ex @@ -1,6 +1,6 @@ defmodule Fact.EventLedger do @moduledoc """ - This is the Judge Judy of the system, it manages the event ledger for a Fact database instance, + This is the Judge Judy of the system, it manages the event ledger for a Fact database instance, handling all commits, enforcing optimistic concurrency control via append conditions. `Fact.EventLedger` is a GenServer responsible for: @@ -28,8 +28,8 @@ defmodule Fact.EventLedger do @typedoc """ Defines the options that can be specified when committing. - - * `:timeout` (default: 5000) - specifies how long the caller should wait for the commit operation to complete in milliseconds + + * `:timeout` (default: 5000) - specifies how long the caller should wait for the commit operation to complete in milliseconds """ @typedoc since: "0.1.0" @type commit_option :: {:timeout, timeout()} @@ -67,7 +67,7 @@ defmodule Fact.EventLedger do Each event is enriched with additional metadata, including a timestamp, and assigning a store position. Events are then written to the configured storage and a reference to the event is appended to the ledger file. - If the commit operation is successful, an `{:appended, {record_id, event}}` message will be published by the + If the commit operation is successful, an `{:appended, {record_id, event}}` message will be published by the `Fact.EventPublisher` for each event that was written. """ @doc since: "0.2.0" @@ -290,6 +290,10 @@ defmodule Fact.EventLedger do event_with_renamed_keys = rename_keys(event, replacements) + deduplicated_tags = + Map.get(event_with_renamed_keys, schema.event_tags, []) + |> Enum.uniq() + enriched_event = %{ schema.event_data => %{}, @@ -300,6 +304,7 @@ defmodule Fact.EventLedger do schema.event_store_timestamp => timestamp } |> Map.merge(event_with_renamed_keys) + |> Map.put(schema.event_tags, deduplicated_tags) {enriched_event, next} end) diff --git a/test/fact_test.exs b/test/fact_test.exs index 3ecdb11..ab5c18a 100644 --- a/test/fact_test.exs +++ b/test/fact_test.exs @@ -32,6 +32,17 @@ defmodule FactTest do assert position == record["store_position"] end + test "append event with duplicate tags are deduplicated", %{db: db} do + event = %{type: "turtle_mutated", tags: ["mutant:1", "mutant:1"]} + assert {:ok, position} = Fact.append(db, event) + + record = + Fact.read(db, :all, position: position - 1, count: 1) + |> List.first() + + assert length(record["event_tags"]) == 1 + end + test "append and read it back via a type query", %{db: db} do event = %{type: "pizza_ordered"} {:ok, position} = Fact.append(db, event) diff --git a/test/test_helper.exs b/test/test_helper.exs index bb3eefa..e840e11 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,9 +1,6 @@ defmodule TestHelper do require Logger - @doc """ - Test helper method to create a database with a random name. - """ @doc """ Creates a CAS-mode database (hash@1 record file name) for MMR testing. """ @@ -29,6 +26,9 @@ defmodule TestHelper do path end + @doc """ + Test helper method to create a database with a random name. + """ def create_db(name_prefix) do path = :uuid.get_v4() @@ -60,8 +60,8 @@ defmodule TestHelper do @doc """ Subscribes to the database and waits for all the indexers to index events - up to the specified position. - + up to the specified position. + This uses a really naive approach when waiting, it just expects to receive some message with the timeout window, then waits again resetting the time window, a continues until the expected message is received. From 872141983a64d1c073548686a63a14b9c46c271e Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Fri, 17 Apr 2026 22:32:36 -0700 Subject: [PATCH 14/17] doc: organize module docs --- guides/components/encryption.md | 8 ++++---- mix.exs | 13 ++++++++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/guides/components/encryption.md b/guides/components/encryption.md index 744549c..47504ca 100644 --- a/guides/components/encryption.md +++ b/guides/components/encryption.md @@ -42,9 +42,9 @@ The wrapped DEK (encrypted with both the primary and recovery KEKs) is stored in alongside the database configuration. The KEKs themselves are never written to disk. ``` - ┌─────────────┐ - │ Primary KEK │ (runtime only, never on disk) - └──────┬──────┘ + ┌────────────────┐ + │ Primary KEK │ (runtime only, never on disk) + └────────┬───────┘ │ wraps ▼ ┌────────────────┐ @@ -53,7 +53,7 @@ alongside the database configuration. The KEKs themselves are never written to d │ encrypts ▼ ┌────────────────┐ - │ Record Files │ (encrypted on disk) + │ Record Files │ (encrypted on disk) └────────────────┘ ``` diff --git a/mix.exs b/mix.exs index cd1c504..bf1b1c0 100644 --- a/mix.exs +++ b/mix.exs @@ -145,12 +145,21 @@ defmodule Fact.MixProject do Fact.Storage ], Core: [ + Fact.BuildInfo, Fact.Database, Fact.EventLedger, Fact.EventReader, Fact.EventStreamWriter, Fact.Query, - Fact.QueryItem + Fact.QueryItem, + Fact.RecordCache, + Fact.WriteAheadLog, + Fact.WriteAheadLog.Entry + ], + Encryption: [ + Fact.Encryption.KEKProvider, + Fact.Encryption.KEKProvider.Explicit, + Fact.KeyRing ], Genesis: [ Fact.Genesis.Command.CreateDatabase.V1, @@ -226,11 +235,13 @@ defmodule Fact.MixProject do Fact.Seam.FileName.Registry, Fact.Seam.FileReader, Fact.Seam.FileReader.Adapter, + Fact.Seam.FileReader.Encrypted.V1, Fact.Seam.FileReader.FixedLength.V1, Fact.Seam.FileReader.Full.V1, Fact.Seam.FileReader.Registry, Fact.Seam.FileWriter, Fact.Seam.FileWriter.Adapter, + Fact.Seam.FileWriter.Encrypted.V1, Fact.Seam.FileWriter.Standard.V1, Fact.Seam.FileWriter.Registry, Fact.Seam.Instance, From fce494d822e54e3a85bbabbd6499e4f2e8a873b8 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Fri, 17 Apr 2026 23:00:44 -0700 Subject: [PATCH 15/17] chore: compute version based on tag --- mix.exs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 5bfe7ef..6c8aa32 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,6 @@ defmodule Fact.MixProject do use Mix.Project @name "Fact" - @version "0.3.1" @codename "Hardened Shell" @source_url "https://github.com/evntd/fact" @maintainers ["Jake Bruun"] @@ -12,7 +11,7 @@ defmodule Fact.MixProject do [ aliases: aliases(), app: :fact, - version: @version, + version: version(), codename: @codename, elixir: "~> 1.13", start_permanent: Mix.env() == :prod, @@ -332,4 +331,11 @@ defmodule Fact.MixProject do ] ] end + + defp version do + case System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true) do + {"v" <> version, 0} -> String.trim(version) + _ -> "0.0.0-dev" + end + end end From 77229399cb69730a5169e452f5cfbe22d40f73f7 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Fri, 17 Apr 2026 23:04:37 -0700 Subject: [PATCH 16/17] chore: bump version to 0.4.0 --- README.md | 6 +++--- guides/introduction/getting-started.md | 4 ++-- mix.exs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 60a79bf..4485851 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Add `fact` to your dependencies: ```elixir def deps do [ - {:fact, "~> 0.3.1"}, + {:fact, "~> 0.4.0"}, {:jason, "~> 1.4"} # required on Elixir 1.17 and earlier ] end @@ -226,13 +226,13 @@ Look up a database by name at runtime: | `mix fact.merkle.create_proof -p --position ` | Create an inclusion proof | | `mix fact.merkle.verify_proof --proof ` | Verify an inclusion proof | -## Coming in v0.4.0 +## New in v0.4.0 - **Encryption at rest** — AES-256-GCM authenticated encryption for event record files, with envelope encryption (DEK/KEK) and recovery key support - **Custom indexers** — define application-specific indexes with a single callback, configured via `Fact.open/2` - **Metadata indexer** — built-in indexer for event metadata fields (correlation IDs, tenant IDs, etc.) -## Roadmap +## Road to v1.0.0 - Telemetry - Full stack example application diff --git a/guides/introduction/getting-started.md b/guides/introduction/getting-started.md index cc55b15..df63706 100644 --- a/guides/introduction/getting-started.md +++ b/guides/introduction/getting-started.md @@ -4,7 +4,7 @@ ```elixir def deps do - [{:fact, "~> 0.3.1"}] + [{:fact, "~> 0.4.0"}] end ``` @@ -13,7 +13,7 @@ end ```elixir def deps do [ - {:fact, "~> 0.3.1"}, + {:fact, "~> 0.4.0"}, {:jason, "~> 1.4"} ] end diff --git a/mix.exs b/mix.exs index 5bfe7ef..059e563 100644 --- a/mix.exs +++ b/mix.exs @@ -3,7 +3,7 @@ defmodule Fact.MixProject do @name "Fact" @version "0.3.1" - @codename "Hardened Shell" + @codename "AESop" @source_url "https://github.com/evntd/fact" @maintainers ["Jake Bruun"] @authors ["Jake Bruun"] From 558aaf1327de2993917c400b39de4a0cb1e7bfdc Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Fri, 17 Apr 2026 23:35:30 -0700 Subject: [PATCH 17/17] chore: add fetch-depth so checkout to clone tags --- .github/workflows/publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e327834..91d3fe1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -47,6 +47,8 @@ jobs: steps: - uses: actions/checkout@v5 + with: + fetch-depth: 0 - name: Set up Elixir id: beam