From d0ffa1e045b2f975fee6882b2d8f8fd80fe3ce4d Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Wed, 1 Apr 2026 23:54:58 -0700 Subject: [PATCH 1/7] feat: Adds encryption at rest --- guides/components/encryption.md | 224 ++++++++++++++ lib/fact.ex | 19 ++ lib/fact/bootstrap_file.ex | 30 +- lib/fact/bootstrapper.ex | 110 ++++++- lib/fact/database_supervisor.ex | 82 ++++-- lib/fact/encryption/kek_provider.ex | 36 +++ lib/fact/encryption/kek_provider/explicit.ex | 46 +++ lib/fact/genesis/decider.ex | 39 ++- lib/fact/genesis/the_creator.ex | 56 +++- lib/fact/index_checkpoint_file.ex | 1 + lib/fact/index_file.ex | 1 + lib/fact/key_ring.ex | 111 +++++++ lib/fact/ledger_file.ex | 1 + lib/fact/lock_file.ex | 1 + lib/fact/record_file.ex | 13 +- lib/fact/seam/file_reader/encrypted/v1.ex | 89 ++++++ lib/fact/seam/file_reader/registry.ex | 1 + lib/fact/seam/file_writer/encrypted/v1.ex | 137 +++++++++ lib/fact/seam/file_writer/registry.ex | 2 +- lib/fact/supervisor.ex | 24 ++ lib/mix/tasks/fact/create.ex | 104 +++++-- lib/mix/tasks/fact/recover.ex | 196 +++++++++++++ mix.exs | 2 + test/fact/encryption_test.exs | 294 +++++++++++++++++++ 24 files changed, 1531 insertions(+), 88 deletions(-) create mode 100644 guides/components/encryption.md create mode 100644 lib/fact/encryption/kek_provider.ex create mode 100644 lib/fact/encryption/kek_provider/explicit.ex create mode 100644 lib/fact/key_ring.ex create mode 100644 lib/fact/seam/file_reader/encrypted/v1.ex create mode 100644 lib/fact/seam/file_writer/encrypted/v1.ex create mode 100644 lib/mix/tasks/fact/recover.ex create mode 100644 test/fact/encryption_test.exs diff --git a/guides/components/encryption.md b/guides/components/encryption.md new file mode 100644 index 0000000..744549c --- /dev/null +++ b/guides/components/encryption.md @@ -0,0 +1,224 @@ +# Encryption at Rest + +This guide explains Fact's encryption-at-rest feature, which protects event record files on disk using +authenticated encryption. It covers the motivation, how to create and open an encrypted database, key +management, and recovery procedures. + +At a high level: + +- Event record files are encrypted using **AES-256-GCM** (authenticated encryption) +- Encryption is configured at **database creation time** and backed by the seam architecture +- Keys use **envelope encryption** — a Data Encryption Key (DEK) encrypts records, and a Key Encryption + Key (KEK) wraps the DEK +- The DEK never appears in plaintext on disk +- A **recovery KEK** is required at creation time to prevent permanent data loss + +## Motivation + +Event stores accumulate history that is never deleted. This makes them high-value targets for data +exfiltration — a single disk image or backup copy exposes every event ever written. Encryption at rest +ensures that even if the underlying storage is compromised, the event data remains unreadable without +the encryption key. + +Fact encrypts at the **record file level**. Each event record file on disk is individually encrypted with +its own random nonce, providing strong confidentiality guarantees even if an attacker obtains partial +access to the storage directory. + +Ledger files, index files, checkpoint files, and lock files are **not encrypted**. These contain only +record IDs (opaque identifiers) and operational metadata, not event payloads. + +## Envelope Encryption + +Fact uses envelope encryption with two layers of keys: + +- **Data Encryption Key (DEK)** — A 256-bit key generated at database creation time. Used to encrypt + and decrypt every event record file. One DEK per database. +- **Key Encryption Key (KEK)** — Supplied by the operator at runtime. Wraps (encrypts) the DEK so it + can be stored safely on disk. Never touches disk in plaintext. +- **Recovery KEK** — A second KEK that can also unwrap the DEK. Required at creation time. Stored + separately by the operator for disaster recovery. + +The wrapped DEK (encrypted with both the primary and recovery KEKs) is stored in the `.bootstrap` file +alongside the database configuration. The KEKs themselves are never written to disk. + +``` + ┌─────────────┐ + │ Primary KEK │ (runtime only, never on disk) + └──────┬──────┘ + │ wraps + ▼ + ┌────────────────┐ + │ DEK │ (wrapped form stored in .bootstrap) + └────────┬───────┘ + │ encrypts + ▼ + ┌────────────────┐ + │ Record Files │ (encrypted on disk) + └────────────────┘ +``` + +## Creating an Encrypted Database + +Use `mix fact.create` with the `--encrypted` flag: + +```sh +$ mix fact.create -p data/secure_turtles --encrypted +``` + +The primary key and recovery key are **generated automatically** and displayed in the output: + +``` + ================================================================ + 🔐 ENCRYPTION KEYS — SAVE THESE NOW + ================================================================ + + KEY: dGhpcyBpcyBhIHNhbXBsZSBrZXkgZm9yIGRvY3M= + RECOVERY: cmVjb3Zlcnkga2V5IGZvciBkb2NzIHNhbXBsZQ== + + ================================================================ + + Store the key securely (secrets manager, environment variable). + Store the recovery key separately in a safe, offline location. + + These keys will NOT be shown again. If you lose both keys, + your data is unrecoverable. + ================================================================ +``` + +You can also provide your own keys via `--key` and `--recovery-key` (base64-encoded, 16, 24, or +32 bytes decoded): + +```sh +$ mix fact.create -p data/secure_turtles --encrypted \ + --key=$MY_KEY \ + --recovery-key=$MY_RECOVERY_KEY +``` + +Store the primary key securely (e.g., in a secrets manager or vault). Store the recovery key +in a separate, offline location (e.g., a printed key in a safe, a hardware security module). + +## Opening an Encrypted Database + +Supply the base64-encoded key via the `:encryption_key` option. The bootstrapper reads the +wrapped DEK from the `.bootstrap` file automatically — you only need to provide the key itself: + +```elixir +{:ok, db} = Fact.open("data/secure_turtles", + encryption_key: System.fetch_env!("FACT_KEY") +) +``` + +### In a Supervision Tree + +```elixir +# application.ex +def start(_type, _args) do + children = [ + {Fact.Supervisor, + databases: [ + {"data/secure_turtles", + encryption_key: System.fetch_env!("FACT_KEY")} + ]} + ] + + Supervisor.start_link(children, strategy: :one_for_one) +end +``` + +## KEK Providers + +For advanced use cases (vault integrations, per-database key resolution), Fact supports a provider +behaviour (`Fact.Encryption.KEKProvider`). The `:encryption_key` option uses the built-in +`Fact.Encryption.KEKProvider.Explicit` provider under the hood. + +Custom providers can integrate with external key management systems: + +```elixir +defmodule MyApp.VaultKEKProvider do + @behaviour Fact.Encryption.KEKProvider + + @impl true + def fetch_kek(opts) do + database_id = Keyword.fetch!(opts, :database_id) + {:ok, MyApp.Vault.get_key("fact/#{database_id}/kek")} + end +end +``` + +```elixir +Fact.open("data/secure_turtles", + encryption: [ + kek_provider: MyApp.VaultKEKProvider, + database_id: "my_db" + ] +) +``` + +## Recovery + +If the primary key is lost or compromised, use the `mix fact.recover` task with the recovery key. +This task generates a new DEK, a new primary key, and a new recovery key, then re-encrypts all +record files in place. + +```sh +$ mix fact.recover -p data/secure_turtles --recovery-key=$RECOVERY_KEY +``` + +The recovery task will: + +1. Unwrap the existing DEK using the recovery key +2. Decrypt all record files with the old DEK +3. Generate a new DEK, primary key, and recovery key +4. Re-encrypt all record files with the new DEK +5. Update the `.bootstrap` file with the new wrapped DEKs +6. Display the new keys + +The recovery key **cannot** be used directly with `Fact.open`. Recovery is an offline operation +that must be performed while the database is not running. This ensures the key rotation is atomic +and no reads or writes occur with stale keys. + +## File Format + +Each encrypted record file has the following binary layout: + +``` +┌──────────┬──────────────┬─────────────────┐ +│ Nonce │ Auth Tag │ Ciphertext │ +│ 12 bytes │ 16 bytes │ variable │ +└──────────┴──────────────┴─────────────────┘ +``` + +- **Nonce** (12 bytes) — A random initialization vector, unique per file. Generated using + `:crypto.strong_rand_bytes/1`. +- **Auth Tag** (16 bytes) — The GCM authentication tag. Verifies that the ciphertext has not been + tampered with. +- **Ciphertext** (variable) — The encrypted event record payload. + +Since record files are write-once (WORM), each file gets a unique nonce at write time. Nonce reuse +is impossible under normal operation. + +## What is Not Encrypted + +The following files remain in plaintext: + +| File Type | Contents | Why Not Encrypted | +|-----------|----------|-------------------| +| `.bootstrap` | Database config + wrapped DEK | Must be readable to bootstrap | +| `.ledger` | Record IDs (opaque identifiers) | Performance; no event payloads | +| Index files | Record IDs grouped by dimension | Performance; no event payloads | +| Checkpoint files | Integer positions | Operational metadata only | +| Lock files | Process metadata | Operational metadata only | + +Record IDs are opaque strings (UUIDs or content hashes) that do not reveal event content. The ledger +and index files enable efficient reads without decrypting every record on disk. + +## Security Considerations + +- **Key storage**: Never commit KEKs to source control or store them alongside the database. Use a + secrets manager, environment variables injected at deploy time, or a hardware security module. +- **Key rotation**: The DEK is permanent for the lifetime of the database. To rotate the KEK, unwrap + the DEK with the old KEK, re-wrap with a new KEK, and update the bootstrap file. +- **Backup encryption**: Database backups created with `mix fact.backup` include encrypted record files + as-is. The backup is only useful to someone who also has the KEK. +- **Memory**: The DEK is held in memory by the `Fact.KeyRing` process for the lifetime of the database. + It is not written to disk or logged. diff --git a/lib/fact.ex b/lib/fact.ex index 25da3c0..6fa7c60 100644 --- a/lib/fact.ex +++ b/lib/fact.ex @@ -628,6 +628,25 @@ defmodule Fact do Fact.Supervisor.start_database(path, opts) end + @doc """ + Closes a running database. + + Shuts down the database's supervision tree and releases its lock. After closing, + the database can be reopened with `open/2` or opened by another process. + + ## Examples + + iex> {:ok, db} = Fact.open("data/turtles") + iex> Fact.close(db) + :ok + + """ + @doc since: "0.3.1" + @spec close(database_id()) :: :ok | {:error, :not_found} + def close(database_id) when is_binary(database_id) do + Fact.Supervisor.stop_database(database_id) + end + @doc """ Read from an event source. diff --git a/lib/fact/bootstrap_file.ex b/lib/fact/bootstrap_file.ex index 6505245..8f568f3 100644 --- a/lib/fact/bootstrap_file.ex +++ b/lib/fact/bootstrap_file.ex @@ -186,19 +186,27 @@ defmodule Fact.BootstrapFile do It writes just enough configuration, so that the genesis record (`Fact.Genesis.Event.DatabaseCreated.V1`) can be read and a full `Fact.Context` can be loaded. """ - @doc since: "0.2.0" - @spec write(Path.t(), genesis_record :: Fact.record()) :: :ok | {:error, term()} - def write(path, {record_id, event}) do + @doc since: "0.3.1" + @spec write(Path.t(), genesis_record :: Fact.record(), keyword()) :: :ok | {:error, term()} + def write(path, {record_id, event}, opts \\ []) do this = get_bootstrap_write_context() - {:ok, bootstrap_record} = - Encoder.encode(this, %{ - record_id: record_id, - event_schema: event.event_schema, - record_file_decoder: event.record_file_decoder, - record_file_reader: event.record_file_reader, - storage: event.storage - }) + record = %{ + record_id: record_id, + event_schema: event.event_schema, + record_file_decoder: event.record_file_decoder, + record_file_reader: event.record_file_reader, + record_file_writer: event.record_file_writer, + storage: event.storage + } + + record = + case Keyword.get(opts, :encryption) do + nil -> record + encryption -> Map.put(record, :encryption, encryption) + end + + {:ok, bootstrap_record} = Encoder.encode(this, record) Writer.write(this, Path.join(path, Name.get(this)), bootstrap_record) end diff --git a/lib/fact/bootstrapper.ex b/lib/fact/bootstrapper.ex index d7597de..c29c443 100644 --- a/lib/fact/bootstrapper.ex +++ b/lib/fact/bootstrapper.ex @@ -71,14 +71,16 @@ defmodule Fact.Bootstrapper do @impl true @doc false def handle_continue(:bootstrap, %{path: path, caller: caller, opts: opts} = state) do - with {:ok, context} <- load_context(path) do + with {:ok, context, encryption_runtime_opts} <- load_context(path, opts) do + merged_opts = Keyword.merge(opts, encryption_runtime_opts) + case Fact.Registry.get_context(context.database_id) do {:ok, _} -> maybe_send(caller, {:database_started, context.database_id}) {:stop, :normal, state} {:error, _} -> - case start_database(context, opts) do + case start_database(context, merged_opts) do {:ok, _pid} -> maybe_send(caller, {:database_started, context.database_id}) {:stop, :normal, state} @@ -111,31 +113,117 @@ defmodule Fact.Bootstrapper do end end - defp load_context_from_bootstrap_file(path) do - with {:ok, bootstrap} <- Fact.BootstrapFile.read(path) do + defp load_context_from_bootstrap_file(path, opts) do + with {:ok, bootstrap} <- Fact.BootstrapFile.read(path), + {:ok, dek, encryption_runtime_opts} <- resolve_encryption(bootstrap, opts) do bootstrap_context = %Fact.Context{ event_schema: Fact.Event.Schema.from_config(bootstrap["event_schema"]), record_file_decoder: Fact.RecordFile.Decoder.from_config(bootstrap["record_file_decoder"]), - record_file_reader: Fact.RecordFile.Reader.from_config(bootstrap["record_file_reader"]), + record_file_reader: + Fact.RecordFile.Reader.from_config( + bootstrap["record_file_reader"] || + %{"family" => "full", "version" => 1, "options" => %{}} + ), storage: Fact.Storage.from_config(bootstrap["storage"]) } - {_, event} = Fact.RecordFile.read(bootstrap_context, bootstrap["record_id"]) + {_, event} = read_genesis_record(bootstrap_context, bootstrap["record_id"], dek) event_data = Map.get(event, Fact.Event.Schema.get(bootstrap_context).event_data) + context = Fact.Context.from_genesis_event_data(event_data) + + {:ok, context, encryption_runtime_opts} + end + end + + defp resolve_encryption(bootstrap, opts) do + case bootstrap["encryption"] do + nil -> + {:ok, nil, []} + + encryption_config -> + with {:ok, kek} <- fetch_kek(opts), + {:ok, wrapped_dek} <- get_wrapped_dek(encryption_config), + {:ok, dek} <- unwrap_dek(wrapped_dek, kek) do + {:ok, dek, [encryption: [wrapped_dek: wrapped_dek, kek: kek]]} + end + end + end + + # When encrypted, read raw file with Full.V1 and decrypt before decoding. + defp read_genesis_record(context, record_id, dek) when is_binary(dek) do + {:ok, record_path} = {:ok, Fact.Storage.records_path(context, record_id)} + + {:ok, stream} = + Fact.Seam.FileReader.Full.V1.read(%Fact.Seam.FileReader.Full.V1{}, record_path, []) + + encrypted_binary = Enum.at(stream, 0) + {:ok, plaintext} = Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(dek, encrypted_binary) + {:ok, record} = Fact.RecordFile.Decoder.decode(context, plaintext) + {record_id, record} + end + + defp read_genesis_record(context, record_id, nil) do + Fact.RecordFile.read(context, record_id) + end + + defp fetch_kek(opts) do + case Keyword.get(opts, :encryption_key) do + value when is_binary(value) -> + decode_encryption_key(value) - {:ok, Fact.Context.from_genesis_event_data(event_data)} + nil -> + encryption_opts = Keyword.get(opts, :encryption, []) + + case Keyword.get(encryption_opts, :kek_provider) do + nil -> + {:error, :encryption_key_required} + + provider -> + provider.fetch_kek(encryption_opts) + end + end + end + + defp decode_encryption_key(value) do + case Base.decode64(value) do + {:ok, key} when byte_size(key) in [16, 24, 32] -> + {:ok, key} + + {:ok, key} -> + {:error, {:invalid_encryption_key_size, byte_size(key)}} + + :error -> + {:error, {:invalid_encryption_key, "must be valid base64"}} + end + end + + defp unwrap_dek(wrapped_dek, kek) do + case Fact.KeyRing.unwrap_dek(wrapped_dek, kek) do + {:ok, dek} -> {:ok, dek} + {:error, :dek_unwrap_failed} -> {:error, :invalid_encryption_key} + error -> error end end - defp load_context(path) do - case load_context_from_bootstrap_file(path) do + defp get_wrapped_dek(encryption_config) do + {:ok, Base.decode64!(encryption_config["wrapped_dek_primary"])} + end + + defp load_context(path, opts) do + case load_context_from_bootstrap_file(path, opts) do {:error, :bootstraps_not_found} -> - load_context_v1(path) + case load_context_v1(path) do + {:ok, context} -> {:ok, context, []} + error -> error + end - {:ok, _context} = result -> + {:ok, _context, _encryption_opts} = result -> result + + {:error, _reason} = error -> + error end end diff --git a/lib/fact/database_supervisor.ex b/lib/fact/database_supervisor.ex index 731fec0..7083fa9 100644 --- a/lib/fact/database_supervisor.ex +++ b/lib/fact/database_supervisor.ex @@ -98,35 +98,41 @@ defmodule Fact.DatabaseSupervisor do Keyword.get(opts, :wal, []) cache_opts = Keyword.get(opts, :cache, []) + key_ring_children = key_ring_children(database_id, opts) - children = [ - {Registry, keys: :unique, name: Fact.Registry.registry(database_id)}, - {Phoenix.PubSub, name: Fact.Registry.pubsub(database_id)}, - {Fact.EventPublisher, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventPublisher)}, - {Fact.Database, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.Database)}, - {Fact.WriteAheadLog, wal_opts}, - {Fact.EventLedger, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventLedger)}, - {Fact.EventStreamIndexer, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventStreamIndexer)}, - {Fact.EventTagsIndexer, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventTagsIndexer)}, - {Fact.EventTypeIndexer, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventTypeIndexer)}, - {Fact.EventStreamCategoryIndexer, - database_id: database_id, - name: Fact.Registry.via(database_id, Fact.EventStreamCategoryIndexer)}, - {Fact.EventStreamsIndexer, - database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventStreamsIndexer)}, - {Fact.EventStreamsByCategoryIndexer, - database_id: database_id, - name: Fact.Registry.via(database_id, Fact.EventStreamsByCategoryIndexer)}, - {DynamicSupervisor, - strategy: :one_for_one, - name: Fact.Registry.via(database_id, Fact.EventStreamWriterSupervisor)} - ] + children = + [ + {Registry, keys: :unique, name: Fact.Registry.registry(database_id)} + ] ++ + key_ring_children ++ + [ + {Phoenix.PubSub, name: Fact.Registry.pubsub(database_id)}, + {Fact.EventPublisher, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventPublisher)}, + {Fact.Database, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.Database)}, + {Fact.WriteAheadLog, wal_opts}, + {Fact.EventLedger, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventLedger)}, + {Fact.EventStreamIndexer, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventStreamIndexer)}, + {Fact.EventTagsIndexer, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventTagsIndexer)}, + {Fact.EventTypeIndexer, + database_id: database_id, name: Fact.Registry.via(database_id, Fact.EventTypeIndexer)}, + {Fact.EventStreamCategoryIndexer, + database_id: database_id, + name: Fact.Registry.via(database_id, Fact.EventStreamCategoryIndexer)}, + {Fact.EventStreamsIndexer, + database_id: database_id, + name: Fact.Registry.via(database_id, Fact.EventStreamsIndexer)}, + {Fact.EventStreamsByCategoryIndexer, + database_id: database_id, + name: Fact.Registry.via(database_id, Fact.EventStreamsByCategoryIndexer)}, + {DynamicSupervisor, + strategy: :one_for_one, + name: Fact.Registry.via(database_id, Fact.EventStreamWriterSupervisor)} + ] cache_children = case Keyword.get(cache_opts, :max_size) do @@ -149,6 +155,26 @@ defmodule Fact.DatabaseSupervisor do Supervisor.init(children ++ cache_children ++ merkle_children, strategy: :one_for_one) end + defp key_ring_children(database_id, opts) do + encryption_opts = Keyword.get(opts, :encryption, []) + + case Keyword.get(encryption_opts, :wrapped_dek) do + nil -> + [] + + wrapped_dek -> + kek = Keyword.fetch!(encryption_opts, :kek) + + [ + {Fact.KeyRing, + database_id: database_id, + wrapped_dek: wrapped_dek, + kek: kek, + name: Fact.Registry.via(database_id, Fact.KeyRing)} + ] + end + end + defp merkle_children(context, database_id, opts) do merkle_opts = merkle_opts(opts) diff --git a/lib/fact/encryption/kek_provider.ex b/lib/fact/encryption/kek_provider.ex new file mode 100644 index 0000000..7d04a64 --- /dev/null +++ b/lib/fact/encryption/kek_provider.ex @@ -0,0 +1,36 @@ +defmodule Fact.Encryption.KEKProvider do + @moduledoc """ + Behaviour for providing Key Encryption Keys (KEKs) at runtime. + + A KEK provider is responsible for supplying the key used to unwrap the + Data Encryption Key (DEK) when a database is opened. The DEK is stored + in the `.bootstrap` file in wrapped (encrypted) form and must be unwrapped + before the database can read or write encrypted record files. + + ## Implementations + + * `Fact.Encryption.KEKProvider.Explicit` — takes the KEK directly from options. + """ + @moduledoc since: "0.4.0" + + @typedoc """ + A module implementing the `Fact.Encryption.KEKProvider` behaviour. + """ + @typedoc since: "0.4.0" + @type t :: module() + + @typedoc """ + Options passed to `c:fetch_kek/1`. The shape depends on the provider implementation. + """ + @typedoc since: "0.4.0" + @type opts :: keyword() + + @doc """ + Fetches the Key Encryption Key. + + Returns `{:ok, kek}` where `kek` is a binary encryption key, or + `{:error, reason}` if the key cannot be retrieved. + """ + @doc since: "0.4.0" + @callback fetch_kek(opts()) :: {:ok, binary()} | {:error, term()} +end diff --git a/lib/fact/encryption/kek_provider/explicit.ex b/lib/fact/encryption/kek_provider/explicit.ex new file mode 100644 index 0000000..2e72ba8 --- /dev/null +++ b/lib/fact/encryption/kek_provider/explicit.ex @@ -0,0 +1,46 @@ +defmodule Fact.Encryption.KEKProvider.Explicit do + @moduledoc """ + A `Fact.Encryption.KEKProvider` that takes the KEK directly from the options. + + This is the simplest provider — the caller supplies the key material when opening + the database. Key sourcing (environment variables, vault clients, config files) is + the caller's responsibility. + + ## Examples + + Fact.open("data/turtles", + encryption: [ + kek_provider: Fact.Encryption.KEKProvider.Explicit, + kek: System.fetch_env!("FACT_KEK") |> Base.decode64!() + ] + ) + """ + @moduledoc since: "0.4.0" + + @behaviour Fact.Encryption.KEKProvider + + @doc """ + Fetches the KEK from the `:kek` option. + + Expects `opts` to contain a `:kek` key with a binary value of 16, 24, or 32 bytes + (corresponding to AES-128, AES-192, or AES-256). + """ + @doc since: "0.4.0" + @impl true + @spec fetch_kek(keyword()) :: {:ok, binary()} | {:error, term()} + def fetch_kek(opts) do + case Keyword.fetch(opts, :kek) do + {:ok, kek} when is_binary(kek) and byte_size(kek) in [16, 24, 32] -> + {:ok, kek} + + {:ok, kek} when is_binary(kek) -> + {:error, {:invalid_kek_size, byte_size(kek)}} + + {:ok, _} -> + {:error, :kek_must_be_binary} + + :error -> + {:error, :kek_not_provided} + end + end +end diff --git a/lib/fact/genesis/decider.ex b/lib/fact/genesis/decider.ex index 0a66062..e7a7385 100644 --- a/lib/fact/genesis/decider.ex +++ b/lib/fact/genesis/decider.ex @@ -25,7 +25,8 @@ defmodule Fact.Genesis.Decider do def decide(:initial_state, %CreateDatabase.V1{args: args} = _command) do with args <- resolve_name_and_path(args), :ok <- verify_path(Keyword.get(args, :path)), - {:ok, configuration} <- build_configuration(args) do + {:ok, configuration} <- build_configuration(args), + {:ok, encryption_meta} <- resolve_encryption(args, configuration) do info = %{ database_id: generate_database_id(), database_name: Keyword.get(args, :name), @@ -36,7 +37,8 @@ defmodule Fact.Genesis.Decider do otp_version: otp_version() } - {:ok, [struct(DatabaseCreated.V1, Map.merge(info, configuration))]} + event = struct(DatabaseCreated.V1, Map.merge(info, configuration)) + {:ok, [event], encryption_meta} end end @@ -271,6 +273,39 @@ defmodule Fact.Genesis.Decider do end end + defp resolve_encryption(args, %{record_file_writer: %{family: :encrypted}} = _config) do + with {:ok, kek} <- resolve_key(Keyword.get(args, :key)), + {:ok, recovery_kek} <- resolve_key(Keyword.get(args, :recovery_key)) do + dek = Fact.KeyRing.generate_dek() + + {:ok, + %{ + dek: dek, + kek: kek, + recovery_kek: recovery_kek, + wrapped_dek_primary: Fact.KeyRing.wrap_dek(dek, kek), + wrapped_dek_recovery: Fact.KeyRing.wrap_dek(dek, recovery_kek) + }} + end + end + + defp resolve_encryption(_args, _config), do: {:ok, nil} + + defp resolve_key(nil), do: {:ok, :crypto.strong_rand_bytes(32)} + + defp resolve_key(value) when is_binary(value) do + case Base.decode64(value) do + {:ok, key} when byte_size(key) in [16, 24, 32] -> + {:ok, key} + + {:ok, key} -> + {:error, {:invalid_key_size, byte_size(key)}} + + :error -> + {:error, {:invalid_key, "must be valid base64"}} + end + end + @hash_algorithm_bit_length %{ md5: 128, sha: 160, diff --git a/lib/fact/genesis/the_creator.ex b/lib/fact/genesis/the_creator.ex index a5f9b89..17b7600 100644 --- a/lib/fact/genesis/the_creator.ex +++ b/lib/fact/genesis/the_creator.ex @@ -21,10 +21,14 @@ defmodule Fact.Genesis.TheCreator do Creates a Fact database from the `Fact.Genesis.Event.DatabaseCreated.V1` event. It initializes storage, and appends the event as the first record of every database. + + When `encryption_meta` is provided, the genesis record is encrypted manually + (since the KeyRing process is not yet running) and the bootstrap file includes + the wrapped DEK for later decryption. """ - @doc since: "0.1.0" - @spec let_there_be_light(%DatabaseCreated.V1{}) :: :ok - def let_there_be_light(%DatabaseCreated.V1{} = event) do + @doc since: "0.3.1" + @spec let_there_be_light(%DatabaseCreated.V1{}, map() | nil) :: :ok + def let_there_be_light(%DatabaseCreated.V1{} = event, encryption_meta \\ nil) do with context <- DatabaseCreated.V1.to_context(event), {:ok, path} <- Storage.initialize_storage(context) do schema = Event.Schema.get(context) @@ -40,14 +44,54 @@ defmodule Fact.Genesis.TheCreator do schema.event_tags => ["__fact__:#{event.database_id}"] } - {:ok, record_id} = Fact.RecordFile.write(context, genesis) + {:ok, record_id} = write_genesis_record(context, genesis, encryption_meta) {:ok, ^record_id} = Fact.LedgerFile.write(context, record_id) + bootstrap_opts = build_bootstrap_opts(encryption_meta) + # writes the bare minimum into a special .bootstrap file - # with just enough to read the genesis event and start the system. - :ok = Fact.BootstrapFile.write(path, {record_id, event}) + # with just enough to read the genesis event and start the system. + :ok = Fact.BootstrapFile.write(path, {record_id, event}, bootstrap_opts) :ok end end + + # When encrypted, we encode the genesis record normally, then encrypt and write + # it manually. The KeyRing process doesn't exist yet during creation, so we + # can't use the encrypted FileWriter seam. + defp write_genesis_record(context, genesis, %{dek: dek} = _meta) do + with {:ok, encoded} <- Fact.RecordFile.Encoder.encode(context, genesis), + {:ok, record_id} <- Fact.RecordFile.Name.get(context, {genesis, encoded}), + {:ok, record_path} <- {:ok, Storage.records_path(context, record_id)} do + nonce = :crypto.strong_rand_bytes(12) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, encoded, "", true) + + encrypted = <> + + :ok = File.write!(record_path, encrypted) + File.chmod!(record_path, 0o444) + + {:ok, record_id} + end + end + + defp write_genesis_record(context, genesis, nil) do + Fact.RecordFile.write(context, genesis) + end + + defp build_bootstrap_opts(nil), do: [] + + defp build_bootstrap_opts(%{wrapped_dek_primary: primary, wrapped_dek_recovery: recovery}) do + [ + encryption: %{ + family: :aes_gcm, + version: 1, + wrapped_dek_primary: Base.encode64(primary), + wrapped_dek_recovery: Base.encode64(recovery) + } + ] + end end diff --git a/lib/fact/index_checkpoint_file.ex b/lib/fact/index_checkpoint_file.ex index 4562f7b..42b5abd 100644 --- a/lib/fact/index_checkpoint_file.ex +++ b/lib/fact/index_checkpoint_file.ex @@ -68,6 +68,7 @@ defmodule Fact.IndexCheckpointFile do """ use Fact.Seam.FileWriter.Adapter, context: :index_checkpoint_file_writer, + allowed_impls: [{:standard, 1}], fixed_options: %{ {:standard, 1} => %{ access: :write, diff --git a/lib/fact/index_file.ex b/lib/fact/index_file.ex index eb2c483..482c8e9 100644 --- a/lib/fact/index_file.ex +++ b/lib/fact/index_file.ex @@ -71,6 +71,7 @@ defmodule Fact.IndexFile do """ use Fact.Seam.FileWriter.Adapter, context: :index_file_writer, + allowed_impls: [{:standard, 1}], fixed_options: %{ {:standard, 1} => %{ access: :append, diff --git a/lib/fact/key_ring.ex b/lib/fact/key_ring.ex new file mode 100644 index 0000000..4bab4c2 --- /dev/null +++ b/lib/fact/key_ring.ex @@ -0,0 +1,111 @@ +defmodule Fact.KeyRing do + @moduledoc """ + Holds the Data Encryption Key (DEK) in memory for a database instance. + + The `Fact.KeyRing` is started as part of the `Fact.DatabaseSupervisor` when the database + is configured with encryption. At startup, it unwraps the DEK using the configured + `Fact.Encryption.KEKProvider` and holds it in memory for the lifetime of the database. + + The encrypted `Fact.Seam.FileWriter.Encrypted.V1` and `Fact.Seam.FileReader.Encrypted.V1` + retrieve the DEK via `get_dek/1`. + """ + @moduledoc since: "0.4.0" + + use GenServer + + @nonce_size 12 + @auth_tag_size 16 + + @typedoc since: "0.4.0" + @type option :: + {:database_id, Fact.database_id()} + | {:wrapped_dek, binary()} + | {:kek, binary()} + | {:name, GenServer.name()} + + @doc """ + Retrieves the DEK for the given database. + """ + @doc since: "0.4.0" + @spec get_dek(Fact.database_id()) :: {:ok, binary()} | {:error, term()} + def get_dek(database_id) do + GenServer.call(Fact.Registry.via(database_id, __MODULE__), :get_dek) + end + + @doc false + @spec child_spec([option()]) :: Supervisor.child_spec() + def child_spec(opts) do + database_id = Keyword.fetch!(opts, :database_id) + + %{ + id: {__MODULE__, database_id}, + start: {__MODULE__, :start_link, [opts]} + } + end + + @doc false + @spec start_link([option()]) :: GenServer.on_start() + def start_link(opts) do + {start_opts, init_opts} = Keyword.split(opts, [:name]) + GenServer.start_link(__MODULE__, init_opts, start_opts) + end + + @impl true + def init(opts) do + wrapped_dek = Keyword.fetch!(opts, :wrapped_dek) + kek = Keyword.fetch!(opts, :kek) + + case unwrap_dek(wrapped_dek, kek) do + {:ok, dek} -> + {:ok, %{dek: dek}} + + {:error, reason} -> + {:stop, reason} + end + end + + @impl true + def handle_call(:get_dek, _from, %{dek: dek} = state) do + {:reply, {:ok, dek}, state} + end + + @doc """ + Wraps a DEK with a KEK using AES-256-GCM. + + Returns the wrapped DEK as `<>`. + """ + @doc since: "0.4.0" + @spec wrap_dek(dek :: binary(), kek :: binary()) :: binary() + def wrap_dek(dek, kek) do + nonce = :crypto.strong_rand_bytes(@nonce_size) + {ciphertext, auth_tag} = :crypto.crypto_one_time_aead(:aes_256_gcm, kek, nonce, dek, "", true) + <> + end + + @doc """ + Unwraps a DEK using a KEK. + + Expects the wrapped DEK format: `<>`. + """ + @doc since: "0.4.0" + @spec unwrap_dek(wrapped_dek :: binary(), kek :: binary()) :: {:ok, binary()} | {:error, term()} + def unwrap_dek( + <>, + kek + ) do + case :crypto.crypto_one_time_aead(:aes_256_gcm, kek, nonce, ciphertext, "", auth_tag, false) do + dek when is_binary(dek) -> {:ok, dek} + :error -> {:error, :dek_unwrap_failed} + end + end + + def unwrap_dek(_wrapped_dek, _kek), do: {:error, :invalid_wrapped_dek} + + @doc """ + Generates a random 256-bit DEK. + """ + @doc since: "0.4.0" + @spec generate_dek() :: binary() + def generate_dek, do: :crypto.strong_rand_bytes(32) +end diff --git a/lib/fact/ledger_file.ex b/lib/fact/ledger_file.ex index 54c868a..8f31b7e 100644 --- a/lib/fact/ledger_file.ex +++ b/lib/fact/ledger_file.ex @@ -69,6 +69,7 @@ defmodule Fact.LedgerFile do """ use Fact.Seam.FileWriter.Adapter, context: :ledger_file_writer, + allowed_impls: [{:standard, 1}], fixed_options: %{ {:standard, 1} => %{ access: :append, diff --git a/lib/fact/lock_file.ex b/lib/fact/lock_file.ex index 71fc485..e8db44d 100644 --- a/lib/fact/lock_file.ex +++ b/lib/fact/lock_file.ex @@ -70,6 +70,7 @@ defmodule Fact.LockFile do """ use Fact.Seam.FileWriter.Adapter, context: :lock_file_writer, + allowed_impls: [{:standard, 1}], fixed_options: %{ {:standard, 1} => %{ access: :write, diff --git a/lib/fact/record_file.ex b/lib/fact/record_file.ex index 170405f..039657c 100644 --- a/lib/fact/record_file.ex +++ b/lib/fact/record_file.ex @@ -76,7 +76,8 @@ defmodule Fact.RecordFile do """ use Fact.Seam.FileReader.Adapter, context: :record_file_reader, - allowed_impls: [{:full, 1}] + allowed_impls: [{:encrypted, 1}, {:full, 1}], + default_impl: {:full, 1} end defmodule Writer do @@ -91,6 +92,8 @@ defmodule Fact.RecordFile do """ use Fact.Seam.FileWriter.Adapter, context: :record_file_writer, + allowed_impls: [{:encrypted, 1}, {:standard, 1}], + default_impl: {:standard, 1}, fixed_options: %{ {:standard, 1} => %{ access: :write, @@ -99,6 +102,14 @@ defmodule Fact.RecordFile do raw: true, sync: false, worm: true + }, + {:encrypted, 1} => %{ + access: :write, + binary: true, + exclusive: true, + raw: false, + sync: false, + worm: true } } end diff --git a/lib/fact/seam/file_reader/encrypted/v1.ex b/lib/fact/seam/file_reader/encrypted/v1.ex new file mode 100644 index 0000000..8611ba8 --- /dev/null +++ b/lib/fact/seam/file_reader/encrypted/v1.ex @@ -0,0 +1,89 @@ +defmodule Fact.Seam.FileReader.Encrypted.V1 do + @moduledoc """ + Encrypted V1 implementation of `Fact.Seam.FileReader`. + + Reads a file written by `Fact.Seam.FileWriter.Encrypted.V1`, parsing the header + to extract the nonce and authentication tag, then decrypting the ciphertext. + + The expected file format is: + + <> + + The Data Encryption Key (DEK) is retrieved from `Fact.KeyRing` at read time. + + ## Options + + * `:cipher` - The cipher algorithm. Currently only `:aes_gcm` is supported. + Defaults to `:aes_gcm`. + """ + @moduledoc since: "0.4.0" + + use Fact.Seam.FileReader, + family: :encrypted, + version: 1 + + import Fact.Seam.Parsers, only: [parse_existing_atom: 1] + + @nonce_size 12 + @auth_tag_size 16 + @header_size @nonce_size + @auth_tag_size + + @enforce_keys [:cipher] + defstruct [:cipher] + + @impl true + def default_options(), do: %{cipher: :aes_gcm} + + @impl true + def option_specs() do + %{ + cipher: %{ + allowed: [:aes_gcm], + parse: &parse_existing_atom/1, + error: :invalid_cipher_option + } + } + end + + @impl true + def prepare_options(options), do: Map.take(options, [:cipher]) + + @impl true + def read(%__MODULE__{cipher: :aes_gcm}, path, options) do + context = Keyword.get(options, :__context__) + database_id = context.database_id + + with {:ok, dek} <- Fact.KeyRing.get_dek(database_id), + {:ok, binary} <- File.read(path), + {:ok, plaintext} <- decrypt_binary(dek, binary) do + {:ok, Stream.map([plaintext], & &1)} + end + end + + @doc """ + Decrypts a binary that was written by `Fact.Seam.FileWriter.Encrypted.V1`. + + This is used by the bootstrapper to decrypt the genesis record before the + `Fact.KeyRing` process is started. + """ + @doc since: "0.4.0" + @spec decrypt_binary(dek :: binary(), encrypted :: binary()) :: + {:ok, binary()} | {:error, term()} + def decrypt_binary( + dek, + <> + ) do + case :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, ciphertext, "", auth_tag, false) do + plaintext when is_binary(plaintext) -> + {:ok, plaintext} + + :error -> + {:error, :decryption_failed} + end + end + + def decrypt_binary(_dek, binary) when byte_size(binary) < @header_size do + {:error, :invalid_encrypted_file} + end +end diff --git a/lib/fact/seam/file_reader/registry.ex b/lib/fact/seam/file_reader/registry.ex index 325739e..8ae70c0 100644 --- a/lib/fact/seam/file_reader/registry.ex +++ b/lib/fact/seam/file_reader/registry.ex @@ -6,6 +6,7 @@ defmodule Fact.Seam.FileReader.Registry do """ use Fact.Seam.Registry, impls: [ + Fact.Seam.FileReader.Encrypted.V1, Fact.Seam.FileReader.Full.V1, Fact.Seam.FileReader.FixedLength.V1 ] diff --git a/lib/fact/seam/file_writer/encrypted/v1.ex b/lib/fact/seam/file_writer/encrypted/v1.ex new file mode 100644 index 0000000..fe1c09c --- /dev/null +++ b/lib/fact/seam/file_writer/encrypted/v1.ex @@ -0,0 +1,137 @@ +defmodule Fact.Seam.FileWriter.Encrypted.V1 do + @moduledoc """ + Encrypted V1 implementation of `Fact.Seam.FileWriter`. + + Encrypts the value before writing it to disk. Each file is written with a header + containing a random nonce and authentication tag, followed by the ciphertext. + + The file format is: + + <> + + The Data Encryption Key (DEK) is retrieved from `Fact.KeyRing` at write time. + + ## Options + + * `:cipher` - The cipher algorithm. Currently only `:aes_gcm` is supported. + Defaults to `:aes_gcm`. + * `:access` - File access mode (`:write` or `:append`). Defaults to `:write`. + * `:binary` - Whether to open in binary mode. Defaults to `true`. + * `:exclusive` - Fail if the file already exists. Defaults to `true`. + * `:raw` - Use raw file descriptors. Defaults to `false`. + * `:sync` - Fsync after write. Defaults to `false`. + * `:worm` - Mark file read-only after write. Defaults to `false`. + """ + @moduledoc since: "0.4.0" + + use Fact.Seam.FileWriter, + family: :encrypted, + version: 1 + + import Fact.Seam.Parsers, only: [parse_existing_atom: 1] + + @nonce_size 12 + @auth_tag_size 16 + + @enforce_keys [:cipher, :modes, :sync, :worm] + defstruct [:cipher, :modes, :sync, :worm] + + @impl true + def default_options(), + do: %{ + cipher: :aes_gcm, + access: :write, + binary: true, + exclusive: true, + raw: false, + sync: false, + worm: false + } + + @impl true + def option_specs() do + %{ + cipher: %{ + allowed: [:aes_gcm], + parse: &parse_existing_atom/1, + error: :invalid_cipher_option + }, + access: %{ + allowed: [:write, :append], + parse: &parse_existing_atom/1, + error: :invalid_access_option + }, + binary: %{ + allowed: [true, false], + parse: &parse_existing_atom/1, + error: :invalid_binary_option + }, + exclusive: %{ + allowed: [true, false], + parse: &parse_existing_atom/1, + error: :invalid_exclusive_option + }, + raw: %{ + allowed: [true, false], + parse: &parse_existing_atom/1, + error: :invalid_raw_option + }, + sync: %{ + allowed: [true, false], + parse: &parse_existing_atom/1, + error: :invalid_sync_option + }, + worm: %{ + allowed: [true, false], + parse: &parse_existing_atom/1, + error: :invalid_worm_option + } + } + end + + @impl true + def prepare_options(options) do + modes = + [ + options.access, + if(options.binary, do: :binary, else: nil), + if(options.exclusive, do: :exclusive, else: nil), + if(options.raw, do: :raw, else: nil) + ] + |> Enum.reject(&is_nil/1) + + options + |> Map.take([:cipher, :sync, :worm]) + |> Map.put(:modes, modes) + end + + @impl true + def write( + %__MODULE__{cipher: :aes_gcm, modes: modes, sync: sync, worm: worm}, + path, + value, + options + ) do + context = Keyword.get(options, :__context__) + database_id = context.database_id + + with {:ok, dek} <- Fact.KeyRing.get_dek(database_id) do + nonce = :crypto.strong_rand_bytes(@nonce_size) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, value, "", true) + + encrypted_value = + <> + + with {:ok, fd} <- File.open(path, modes), + :ok <- IO.binwrite(fd, encrypted_value), + :ok <- if(sync, do: :file.sync(fd), else: :ok), + :ok <- File.close(fd), + :ok <- if(worm, do: File.chmod(path, 0o444), else: :ok) do + :ok + end + end + end +end diff --git a/lib/fact/seam/file_writer/registry.ex b/lib/fact/seam/file_writer/registry.ex index a5cac36..0a90416 100644 --- a/lib/fact/seam/file_writer/registry.ex +++ b/lib/fact/seam/file_writer/registry.ex @@ -5,5 +5,5 @@ defmodule Fact.Seam.FileWriter.Registry do Provides a lookup for allowed implementations, their versions, and the latest default implementation. """ use Fact.Seam.Registry, - impls: [Fact.Seam.FileWriter.Standard.V1] + impls: [Fact.Seam.FileWriter.Encrypted.V1, Fact.Seam.FileWriter.Standard.V1] end diff --git a/lib/fact/supervisor.ex b/lib/fact/supervisor.ex index 6d94f6c..d6ec15d 100644 --- a/lib/fact/supervisor.ex +++ b/lib/fact/supervisor.ex @@ -86,6 +86,30 @@ defmodule Fact.Supervisor do end end + @doc """ + Stops a running database. + + This terminates the `Fact.DatabaseSupervisor` for the given database and removes it from the + supervision tree. The database lock is released as part of the shutdown. + + Returns `:ok` on success, or `{:error, :not_found}` if no database with the given identifier + is running. + """ + @doc since: "0.3.1" + @spec stop_database(Fact.database_id()) :: :ok | {:error, :not_found} + def stop_database(database_id) when is_binary(database_id) do + child_id = {Fact.DatabaseSupervisor, database_id} + + case Supervisor.terminate_child(__MODULE__, child_id) do + :ok -> + Supervisor.delete_child(__MODULE__, child_id) + :ok + + {:error, :not_found} -> + {:error, :not_found} + end + end + @doc """ Starts the `Fact.Supervisor`. diff --git a/lib/mix/tasks/fact/create.ex b/lib/mix/tasks/fact/create.ex index d895dd1..0877aed 100644 --- a/lib/mix/tasks/fact/create.ex +++ b/lib/mix/tasks/fact/create.ex @@ -3,15 +3,15 @@ defmodule Mix.Tasks.Fact.Create do Creates a new database. ### Usage - + ### Options - + There are a lot of options to control how the database stores information. However many of these options currently only support a single value. Each represents a **seam** within the system, and each can be supplied with a set of implementation specific options. Each seam implementation is configured using `{family}@{version}` format, while options are specified using a comma-delimited list of key value pairs `{key1}={value1},{key2}={value2}`. - + Maybe this all just a big old YAGNI...but I think it will prove its worth in time. #### Event options @@ -22,30 +22,30 @@ defmodule Mix.Tasks.Fact.Create do * `--event-schema-options` - Configuration options for the selected event schema. (default: `""`) #### Index file options - - * `--index-file-decoder` - Controls how record are decoded after reading. (default: `raw@1`) + + * `--index-file-decoder` - Controls how record are decoded after reading. (default: `raw@1`) * `--index-file-decoder-options` - Configuration options for the selected record file decoder. (default: `""`) * `--index-file-encoder` - Controls how record are encoded for writing. (default: `delimited@1`) * `--index-file-encoder-options` - Configuration options for the selected record file encoder. (default: `""`) * `--index-file-name` - Controls how record files are named. (default: `raw@1`) - * `--index-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) + * `--index-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) * `--index-file-reader` - Controls how record files are read. (default: `fixed_length@1`) * `--index-file-reader-options` - Configuration options for the selected record file reader. (default: `""`) * `--index-file-writer` - Controls how record files are written. (default: `standard@1`) * `--index-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) - - #### Index checkpoint file options + + #### Index checkpoint file options * `--index-checkpoint-file-decoder` - Controls how record are decoded after reading. (default: `integer@1`) * `--index-checkpoint-file-decoder-options` - Configuration options for the selected record file decoder. (default: `""`) * `--index-checkpoint-file-encoder` - Controls how record are encoded for writing. (default: `integer@1`) * `--index-checkpoint-file-encoder-options` - Configuration options for the selected record file encoder. (default: `""`) * `--index-checkpoint-file-name` - Controls how record files are named. (default: `fixed@1`) - * `--index-checkpoint-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) + * `--index-checkpoint-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) * `--index-checkpoint-file-reader` - Controls how record files are read. (default: `full@1`) * `--index-checkpoint-file-reader-options` - Configuration options for the selected record file reader. (default: `""`) * `--index-checkpoint-file-writer` - Controls how record files are written. (default: `standard@1`) - * `--index-checkpoint-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) + * `--index-checkpoint-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) #### Ledger file options @@ -54,7 +54,7 @@ defmodule Mix.Tasks.Fact.Create do * `--ledger-file-encoder` - Controls how record are encoded for writing. (default: `delimited@1`) * `--ledger-file-encoder-options` - Configuration options for the selected record file encoder. (default: `""`) * `--ledger-file-name` - Controls how record files are named. (default: `fixed@1`) - * `--ledger-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) + * `--ledger-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) * `--ledger-file-reader` - Controls how record files are read. (default: `fixed_length@1`) * `--ledger-file-reader-options` - Configuration options for the selected record file reader. (default: `""`) * `--ledger-file-writer` - Controls how record files are written. (default: `standard@1`) @@ -67,30 +67,37 @@ defmodule Mix.Tasks.Fact.Create do * `--lock-file-encoder` - Controls how record are encoded for writing. (default: `json@1`) * `--lock-file-encoder-options` - Configuration options for the selected record file encoder. (default: `""`) * `--lock-file-name` - Controls how record files are named. (default: `fixed@1`) - * `--lock-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) + * `--lock-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) * `--lock-file-reader` - Controls how record files are read. (default: `full@1`) * `--lock-file-reader-options` - Configuration options for the selected record file reader. (default: `""`) * `--lock-file-writer` - Controls how record files are written. (default: `standard@1`) - * `--lock-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) + * `--lock-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) #### Record file options - + * `--record-file-decoder` - Controls how record are decoded after reading. (default: `json@1`) * `--record-file-decoder-options` - Configuration options for the selected record file decoder. (default: `""`) * `--record-file-encoder` - Controls how record are encoded for writing. (default: `json@1`) * `--record-file-encoder-options` - Configuration options for the selected record file encoder. (default: `""`) * `--record-file-name` - Controls how record files are named. (default: `event_id@1`) - * `--record-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) + * `--record-file-name-options` - Configuration options for the selected record file name seam. (default: `""`) * `--record-file-reader` - Controls how record files are read. (default: `full@1`) * `--record-file-reader-options` - Configuration options for the selected record file reader. (default: `""`) * `--record-file-writer` - Controls how record files are written. (default: `standard@1`) * `--record-file-writer-options` - Configuration options for the selected record file writer. (default: `""`) - + #### Storage options - + * `--storage` - Controls how the files are organized within the file system. (default: `standard@1`) * `--storage-options` - Configuration options for the selected storage. (default: `""`) + #### Encryption options + + * `--encrypted` - Enables encryption at rest for event record files. This is a shorthand that + sets the record file writer and reader to `encrypted@1` with `cipher=aes_gcm`. + * `--key` - Base64-encoded primary encryption key (16, 24, or 32 bytes decoded). Generated if not provided. + * `--recovery-key` - Base64-encoded recovery encryption key (16, 24, or 32 bytes decoded). Generated if not provided. + """ use Mix.Task @@ -160,6 +167,9 @@ defmodule Mix.Tasks.Fact.Create do record_file_writer_options: :string, storage: :string, storage_options: :string, + encrypted: :boolean, + key: :string, + recovery_key: :string, quiet: :boolean ] @@ -207,15 +217,17 @@ defmodule Mix.Tasks.Fact.Create do Mix.raise("Unexpected arguments: #{Enum.join(argv, " ")}") end + parsed = expand_encrypted_flag(parsed) command = %CreateDatabase.V1{args: parsed} state = Decider.initial_state() - with {:ok, [genesis_event]} <- Decider.decide(state, command) do - TheCreator.let_there_be_light(genesis_event) + with {:ok, [genesis_event], encryption_meta} <- Decider.decide(state, command) do + TheCreator.let_there_be_light(genesis_event, encryption_meta) unless Keyword.get(parsed, :quiet, false) do display_banner() display_results(genesis_event, Keyword.get(command.args, :path)) + display_encryption_keys(encryption_meta) display_next_steps() end @@ -228,6 +240,19 @@ defmodule Mix.Tasks.Fact.Create do end end + defp expand_encrypted_flag(parsed) do + if Keyword.get(parsed, :encrypted, false) do + parsed + |> Keyword.delete(:encrypted) + |> Keyword.put_new(:record_file_writer, "encrypted@1") + |> Keyword.put_new(:record_file_writer_options, "cipher=aes_gcm") + |> Keyword.put_new(:record_file_reader, "encrypted@1") + |> Keyword.put_new(:record_file_reader_options, "cipher=aes_gcm") + else + parsed + end + end + defp display_banner() do # ANSI Shadow Mix.shell().info(""" @@ -235,25 +260,25 @@ defmodule Mix.Tasks.Fact.Create do ███████╗ █████╗ ██████╗ ████████╗ ██╔════╝██╔══██╗██╔════╝ ╚══██╔══╝ - █████╗ ███████║██║ ██║ - ██╔══╝ ██╔══██║██║ ██║ - ██║ ██║ ██║╚██████╗ ██║ - ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ v#{Fact.BuildInfo.version()} (#{Fact.BuildInfo.codename()}) + █████╗ ███████║██║ ██║ + ██╔══╝ ██╔══██║██║ ██║ + ██║ ██║ ██║╚██████╗ ██║ + ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ v#{Fact.BuildInfo.version()} (#{Fact.BuildInfo.codename()}) """) end defp display_results(event, path) do Mix.shell().info(""" - + 🐢 "#{Enum.random(@quotes)}" - - ================================================================ + + ================================================================ ID: #{event.database_id} NAME: #{event.database_name} PATH: #{Path.absname(path)} ================================================================ - + Try it out... $ iex -S mix @@ -264,10 +289,33 @@ defmodule Mix.Tasks.Fact.Create do """) end + defp display_encryption_keys(nil), do: :ok + + defp display_encryption_keys(%{kek: kek, recovery_kek: recovery_kek}) do + Mix.shell().info(""" + + ================================================================ + 🔐 ENCRYPTION KEYS — SAVE THESE NOW + ================================================================ + + KEY: #{Base.encode64(kek)} + RECOVERY: #{Base.encode64(recovery_kek)} + + ================================================================ + + Store the key securely (secrets manager, environment variable). + Store the recovery key separately in a safe, offline location. + + These keys will NOT be shown again. If you lose both keys, + your data is unrecoverable. + ================================================================ + """) + end + defp display_next_steps() do Mix.shell().info(""" Next Steps: - + 📖 \e]8;;#{Fact.BuildInfo.docs_url()}\e\\Read the documentation\e]8;;\e\\ at #{Fact.BuildInfo.docs_url()} 🤓 \e]8;;https://leanpub.com/eventmodeling-and-eventsourcing\e\\Learn to understand event sourcing\e]8;;\e\\ 🫆 \e]8;;https://eventmodeling.org\e\\Design and deliver better systems with Event Modeling\e]8;;\e\\ diff --git a/lib/mix/tasks/fact/recover.ex b/lib/mix/tasks/fact/recover.ex new file mode 100644 index 0000000..0243f6a --- /dev/null +++ b/lib/mix/tasks/fact/recover.ex @@ -0,0 +1,196 @@ +defmodule Mix.Tasks.Fact.Recover do + @moduledoc """ + Recovers an encrypted database using the recovery key. + + This task performs a full key rotation: it decrypts all record files with the + old Data Encryption Key (DEK), generates a new DEK along with new primary and + recovery keys, re-encrypts every record, and updates the `.bootstrap` file. + + The database **must not be running** during recovery. + + ## Usage + + mix fact.recover --path data/my_db --recovery-key + + ## Options + + * `--path` / `-p` — (required) Path to the database directory. + * `--recovery-key` — (required) Base64-encoded recovery key. + """ + + use Mix.Task + + @shortdoc "Recovers an encrypted database with a recovery key" + + @switches [ + path: :string, + recovery_key: :string + ] + + @aliases [p: :path] + + @impl true + def run(args) do + {parsed, _argv, _invalid} = OptionParser.parse(args, strict: @switches, aliases: @aliases) + + path = fetch_required!(parsed, :path) + recovery_key_b64 = fetch_required!(parsed, :recovery_key) + + unless File.dir?(path) do + Mix.raise("Path does not exist or is not a directory: #{path}") + end + + with {:ok, recovery_key} <- decode_key(recovery_key_b64), + {:ok, bootstrap} <- Fact.BootstrapFile.read(path), + {:ok, encryption_config} <- get_encryption_config(bootstrap), + {:ok, old_dek} <- unwrap_with_recovery(encryption_config, recovery_key), + {:ok, context} <- load_context(bootstrap, old_dek) do + # Collect all record IDs from the ledger + record_ids = Fact.LedgerFile.read(context) |> Enum.to_list() + total = length(record_ids) + + Mix.shell().info("Recovering #{total} record(s)...\n") + + # Generate new keys + new_dek = Fact.KeyRing.generate_dek() + new_key = :crypto.strong_rand_bytes(32) + new_recovery_key = :crypto.strong_rand_bytes(32) + + # Re-encrypt each record + record_ids + |> Enum.with_index(1) + |> Enum.each(fn {record_id, index} -> + record_path = Fact.Storage.records_path(context, record_id) + + # Make writable (records are WORM) + File.chmod!(record_path, 0o644) + + # Decrypt with old DEK + {:ok, encrypted} = File.read(record_path) + {:ok, plaintext} = Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(old_dek, encrypted) + + # Re-encrypt with new DEK + nonce = :crypto.strong_rand_bytes(12) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, new_dek, nonce, plaintext, "", true) + + new_encrypted = <> + + File.write!(record_path, new_encrypted) + File.chmod!(record_path, 0o444) + + percent = div(index * 100, total) + Mix.shell().info("\r [#{percent}%] #{index}/#{total}") + end) + + # Update bootstrap with new wrapped DEKs + new_wrapped_primary = Fact.KeyRing.wrap_dek(new_dek, new_key) + new_wrapped_recovery = Fact.KeyRing.wrap_dek(new_dek, new_recovery_key) + + updated_bootstrap = %{ + bootstrap + | "encryption" => %{ + bootstrap["encryption"] + | "wrapped_dek_primary" => Base.encode64(new_wrapped_primary), + "wrapped_dek_recovery" => Base.encode64(new_wrapped_recovery) + } + } + + write_bootstrap(path, updated_bootstrap) + + Mix.shell().info(""" + + ================================================================ + Recovery complete. #{total} record(s) re-encrypted. + ================================================================ + 🔐 NEW ENCRYPTION KEYS — SAVE THESE NOW + ================================================================ + + KEY: #{Base.encode64(new_key)} + RECOVERY: #{Base.encode64(new_recovery_key)} + + ================================================================ + + The old keys are no longer valid. + Store these keys securely — they will NOT be shown again. + ================================================================ + """) + else + {:error, reason} -> + Mix.raise("Recovery failed: #{inspect(reason)}") + end + end + + defp fetch_required!(opts, key) do + case Keyword.get(opts, key) do + nil -> + Mix.raise("Missing required option: --#{key |> to_string() |> String.replace("_", "-")}") + + value -> + value + end + end + + defp decode_key(b64) do + case Base.decode64(b64) do + {:ok, key} when byte_size(key) in [16, 24, 32] -> {:ok, key} + {:ok, key} -> {:error, {:invalid_key_size, byte_size(key)}} + :error -> {:error, :invalid_base64} + end + end + + defp get_encryption_config(%{"encryption" => config}) when is_map(config), do: {:ok, config} + defp get_encryption_config(_), do: {:error, :database_not_encrypted} + + defp unwrap_with_recovery(encryption_config, recovery_key) do + wrapped = Base.decode64!(encryption_config["wrapped_dek_recovery"]) + Fact.KeyRing.unwrap_dek(wrapped, recovery_key) + end + + defp load_context(bootstrap, dek) do + bootstrap_context = %Fact.Context{ + event_schema: Fact.Event.Schema.from_config(bootstrap["event_schema"]), + record_file_decoder: Fact.RecordFile.Decoder.from_config(bootstrap["record_file_decoder"]), + record_file_reader: + Fact.RecordFile.Reader.from_config( + bootstrap["record_file_reader"] || + %{"family" => "full", "version" => 1, "options" => %{}} + ), + storage: Fact.Storage.from_config(bootstrap["storage"]) + } + + # Read the genesis record to build the full context + record_id = bootstrap["record_id"] + {:ok, record_path} = {:ok, Fact.Storage.records_path(bootstrap_context, record_id)} + + {:ok, stream} = + Fact.Seam.FileReader.Full.V1.read(%Fact.Seam.FileReader.Full.V1{}, record_path, []) + + encrypted_binary = Enum.at(stream, 0) + {:ok, plaintext} = Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(dek, encrypted_binary) + {:ok, event} = Fact.RecordFile.Decoder.decode(bootstrap_context, plaintext) + + event_data = Map.get(event, Fact.Event.Schema.get(bootstrap_context).event_data) + {:ok, Fact.Context.from_genesis_event_data(event_data)} + end + + defp write_bootstrap(path, bootstrap_data) do + bootstrap_context = %Fact.BootstrapFile.Context{ + encoder: Fact.BootstrapFile.Encoder.from_config(%{family: :json, version: 1, options: %{}}), + name: Fact.BootstrapFile.Name.from_config(%{family: :fixed, version: 1, options: %{}}), + writer: + Fact.BootstrapFile.Writer.from_config(%{family: :standard, version: 1, options: %{}}) + } + + bootstrap_path = Path.join(path, Fact.BootstrapFile.Name.get(bootstrap_context)) + + # Bootstrap is WORM, make writable before overwriting + File.chmod!(bootstrap_path, 0o644) + + {:ok, encoded} = Fact.BootstrapFile.Encoder.encode(bootstrap_context, bootstrap_data) + Fact.BootstrapFile.Writer.write(bootstrap_context, bootstrap_path, encoded) + + File.chmod!(bootstrap_path, 0o444) + end +end diff --git a/mix.exs b/mix.exs index 5bfe7ef..cd1c504 100644 --- a/mix.exs +++ b/mix.exs @@ -70,6 +70,7 @@ defmodule Fact.MixProject do extra_section: "GUIDES", extras: [ "LICENSE", + "guides/components/encryption.md", "guides/components/merkle-mountain-range.md", "guides/components/queries-and-indexes.md", "guides/components/record-cache.md", @@ -87,6 +88,7 @@ defmodule Fact.MixProject do "guides/introduction/process-model.md" ], Components: [ + "guides/components/encryption.md", "guides/components/merkle-mountain-range.md", "guides/components/queries-and-indexes.md", "guides/components/record-cache.md", diff --git a/test/fact/encryption_test.exs b/test/fact/encryption_test.exs new file mode 100644 index 0000000..082ff22 --- /dev/null +++ b/test/fact/encryption_test.exs @@ -0,0 +1,294 @@ +defmodule Fact.EncryptionTest do + @moduledoc false + + use ExUnit.Case + + @moduletag :capture_log + + describe "KEKProvider.Explicit" do + alias Fact.Encryption.KEKProvider.Explicit + + test "returns KEK when valid 32-byte key provided" do + kek = :crypto.strong_rand_bytes(32) + assert {:ok, ^kek} = Explicit.fetch_kek(kek: kek) + end + + test "accepts 16-byte key" do + kek = :crypto.strong_rand_bytes(16) + assert {:ok, ^kek} = Explicit.fetch_kek(kek: kek) + end + + test "accepts 24-byte key" do + kek = :crypto.strong_rand_bytes(24) + assert {:ok, ^kek} = Explicit.fetch_kek(kek: kek) + end + + test "returns error for invalid key size" do + assert {:error, {:invalid_kek_size, 10}} = + Explicit.fetch_kek(kek: :crypto.strong_rand_bytes(10)) + end + + test "returns error when kek not provided" do + assert {:error, :kek_not_provided} = Explicit.fetch_kek([]) + end + + test "returns error when kek is not binary" do + assert {:error, :kek_must_be_binary} = Explicit.fetch_kek(kek: 12345) + end + end + + describe "KeyRing" do + test "wrap and unwrap DEK round-trips" do + dek = Fact.KeyRing.generate_dek() + kek = :crypto.strong_rand_bytes(32) + + wrapped = Fact.KeyRing.wrap_dek(dek, kek) + assert {:ok, ^dek} = Fact.KeyRing.unwrap_dek(wrapped, kek) + end + + test "unwrap fails with wrong KEK" do + dek = Fact.KeyRing.generate_dek() + kek = :crypto.strong_rand_bytes(32) + wrong_kek = :crypto.strong_rand_bytes(32) + + wrapped = Fact.KeyRing.wrap_dek(dek, kek) + assert {:error, :dek_unwrap_failed} = Fact.KeyRing.unwrap_dek(wrapped, wrong_kek) + end + + test "unwrap fails with invalid wrapped DEK" do + kek = :crypto.strong_rand_bytes(32) + assert {:error, :invalid_wrapped_dek} = Fact.KeyRing.unwrap_dek(<<1, 2, 3>>, kek) + end + + test "generate_dek returns 32 bytes" do + dek = Fact.KeyRing.generate_dek() + assert byte_size(dek) == 32 + end + end + + describe "Encrypted FileWriter and FileReader round-trip" do + setup do + dir = Path.join("tmp", "encryption_rw_test_#{:rand.uniform(1_000_000)}") + File.mkdir_p!(dir) + on_exit(fn -> File.rm_rf(dir) end) + + {:ok, dir: dir} + end + + test "encrypts and decrypts a value", %{dir: _dir} do + dek = Fact.KeyRing.generate_dek() + plaintext = ~s({"event_type":"turtle_hatched","event_data":{"name":"Shelldon"}}) + + nonce = :crypto.strong_rand_bytes(12) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, plaintext, "", true) + + encrypted = <> + + {:ok, decrypted} = Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(dek, encrypted) + assert decrypted == plaintext + end + + test "decrypt_binary fails with wrong key", %{dir: _dir} do + dek = Fact.KeyRing.generate_dek() + wrong_dek = Fact.KeyRing.generate_dek() + plaintext = "secret data" + + nonce = :crypto.strong_rand_bytes(12) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, plaintext, "", true) + + encrypted = <> + + assert {:error, :decryption_failed} = + Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(wrong_dek, encrypted) + end + + test "decrypt_binary fails with truncated file" do + dek = Fact.KeyRing.generate_dek() + + assert {:error, :invalid_encrypted_file} = + Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(dek, <<1, 2, 3>>) + end + end + + describe "end-to-end encrypted database" do + setup do + path = + :uuid.get_v4() + |> :uuid.uuid_to_string(:nodash) + |> to_string() + |> then(fn uuid -> "e2e_enc_" <> uuid end) + |> String.downcase() + |> String.split_at(63) + |> then(fn {name, _} -> Path.join("tmp", name) end) + + # Create encrypted database — keys are auto-generated + args = [ + path: path, + record_file_writer: "encrypted@1", + record_file_writer_options: "cipher=aes_gcm", + record_file_reader: "encrypted@1", + record_file_reader_options: "cipher=aes_gcm" + ] + + command = %Fact.Genesis.Command.CreateDatabase.V1{args: args} + + {:ok, [genesis_event], encryption_meta} = + Fact.Genesis.Decider.decide(Fact.Genesis.Decider.initial_state(), command) + + Fact.Genesis.TheCreator.let_there_be_light(genesis_event, encryption_meta) + + on_exit(fn -> TestHelper.rm_rf(path) end) + + {:ok, path: path, encryption_meta: encryption_meta} + end + + test "keys are auto-generated when not provided", %{encryption_meta: meta} do + assert byte_size(meta.kek) == 32 + assert byte_size(meta.recovery_kek) == 32 + assert byte_size(meta.dek) == 32 + assert meta.kek != meta.recovery_kek + end + + test "genesis record file is not plaintext on disk", %{path: path} do + record_file = + Path.join(path, "events/**/*") + |> Path.wildcard() + |> Enum.reject(&File.dir?/1) + |> hd() + + {:ok, bytes} = File.read(record_file) + + refute String.starts_with?(bytes, "{") + assert byte_size(bytes) > 28 + end + + test "bootstrap file contains encryption metadata", %{path: path} do + {:ok, bootstrap} = Fact.BootstrapFile.read(path) + + assert bootstrap["encryption"] + assert bootstrap["encryption"]["wrapped_dek_primary"] + assert bootstrap["encryption"]["wrapped_dek_recovery"] + assert bootstrap["encryption"]["family"] == "aes_gcm" + end + + test "open and read with encryption_key", %{path: path, encryption_meta: meta} do + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + + events = Fact.read(db, :all) + assert length(events) == 1 + assert hd(events)["event_type"] == "Elixir.Fact.Genesis.Event.DatabaseCreated.V1" + end + + test "append and read encrypted events", %{path: path, encryption_meta: meta} do + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + + event = %{type: "turtle_hatched", data: %{name: "Shelldon"}} + {:ok, position} = Fact.append(db, event) + + TestHelper.subscribe_and_wait(db, position) + + [record] = Fact.read(db, :all, position: position - 1, count: 1) + assert record["event_type"] == "turtle_hatched" + assert record["event_data"]["name"] == "Shelldon" + end + + test "wrong encryption key returns clear error", %{path: path} do + wrong_key = :crypto.strong_rand_bytes(32) |> Base.encode64() + assert {:error, :invalid_encryption_key} = Fact.open(path, encryption_key: wrong_key) + end + + test "missing encryption key returns clear error", %{path: path} do + assert {:error, :encryption_key_required} = Fact.open(path) + end + + test "mix fact.recover rotates keys and re-encrypts records", %{ + path: path, + encryption_meta: meta + } do + # Open, append an event, then close the database + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + {:ok, position} = Fact.append(db, %{type: "turtle_hatched", data: %{name: "Shelldon"}}) + TestHelper.subscribe_and_wait(db, position) + # subscribe_and_wait links us to PubSub; trap exits during close + Process.flag(:trap_exit, true) + :ok = Fact.close(db) + Process.flag(:trap_exit, false) + + # Run recovery with the recovery key + Mix.Tasks.Fact.Recover.run([ + "--path", + path, + "--recovery-key", + Base.encode64(meta.recovery_kek) + ]) + + # Old key should no longer work — bootstrap has new wrapped DEKs + {:ok, bootstrap} = Fact.BootstrapFile.read(path) + old_wrapped = Base.decode64!(bootstrap["encryption"]["wrapped_dek_primary"]) + assert {:error, :dek_unwrap_failed} = Fact.KeyRing.unwrap_dek(old_wrapped, meta.kek) + end + + test "--encrypted flag via mix task creates encrypted database" do + path = + :uuid.get_v4() + |> :uuid.uuid_to_string(:nodash) + |> to_string() + |> then(fn uuid -> "e2e_enc_flag_" <> uuid end) + |> String.downcase() + |> String.split_at(63) + |> then(fn {name, _} -> Path.join("tmp", name) end) + + Mix.Tasks.Fact.Create.run(["--path", path, "--encrypted"]) + + {:ok, bootstrap} = Fact.BootstrapFile.read(path) + assert bootstrap["encryption"] + assert bootstrap["record_file_writer"]["family"] == "encrypted" + + TestHelper.rm_rf(path) + end + + test "user-provided keys are used when specified" do + path = + :uuid.get_v4() + |> :uuid.uuid_to_string(:nodash) + |> to_string() + |> then(fn uuid -> "e2e_enc_custom_" <> uuid end) + |> String.downcase() + |> String.split_at(63) + |> then(fn {name, _} -> Path.join("tmp", name) end) + + my_key = :crypto.strong_rand_bytes(32) + my_recovery = :crypto.strong_rand_bytes(32) + + args = [ + path: path, + record_file_writer: "encrypted@1", + record_file_writer_options: "cipher=aes_gcm", + record_file_reader: "encrypted@1", + record_file_reader_options: "cipher=aes_gcm", + key: Base.encode64(my_key), + recovery_key: Base.encode64(my_recovery) + ] + + command = %Fact.Genesis.Command.CreateDatabase.V1{args: args} + + {:ok, [genesis_event], encryption_meta} = + Fact.Genesis.Decider.decide(Fact.Genesis.Decider.initial_state(), command) + + assert encryption_meta.kek == my_key + assert encryption_meta.recovery_kek == my_recovery + + Fact.Genesis.TheCreator.let_there_be_light(genesis_event, encryption_meta) + + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(my_key)) + events = Fact.read(db, :all) + assert length(events) == 1 + + TestHelper.rm_rf(path) + end + end +end From ed34d41dcfa923bef59a0504eeb3a015a7e5eb83 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Wed, 1 Apr 2026 23:59:21 -0700 Subject: [PATCH 2/7] fix: prevent DEK exposure in crash dumps, :sys.get_state, observer, and trace outputs. --- lib/fact/key_ring.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/fact/key_ring.ex b/lib/fact/key_ring.ex index 4bab4c2..fb95c07 100644 --- a/lib/fact/key_ring.ex +++ b/lib/fact/key_ring.ex @@ -55,6 +55,10 @@ defmodule Fact.KeyRing do wrapped_dek = Keyword.fetch!(opts, :wrapped_dek) kek = Keyword.fetch!(opts, :kek) + # Prevent the DEK from appearing in crash dumps, :sys.get_state, + # observer, and trace output. + Process.flag(:sensitive, true) + case unwrap_dek(wrapped_dek, kek) do {:ok, dek} -> {:ok, %{dek: dek}} From 772543334383c33eb4f56bcccc77ce19aca62df2 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Thu, 2 Apr 2026 00:02:53 -0700 Subject: [PATCH 3/7] fix: add decryption for verification --- lib/fact/merkle_mountain_range.ex | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/lib/fact/merkle_mountain_range.ex b/lib/fact/merkle_mountain_range.ex index 248dbbf..b14e5e4 100644 --- a/lib/fact/merkle_mountain_range.ex +++ b/lib/fact/merkle_mountain_range.ex @@ -456,12 +456,13 @@ defmodule Fact.MerkleMountainRange do defp do_verify(state) do {:ok, context} = Fact.Registry.get_context(state.database_id) prev_leaf_hash = :binary.copy(<<0>>, state.hash_size) + dek = resolve_dek(context) tampered = Fact.LedgerFile.read(state.database_id) |> Enum.with_index() |> Enum.reduce({[], prev_leaf_hash}, fn {record_id, index}, {tampered, prev_hash} -> - case verify_record(context, state, record_id, index, prev_hash) do + case verify_record(context, state, record_id, index, prev_hash, dek) do {:ok, _store_position, leaf_hash} -> {tampered, leaf_hash} @@ -478,8 +479,15 @@ defmodule Fact.MerkleMountainRange do end end - defp verify_record(context, state, record_id, index, prev_hash) do - content_ok? = verify_content_hash(context, record_id) + defp resolve_dek(%{record_file_reader: %{module: mod}} = context) do + if mod.family() == :encrypted do + {:ok, dek} = Fact.KeyRing.get_dek(context.database_id) + dek + end + end + + defp verify_record(context, state, record_id, index, prev_hash, dek) do + content_ok? = verify_content_hash(context, record_id, dek) # Read event and extract position; if the file was corrupted beyond decoding, # treat it as tampered but continue verification with a poison hash. @@ -504,17 +512,23 @@ defmodule Fact.MerkleMountainRange do end end - defp verify_content_hash(context, record_id) do + defp verify_content_hash(context, record_id, dek) do record_path = Fact.Storage.records_path(context, record_id) with {:ok, raw_bytes} <- File.read(record_path), - {:ok, expected_id} <- Fact.RecordFile.Name.get(context, raw_bytes, []) do + {:ok, content} <- decrypt_if_needed(dek, raw_bytes), + {:ok, expected_id} <- Fact.RecordFile.Name.get(context, content, []) do expected_id == record_id else _ -> false end end + defp decrypt_if_needed(nil, raw_bytes), do: {:ok, raw_bytes} + + defp decrypt_if_needed(dek, raw_bytes), + do: Fact.Seam.FileReader.Encrypted.V1.decrypt_binary(dek, raw_bytes) + # ----------------------------- # MMR Position Arithmetic # ----------------------------- From 7f3bb9b7a447dfdb2dd2698f215d538bf51a7aa1 Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Thu, 2 Apr 2026 00:10:35 -0700 Subject: [PATCH 4/7] chore: update doc attributes --- lib/fact.ex | 2 +- lib/fact/bootstrap_file.ex | 2 +- lib/fact/genesis/the_creator.ex | 2 +- lib/fact/supervisor.ex | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/fact.ex b/lib/fact.ex index 6fa7c60..6aab53e 100644 --- a/lib/fact.ex +++ b/lib/fact.ex @@ -641,7 +641,7 @@ defmodule Fact do :ok """ - @doc since: "0.3.1" + @doc since: "0.4.0" @spec close(database_id()) :: :ok | {:error, :not_found} def close(database_id) when is_binary(database_id) do Fact.Supervisor.stop_database(database_id) diff --git a/lib/fact/bootstrap_file.ex b/lib/fact/bootstrap_file.ex index 8f568f3..08981da 100644 --- a/lib/fact/bootstrap_file.ex +++ b/lib/fact/bootstrap_file.ex @@ -186,7 +186,7 @@ defmodule Fact.BootstrapFile do It writes just enough configuration, so that the genesis record (`Fact.Genesis.Event.DatabaseCreated.V1`) can be read and a full `Fact.Context` can be loaded. """ - @doc since: "0.3.1" + @doc since: "0.4.0" @spec write(Path.t(), genesis_record :: Fact.record(), keyword()) :: :ok | {:error, term()} def write(path, {record_id, event}, opts \\ []) do this = get_bootstrap_write_context() diff --git a/lib/fact/genesis/the_creator.ex b/lib/fact/genesis/the_creator.ex index 17b7600..791234c 100644 --- a/lib/fact/genesis/the_creator.ex +++ b/lib/fact/genesis/the_creator.ex @@ -26,7 +26,7 @@ defmodule Fact.Genesis.TheCreator do (since the KeyRing process is not yet running) and the bootstrap file includes the wrapped DEK for later decryption. """ - @doc since: "0.3.1" + @doc since: "0.4.0" @spec let_there_be_light(%DatabaseCreated.V1{}, map() | nil) :: :ok def let_there_be_light(%DatabaseCreated.V1{} = event, encryption_meta \\ nil) do with context <- DatabaseCreated.V1.to_context(event), diff --git a/lib/fact/supervisor.ex b/lib/fact/supervisor.ex index d6ec15d..0bb47da 100644 --- a/lib/fact/supervisor.ex +++ b/lib/fact/supervisor.ex @@ -95,7 +95,7 @@ defmodule Fact.Supervisor do Returns `:ok` on success, or `{:error, :not_found}` if no database with the given identifier is running. """ - @doc since: "0.3.1" + @doc since: "0.4.0" @spec stop_database(Fact.database_id()) :: :ok | {:error, :not_found} def stop_database(database_id) when is_binary(database_id) do child_id = {Fact.DatabaseSupervisor, database_id} From 1ca81c8f7d32a7a27a2ffa6a95d3c53559268e4d Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Tue, 7 Apr 2026 23:33:29 -0700 Subject: [PATCH 5/7] feat: Adds encryption to the write ahead log entries. --- lib/fact/database_supervisor.ex | 9 ++++- lib/fact/write_ahead_log.ex | 60 +++++++++++++++++++++++++++------ test/fact/encryption_test.exs | 40 ++++++++++++++++++++++ 3 files changed, 98 insertions(+), 11 deletions(-) diff --git a/lib/fact/database_supervisor.ex b/lib/fact/database_supervisor.ex index 7083fa9..2a5295a 100644 --- a/lib/fact/database_supervisor.ex +++ b/lib/fact/database_supervisor.ex @@ -93,8 +93,15 @@ defmodule Fact.DatabaseSupervisor do def init({%Fact.Context{database_id: database_id} = context, opts}) do Fact.Registry.register(context) + encryption_opts = Keyword.get(opts, :encryption, []) + encrypted = Keyword.get(encryption_opts, :wrapped_dek) != nil + wal_opts = - [database_id: database_id, name: Fact.Registry.via(database_id, Fact.WriteAheadLog)] ++ + [ + database_id: database_id, + name: Fact.Registry.via(database_id, Fact.WriteAheadLog), + encrypted: encrypted + ] ++ Keyword.get(opts, :wal, []) cache_opts = Keyword.get(opts, :cache, []) diff --git a/lib/fact/write_ahead_log.ex b/lib/fact/write_ahead_log.ex index 802d46f..c4e94f8 100644 --- a/lib/fact/write_ahead_log.ex +++ b/lib/fact/write_ahead_log.ex @@ -163,7 +163,8 @@ defmodule Fact.WriteAheadLog do max_file_size: nil, max_segments: nil, should_fsync: false, - sync_interval: 0 + sync_interval: 0, + encrypted: false # ----------------------------- # Init @@ -195,6 +196,8 @@ defmodule Fact.WriteAheadLog do file = segment_path(database_id, segment_index) {:ok, buffer} = :file.open(file, [:append, :binary, :read, :write]) + encrypted = Keyword.get(opts, :encrypted, false) + state = %__MODULE__{ database_id: database_id, file: file, @@ -203,7 +206,8 @@ defmodule Fact.WriteAheadLog do max_file_size: max_file_size, max_segments: max_segments, should_fsync: enable_fsync, - sync_interval: sync_interval + sync_interval: sync_interval, + encrypted: encrypted } last_seq = get_last_sequence_no(state) @@ -224,6 +228,7 @@ defmodule Fact.WriteAheadLog do {:ok, state} -> seq = state.last_seq + 1 + data = maybe_encrypt(data, state) entry = Entry.create(seq, data, checkpoint?) {:ok, state} = write_entry_to_buffer(entry, %{state | last_seq: seq}) @@ -252,7 +257,7 @@ defmodule Fact.WriteAheadLog do def handle_call({:read_all, from_checkpoint?}, _from, state) do files = segment_files(state.database_id) |> Enum.sort() - {:reply, read_segments(files, from_checkpoint?), state} + {:reply, read_segments(files, from_checkpoint?, state), state} end def handle_call({:read_all_from_offset, offset, from_checkpoint?}, _from, state) do @@ -261,7 +266,7 @@ defmodule Fact.WriteAheadLog do |> Enum.filter(fn {idx, _} -> idx >= offset end) |> Enum.sort() - {:reply, read_segments(files, from_checkpoint?), state} + {:reply, read_segments(files, from_checkpoint?, state), state} end def handle_call(:repair, _from, state) do @@ -286,6 +291,39 @@ defmodule Fact.WriteAheadLog do {:noreply, state} end + # ----------------------------- + # Encryption + # ----------------------------- + + @nonce_size 12 + @auth_tag_size 16 + + defp maybe_encrypt(data, %{encrypted: false}), do: data + + defp maybe_encrypt(data, %{encrypted: true, database_id: database_id}) do + {:ok, dek} = Fact.KeyRing.get_dek(database_id) + nonce = :crypto.strong_rand_bytes(@nonce_size) + + {ciphertext, auth_tag} = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, data, "", true) + + <> + end + + defp maybe_decrypt_entry(entry, %{encrypted: false}), do: entry + + defp maybe_decrypt_entry(entry, %{encrypted: true, database_id: database_id}) do + {:ok, dek} = Fact.KeyRing.get_dek(database_id) + + <> = entry.data + + plaintext = + :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, ciphertext, "", auth_tag, false) + + %{entry | data: plaintext} + end + # ----------------------------- # Helpers # ----------------------------- @@ -368,10 +406,10 @@ defmodule Fact.WriteAheadLog do # Reading # --------------------- - defp read_segments(files, from_checkpoint?) do + defp read_segments(files, from_checkpoint?, state) do Enum.reduce(files, {[], 0}, fn {_idx, path}, {acc, last_cp} -> {:ok, fd} = File.open(path, [:read, :binary]) - {entries, cp} = do_read_entries(fd, from_checkpoint?) + {entries, cp} = do_read_entries(fd, from_checkpoint?, state) File.close(fd) if from_checkpoint? and cp > last_cp do @@ -383,17 +421,19 @@ defmodule Fact.WriteAheadLog do |> elem(0) end - defp do_read_entries(fd, from_checkpoint?) do - read_loop(fd, [], 0, from_checkpoint?) + defp do_read_entries(fd, from_checkpoint?, state) do + read_loop(fd, [], 0, from_checkpoint?, state) end - defp read_loop(fd, acc, last_cp, from_checkpoint?) do + defp read_loop(fd, acc, last_cp, from_checkpoint?, state) do case :file.read(fd, 4) do {:ok, <>} -> case :file.read(fd, size) do {:ok, bin} -> case Entry.deserialize(bin) do {:ok, entry} -> + entry = maybe_decrypt_entry(entry, state) + {new_acc, new_cp} = if entry.is_checkpoint and from_checkpoint? do {[], entry.lsn} @@ -401,7 +441,7 @@ defmodule Fact.WriteAheadLog do {[entry | acc], last_cp} end - read_loop(fd, new_acc, new_cp, from_checkpoint?) + read_loop(fd, new_acc, new_cp, from_checkpoint?, state) {:error, _} -> {Enum.reverse(acc), last_cp} diff --git a/test/fact/encryption_test.exs b/test/fact/encryption_test.exs index 082ff22..758e644 100644 --- a/test/fact/encryption_test.exs +++ b/test/fact/encryption_test.exs @@ -196,6 +196,46 @@ defmodule Fact.EncryptionTest do assert record["event_data"]["name"] == "Shelldon" end + test "WAL segment files do not contain plaintext event data", %{ + path: path, + encryption_meta: meta + } do + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + + {:ok, position} = Fact.append(db, %{type: "turtle_hatched", data: %{name: "Shelldon"}}) + TestHelper.subscribe_and_wait(db, position) + + Fact.WriteAheadLog.sync(db) + + wal_dir = Path.join(path, "wal") + + wal_bytes = + Path.wildcard(Path.join(wal_dir, "*")) + |> Enum.map(&File.read!/1) + |> Enum.join() + + refute String.contains?(wal_bytes, "turtle_hatched") + refute String.contains?(wal_bytes, "Shelldon") + end + + test "WAL recovery works with encrypted entries", %{path: path, encryption_meta: meta} do + {:ok, db} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + + {:ok, position} = Fact.append(db, %{type: "turtle_hatched", data: %{name: "Shelldon"}}) + TestHelper.subscribe_and_wait(db, position) + + # Close and reopen to trigger WAL recovery + Process.flag(:trap_exit, true) + :ok = Fact.close(db) + Process.flag(:trap_exit, false) + + {:ok, db2} = Fact.open(path, encryption_key: Base.encode64(meta.kek)) + + events = Fact.read(db2, :all) + event_types = Enum.map(events, & &1["event_type"]) + assert "turtle_hatched" in event_types + end + test "wrong encryption key returns clear error", %{path: path} do wrong_key = :crypto.strong_rand_bytes(32) |> Base.encode64() assert {:error, :invalid_encryption_key} = Fact.open(path, encryption_key: wrong_key) From b676c14819aef3049476fdea5bb9dfa44982fb3f Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Wed, 8 Apr 2026 07:36:05 -0700 Subject: [PATCH 6/7] chore: format code --- lib/fact/write_ahead_log.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fact/write_ahead_log.ex b/lib/fact/write_ahead_log.ex index c4e94f8..7f5a5ac 100644 --- a/lib/fact/write_ahead_log.ex +++ b/lib/fact/write_ahead_log.ex @@ -315,8 +315,8 @@ defmodule Fact.WriteAheadLog do defp maybe_decrypt_entry(entry, %{encrypted: true, database_id: database_id}) do {:ok, dek} = Fact.KeyRing.get_dek(database_id) - <> = entry.data + <> = + entry.data plaintext = :crypto.crypto_one_time_aead(:aes_256_gcm, dek, nonce, ciphertext, "", auth_tag, false) From 872141983a64d1c073548686a63a14b9c46c271e Mon Sep 17 00:00:00 2001 From: Jake Bruun Date: Fri, 17 Apr 2026 22:32:36 -0700 Subject: [PATCH 7/7] doc: organize module docs --- guides/components/encryption.md | 8 ++++---- mix.exs | 13 ++++++++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/guides/components/encryption.md b/guides/components/encryption.md index 744549c..47504ca 100644 --- a/guides/components/encryption.md +++ b/guides/components/encryption.md @@ -42,9 +42,9 @@ The wrapped DEK (encrypted with both the primary and recovery KEKs) is stored in alongside the database configuration. The KEKs themselves are never written to disk. ``` - ┌─────────────┐ - │ Primary KEK │ (runtime only, never on disk) - └──────┬──────┘ + ┌────────────────┐ + │ Primary KEK │ (runtime only, never on disk) + └────────┬───────┘ │ wraps ▼ ┌────────────────┐ @@ -53,7 +53,7 @@ alongside the database configuration. The KEKs themselves are never written to d │ encrypts ▼ ┌────────────────┐ - │ Record Files │ (encrypted on disk) + │ Record Files │ (encrypted on disk) └────────────────┘ ``` diff --git a/mix.exs b/mix.exs index cd1c504..bf1b1c0 100644 --- a/mix.exs +++ b/mix.exs @@ -145,12 +145,21 @@ defmodule Fact.MixProject do Fact.Storage ], Core: [ + Fact.BuildInfo, Fact.Database, Fact.EventLedger, Fact.EventReader, Fact.EventStreamWriter, Fact.Query, - Fact.QueryItem + Fact.QueryItem, + Fact.RecordCache, + Fact.WriteAheadLog, + Fact.WriteAheadLog.Entry + ], + Encryption: [ + Fact.Encryption.KEKProvider, + Fact.Encryption.KEKProvider.Explicit, + Fact.KeyRing ], Genesis: [ Fact.Genesis.Command.CreateDatabase.V1, @@ -226,11 +235,13 @@ defmodule Fact.MixProject do Fact.Seam.FileName.Registry, Fact.Seam.FileReader, Fact.Seam.FileReader.Adapter, + Fact.Seam.FileReader.Encrypted.V1, Fact.Seam.FileReader.FixedLength.V1, Fact.Seam.FileReader.Full.V1, Fact.Seam.FileReader.Registry, Fact.Seam.FileWriter, Fact.Seam.FileWriter.Adapter, + Fact.Seam.FileWriter.Encrypted.V1, Fact.Seam.FileWriter.Standard.V1, Fact.Seam.FileWriter.Registry, Fact.Seam.Instance,