From 3f98d8b01ca3b328b4aed69ed66f857fb8af2184 Mon Sep 17 00:00:00 2001 From: Marek Petrik Date: Wed, 29 Jul 2026 18:58:02 -0400 Subject: [PATCH 1/9] =?UTF-8?q?Generalize=20probability=20space=20from=20F?= =?UTF-8?q?in=20n=20to=20any=20[FinEnum=20=CE=A9]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sample space of Findist and FinRV is now an arbitrary finitely-enumerable type Ω with [FinEnum Ω] instead of Fin n. Random variables remain bare functions Ω → ρ sharing a single Ω, so a distribution and its random variables are tied to the same set at the type level. Expectation and probability route through the Fintype structure, so their values are independent of the enumeration order. - Findist Ω / FinRV Ω ρ generalized; nonempty now yields Nonempty Ω; singleton replaced by dirac. - nneg_dotProd_pos_ex_pos reproved over any Fintype (Finset.sum_nonpos), dropping the Fin-induction and dotProduct_head_tail. - RV_Unique_Values / expect_def_correct rebuilt on FinEnum.toList. - Discrete RV label spaces kept as Fin k. - Matrix.lean fixed (broken import), generalized to Ω, and added to MDPLib. - Replaced local false_of_le_gt / false_of_lt_ge with Mathlib not_lt_of_ge / not_le_of_gt at all call sites. Claude-Session: https://claude.ai/code/session_01UUYp3BStWn8DX1gXYZvLtn --- MDPLib.lean | 1 + MDPLib/MDP/Histories.lean | 2 +- MDPLib/Probability/Basic.lean | 126 +++++++++++++------------- MDPLib/Probability/Defs.lean | 147 +++++++++++++++---------------- MDPLib/Probability/Matrix.lean | 26 +++--- MDPLib/Probability/Prelude.lean | 7 +- MDPLib/Probability/Quantile.lean | 4 +- MDPLib/Risk/CVaR.lean | 8 +- MDPLib/Risk/VaR.lean | 26 +++--- 9 files changed, 176 insertions(+), 171 deletions(-) diff --git a/MDPLib.lean b/MDPLib.lean index dd26722..ab166c9 100644 --- a/MDPLib.lean +++ b/MDPLib.lean @@ -4,6 +4,7 @@ import MDPLib.Probability.Prelude import MDPLib.Probability.Defs import MDPLib.Probability.Basic +import MDPLib.Probability.Matrix import MDPLib.Probability.Quantile import MDPLib.Risk.VaR diff --git a/MDPLib/MDP/Histories.lean b/MDPLib/MDP/Histories.lean index 81711a4..09a0835 100644 --- a/MDPLib/MDP/Histories.lean +++ b/MDPLib/MDP/Histories.lean @@ -33,7 +33,7 @@ structure MDP : Type where A : ℕ A_ne : 0 < A /-- transition probability s, a, s' -/ - P : Fin S → Fin A → Δ S + P : Fin S → Fin A → Δ (Fin S) /-- reward function s, a, s' -/ r : Fin S → Fin A → Fin S → ℝ diff --git a/MDPLib/Probability/Basic.lean b/MDPLib/Probability/Basic.lean index f781808..f524e39 100644 --- a/MDPLib/Probability/Basic.lean +++ b/MDPLib/Probability/Basic.lean @@ -18,33 +18,28 @@ import Mathlib.Data.Fin.Tuple.Sort -- for Equiv.Perm and permutation operations -/ -section General +section General open Matrix -variable {n : ℕ} {p x : Fin n.succ → ℚ} +variable {Ω : Type} [Fintype Ω] {p x : Ω → ℚ} -theorem dotProduct_head_tail : p ⬝ᵥ x = (vecHead p) * (vecHead x) + (vecTail p) ⬝ᵥ (vecTail x) := by - rw [← cons_dotProduct, cons_head_tail] - -variable {p x : Fin n → ℚ} - -theorem nneg_dotProd_pos_ex_pos (h1 : ∀ ω, p ω ≥ 0) (h2 : ∀ ω, x ω ≥ 0) (h : p ⬝ᵥ x > 0) : ∃ ω, x ω > 0 := by - induction n with - | zero => simp_all - | succ n ih => - by_cases hn : x ⟨0, Nat.zero_lt_succ n⟩ > 0 - · exact ⟨0, hn⟩ - · push Not at hn - have hvh0 : 0 = vecHead x := le_antisymm (h2 0) hn - rewrite [dotProduct_head_tail, ←hvh0] at h - obtain ⟨ω, hω⟩ := ih (Fin.tail h1) (Fin.tail h2) (by simpa [vecHead, vecTail] using h) - use ω.succ +/-- If a dot product of nonnegative vectors is positive, some coordinate of the + second vector is positive. Proved over an arbitrary finite index. -/ +theorem nneg_dotProd_pos_ex_pos (h1 : ∀ ω, p ω ≥ 0) (h2 : ∀ ω, x ω ≥ 0) (h : p ⬝ᵥ x > 0) : ∃ ω, x ω > 0 := by + by_contra hcon + push Not at hcon + have hle : p ⬝ᵥ x ≤ 0 := by + unfold dotProduct + apply Finset.sum_nonpos + intro ω _ + nlinarith [h1 ω, hcon ω] + linarith end General namespace Findist -variable {n : ℕ} {P : Findist n} {B : FinRV n Bool} +variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {B : FinRV Ω Bool} theorem ge_zero : 0 ≤ ℙ[B // P] := by rw [prob_eq_exp_ind] @@ -57,7 +52,7 @@ theorem le_one : ℙ[B // P] ≤ 1 := calc 𝔼[𝕀 ∘ B//P] ≤ 𝔼[1 // P] := exp_monotone ind_le_one _ = 1 := exp_const -theorem in_prob (P : Findist n) : Prob ℙ[B // P] := ⟨ge_zero, le_one⟩ +theorem in_prob (P : Findist Ω) : Prob ℙ[B // P] := ⟨ge_zero, le_one⟩ end Findist @@ -66,7 +61,7 @@ end Findist section RandomVariables -variable {n : ℕ} {P : Findist n} {A B : FinRV n Bool} {X Y : FinRV n ℚ} {t t₁ t₂ : ℚ} +variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {A B : FinRV Ω Bool} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} theorem rvle_monotone (h1 : X ≤ Y) (h2: t₁ ≤ t₂) : 𝕀 ∘ (Y ≤ᵣ t₁) ≤ 𝕀 ∘ (X ≤ᵣ t₂) := by intro ω @@ -95,7 +90,7 @@ theorem rv_le_max_one : (X ≤ᵣ (FinRV.max P X)) = 1 := theorem rv_max_in_image : (FinRV.max P X) ∈ Finset.univ.image X := Finset.max'_mem (Finset.image X Finset.univ) (rv_image_nonempty P X) -theorem rv_omega_ge_min (P : Findist n) : ∀ω, X ω ≥ (FinRV.min P X) := +theorem rv_omega_ge_min (P : Findist Ω) : ∀ω, X ω ≥ (FinRV.min P X) := by intro ω have h : X ω ∈ (Finset.image X Finset.univ) := Finset.mem_image_of_mem X (Finset.mem_univ ω) simpa using Finset.min'_le (Finset.image X Finset.univ) (X ω) h @@ -113,7 +108,7 @@ theorem rv_monotone_sharp {t₁ t₂ : ℚ} : t₁ < t₂ → ∀ ω, (X ≥ᵣ -- results for discrete probability distributions section Atomic -variable (P : Findist n) (X : FinRV n ℚ) (t : ℚ) +variable (P : Findist Ω) (X : FinRV Ω ℚ) (t : ℚ) theorem prob_atomic_omega {b : ℚ} (h : ℙ[X =ᵣ b // P] > 0) : ∃ω, X ω = b := by @@ -139,12 +134,12 @@ theorem rv_le_step_lt_max (h0 : t < (FinRV.max P X)) : ∃q > t, (X ≤ᵣ t) = · exact fun h2 => lt_of_le_of_lt h2 hq_Y · intro h2 have hxω : X ω ∉ 𝓨 := by - by_contra! inY; exact false_of_le_gt (Finset.min'_le 𝓨 (X ω) inY) h2 + by_contra! inY; exact not_lt_of_ge (Finset.min'_le 𝓨 (X ω) inY) h2 rw [Finset.mem_filter] at hxω push Not at hxω exact hxω (Finset.mem_image_of_mem X (Finset.mem_univ ω)) -theorem rv_le_step_lt (P : Findist n) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) := +theorem rv_le_step_lt (P : Findist Ω) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) := by cases' lt_or_ge t (FinRV.max P X) with hlt hge · obtain ⟨q, h⟩ := rv_le_step_lt_max P X t hlt exact ⟨q, ⟨h.1, h.2.1⟩⟩ @@ -264,7 +259,7 @@ end RandomVariables section Probability -variable {n : ℕ} {P : Findist n} {A B C : FinRV n Bool} {X Y : FinRV n ℚ} {t t₁ t₂ : ℚ} +variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {A B C : FinRV Ω Bool} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} theorem prob_compl_sums_to_one : ℙ[B // P] + ℙ[¬ᵣB // P] = 1 := @@ -282,7 +277,7 @@ theorem rv_le_compl_gt : (X ≤ᵣ t) + (X >ᵣ t) = 1 := by theorem prob_le_compl_gt : ℙ[X ≤ᵣ t // P] + ℙ[X >ᵣ t // P] = 1 := by rw [prob_eq_exp_ind, prob_eq_exp_ind, ← exp_additive_two] - have h : (𝕀 ∘ (X ≤ᵣ t)) + (𝕀 ∘ (X >ᵣ t)) = (1 : FinRV n ℚ) := by + have h : (𝕀 ∘ (X ≤ᵣ t)) + (𝕀 ∘ (X >ᵣ t)) = (1 : FinRV Ω ℚ) := by ext ω unfold FinRV.leq FinRV.gt simp [𝕀, indicator] @@ -295,16 +290,16 @@ theorem prob_le_compl_gt : ℙ[X ≤ᵣ t // P] + ℙ[X >ᵣ t // P] = 1 := by exact exp_one theorem prob_gt_of_le : ℙ[X >ᵣ t // P] = 1 - ℙ[X ≤ᵣ t // P] := by - rw [←prob_le_compl_gt] + rw [←prob_le_compl_gt (P := P) (X := X) (t := t)] ring theorem prob_le_of_gt : ℙ[X ≤ᵣ t // P] = 1 - ℙ[X >ᵣ t // P] := by - rw [←prob_le_compl_gt] + rw [←prob_le_compl_gt (P := P) (X := X) (t := t)] ring theorem prob_lt_compl_ge : ℙ[X <ᵣ t // P] + ℙ[X ≥ᵣ t // P] = 1 := by rw [prob_eq_exp_ind, prob_eq_exp_ind, ← exp_additive_two] - have h : (𝕀 ∘ (X <ᵣ t)) + (𝕀 ∘ (X ≥ᵣ t)) = (1 : FinRV n ℚ) := by + have h : (𝕀 ∘ (X <ᵣ t)) + (𝕀 ∘ (X ≥ᵣ t)) = (1 : FinRV Ω ℚ) := by ext ω unfold FinRV.lt FinRV.geq simp [𝕀, indicator] @@ -317,10 +312,10 @@ theorem prob_lt_compl_ge : ℙ[X <ᵣ t // P] + ℙ[X ≥ᵣ t // P] = 1 := by exact exp_one theorem prob_ge_of_lt : ℙ[X ≥ᵣ t // P] = 1 - ℙ[X <ᵣ t // P] := by - rw [← prob_lt_compl_ge]; ring + rw [← prob_lt_compl_ge (P := P) (X := X) (t := t)]; ring theorem prob_lt_of_ge : ℙ[X <ᵣ t // P] = 1 - ℙ[X ≥ᵣ t // P] := by - rw [← prob_lt_compl_ge]; ring + rw [← prob_lt_compl_ge (P := P) (X := X) (t := t)]; ring theorem prob_bool_monotone : A ≤ B → ℙ[A // P] ≤ ℙ[B // P] := fun h => exp_monotone (ind_monotone h) @@ -367,11 +362,11 @@ theorem prob_le_max_of_le_1 {t : ℚ} (h : ℙ[X ≤ᵣ t // P] < 1) : t < FinRV by_contra! hcontra have h1 := prob_le_monotone (P := P) (le_refl X) hcontra rw [prob_le_eq_one] at h1 - exact false_of_lt_ge h h1 + exact not_le_of_gt h h1 section Rounding ---results for discrete probability distributions -variable (P : Findist n) (X : FinRV n ℚ) (t : ℚ) +variable (P : Findist Ω) (X : FinRV Ω ℚ) (t : ℚ) theorem prob_le_step_lt_max (h: t < (FinRV.max P X)) : ∃q > t, ℙ[X ≤ᵣ t // P] = ℙ[X <ᵣ q // P] ∧ q ∈ (Finset.univ.image X) := @@ -459,7 +454,7 @@ end Probability section CDF -variable {n : ℕ} {P : Findist n} {X Y : FinRV n ℚ} {t t₁ t₂ : ℚ} +variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} /-- shows CDF is non-decreasing -/ theorem cdf_nondecreasing : t₁ ≤ t₂ → cdf P X t₁ ≤ cdf P X t₂ := by @@ -477,8 +472,8 @@ end CDF section Expectation -variable {n : ℕ} {P : Findist n} -variable {k : ℕ} {X : FinRV n ℚ} {B : FinRV n Bool} {L : FinRV n (Fin k)} +variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} +variable {k : ℕ} {X : FinRV Ω ℚ} {B : FinRV Ω Bool} {L : FinRV Ω (Fin k)} variable (g : Fin k → ℚ) /-- LOTUS: The law of the unconscious statistician (or similar) -/ @@ -502,8 +497,10 @@ theorem law_total_exp : 𝔼[𝔼[X |ᵣ L // P] // P] = 𝔼[X // P] := _ = 𝔼[X // P] := by rw [←exp_decompose] --- shows that using a set and list is the same -lemma finset_image_eq_list_map_dedup : ∀x, x ∈ Finset.univ.image X ↔ x ∈ (List.ofFn X |> List.dedup) := by - intro x; constructor <;> simp [Fin.univ_image_def,List.mem_toFinset] +lemma finset_image_eq_list_map_dedup : ∀x, x ∈ Finset.univ.image X ↔ x ∈ (((FinEnum.toList Ω).map X) |> List.dedup) := by + intro x + simp only [Finset.mem_image, Finset.mem_univ, true_and, List.mem_dedup, List.mem_map, + FinEnum.mem_toList, true_and] lemma finset_list_eq_list_dedup (l : List ℚ) : l.toFinset = l.dedup.toFinset := @@ -517,32 +514,37 @@ section RV_Unique_Values variable {τ:Type} [DecidableEq τ] -def FinRV.imageList (X : FinRV n τ) : List τ := List.dedup (List.ofFn X) +/-- The distinct values of a random variable, as a deduplicated list built from the + enumeration of the sample space. -/ +def FinRV.imageList (X : FinRV Ω τ) : List τ := List.dedup ((FinEnum.toList Ω).map X) + +/-- The image finset of `X` equals the `toFinset` of its `imageList`. -/ +theorem univ_image_eq_imageList_toFinset (X : FinRV Ω τ) : + Finset.univ.image X = X.imageList.toFinset := by + ext y + simp only [FinRV.imageList, Finset.mem_image, Finset.mem_univ, true_and, List.mem_toFinset, + List.mem_dedup, List.mem_map, FinEnum.mem_toList, true_and] -theorem sum_finset_eq_sum_image (f : ℚ → ℚ) : - (∑ y ∈ (Finset.univ.image X), f y) = ((X.imageList).map f).sum := by - rw [FinRV.imageList, Fin.univ_image_def, Finset.sum_list_map_count,finset_list_eq_list_dedup] - have h : ∀m ∈ X.imageList.toFinset, List.count m (List.ofFn X).dedup = 1 := - fun m hm => List.count_eq_one_of_mem (List.nodup_dedup (List.ofFn X)) - (List.mem_dedup.mp hm) - apply Finset.sum_congr rfl - intro x hx - simp [h x hx] +theorem sum_finset_eq_sum_image (f : ℚ → ℚ) : + (∑ y ∈ (Finset.univ.image X), f y) = ((X.imageList).map f).sum := by + rw [univ_image_eq_imageList_toFinset] + exact List.sum_toFinset f (List.nodup_dedup _) section generic -variable {X : FinRV n τ} +variable {X : FinRV Ω τ} -theorem finrv_image_superset (ω : Fin n) : X ω ∈ X.imageList := by - simp only [FinRV.imageList, List.mem_dedup, List.mem_ofFn.mpr (exists_apply_eq_apply X ω)] +theorem finrv_image_superset (ω : Ω) : X ω ∈ X.imageList := by + simp only [FinRV.imageList, List.mem_dedup, List.mem_map] + exact ⟨ω, FinEnum.mem_toList ω, rfl⟩ theorem finrv_image_superset_exists (ω) : ∃ i : Fin X.imageList.length, X ω = X.imageList[i] := List.exists_mem_iff_get.mp ⟨X ω, ⟨finrv_image_superset ω, rfl⟩⟩ -theorem finrv_image_nodup : X.imageList.Nodup := List.nodup_dedup (List.ofFn X) +theorem finrv_image_nodup : X.imageList.Nodup := List.nodup_dedup _ -- Mathlib seems to be missing this function def List.finIdxOf (L : List τ) (a : τ) (h : a ∈ L) : Fin L.length := @@ -552,11 +554,11 @@ def List.finIdxOf (L : List τ) (a : τ) (h : a ∈ L) : Fin L.length := theorem List.getElem_finIdxOf (L : List τ) (a : τ) (h : a ∈ L) : L[L.finIdxOf a h] = a := getElem_idxOf (idxOf_lt_length_of_mem h) -def FinRV.imageIdxOf (X : FinRV n τ) (ω : Fin n) : Fin (X.imageList.length) := +def FinRV.imageIdxOf (X : FinRV Ω τ) (ω : Ω) : Fin (X.imageList.length) := X.imageList.finIdxOf (X ω) (finrv_image_superset ω) @[simp] -theorem finrv_image_inverse (ω : Fin n) : X.imageList[X.imageIdxOf ω] = X ω := +theorem finrv_image_inverse (ω : Ω) : X.imageList[X.imageIdxOf ω] = X ω := List.getElem_finIdxOf X.imageList (X ω) (finrv_image_superset ω) theorem finrv_image_unique {ω i} (h: X ω = X.imageList[i]) : X.imageIdxOf ω = i := by @@ -598,7 +600,7 @@ end Expectation section Probability -variable {n : ℕ} {k : ℕ} {L : FinRV n (Fin k)} +variable {Ω : Type} [FinEnum Ω] {k : ℕ} {L : FinRV Ω (Fin k)} /-- The law of total probabilities -/ theorem law_of_total_probs : ℙ[B // P] = ∑ i, ℙ[B * (L =ᵣ i) // P] := by @@ -617,22 +619,22 @@ end Probability section Probability_Permutation -variable {n : ℕ} {P : Findist n} {A B : FinRV n Bool} {X Y : FinRV n ℚ} {t : ℚ} +variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {A B : FinRV Ω Bool} {X Y : FinRV Ω ℚ} {t : ℚ} -example (σ : Equiv.Perm (Fin n)) (f g : Fin n → ℚ) : f ⬝ᵥ g = (f ∘ σ) ⬝ᵥ (g ∘ σ) := +example (σ : Equiv.Perm (Ω)) (f g : Ω → ℚ) : f ⬝ᵥ g = (f ∘ σ) ⬝ᵥ (g ∘ σ) := by exact Eq.symm (comp_equiv_dotProduct_comp_equiv f g σ) -example (σ : Equiv.Perm (Fin n)) : (1 : Fin n → ℚ) = (1 : Fin n → ℚ) ∘ σ := rfl +example (σ : Equiv.Perm (Ω)) : (1 : Ω → ℚ) = (1 : Ω → ℚ) ∘ σ := rfl -def Findist.perm (P : Findist n) (σ : Equiv.Perm (Fin n)) : Findist n where +def Findist.perm (P : Findist Ω) (σ : Equiv.Perm (Ω)) : Findist Ω where p := P.p ∘ σ prob := by - have h1 : 1 = (1 : Fin n → ℚ) ∘ σ := rfl + have h1 : 1 = (1 : Ω → ℚ) ∘ σ := rfl rw [h1, comp_equiv_dotProduct_comp_equiv 1 P.p σ] exact P.prob nneg := fun ω => P.nneg (σ ω) -variable (σ : Equiv.Perm (Fin n)) +variable (σ : Equiv.Perm (Ω)) theorem exp_eq_perm : 𝔼[X ∘ σ // P.perm σ] = 𝔼[X // P] := by unfold expect Findist.perm diff --git a/MDPLib/Probability/Defs.lean b/MDPLib/Probability/Defs.lean index 482dc8c..452040c 100644 --- a/MDPLib/Probability/Defs.lean +++ b/MDPLib/Probability/Defs.lean @@ -1,5 +1,6 @@ import MDPLib.Probability.Prelude +import Mathlib.Data.FinEnum -- finitely-enumerable sample spaces import Mathlib.Data.Matrix.Mul -- dot product definitions and results import Mathlib.Algebra.Notation.Pi.Defs -- operations on functions import Mathlib.Algebra.Module.PointwisePi -- for smul_pi @@ -8,12 +9,6 @@ import Mathlib.LinearAlgebra.Matrix.DotProduct -- for monotonicity ----------- Generic results ----------------- -theorem false_of_le_gt {x y : ℚ} : x ≤ y → x > y → False := - by intro h1 h2; grw [h1] at h2; exact (lt_self_iff_false y).mp h2 - -theorem false_of_lt_ge {x y : ℚ} : x < y → x ≥ y → False := - fun h1 h2 => false_of_le_gt h2 h1 - theorem bool_ineq {a b : Bool} (h : a → b) : (a ≤ b) := h theorem bool_eq {a b : Bool} (h1 : a → b) (h2 : b → a) : a = b := Bool.le_antisymm h1 h2 @@ -21,12 +16,12 @@ theorem bool_eq {a b : Bool} (h1 : a → b) (h2 : b → a) : a = b := Bool.le_an --------------------------- Findist --------------------------------------------------------------- -variable {n : ℕ} +variable {Ω : Type} [FinEnum Ω] -/-- Finite probability distribution -/ -structure Findist (n : ℕ) : Type where +/-- Finite probability distribution over a finitely-enumerable sample space `Ω`. -/ +structure Findist (Ω : Type) [FinEnum Ω] : Type where /-- Probaiblity measure -/ - p : Fin n → ℚ + p : Ω → ℚ prob : 1 ⬝ᵥ p = 1 nneg : 0 ≤ p @@ -34,26 +29,27 @@ structure Findist (n : ℕ) : Type where namespace Findist /-- Finite probability distribution -/ -abbrev Delta : ℕ → Type := Findist +abbrev Delta (Ω : Type) [FinEnum Ω] : Type := Findist Ω /-- Finite probability distribution -/ -abbrev Δ : ℕ → Type := Delta - -/-- Single probability distribution -/ -def singleton : Findist 1 := - {p := ![1], - prob := by simp [Matrix.vecHead], - nneg := by simp [Pi.zero_def, Pi.le_def] } - +abbrev Δ (Ω : Type) [FinEnum Ω] : Type := Delta Ω -variable {n : ℕ} +/-- Dirac (point mass) distribution concentrated at `ω₀`. -/ +def dirac (ω₀ : Ω) : Findist Ω where + p := fun ω => if ω = ω₀ then 1 else 0 + prob := by simp [dotProduct] + nneg := by intro ω; by_cases h : ω = ω₀ <;> simp [h] -theorem nonempty (P : Findist n) : n > 0 := - by cases n - · have := P.prob; simp_all only [Matrix.dotProduct_of_isEmpty, zero_ne_one] - · simp only [gt_iff_lt, lt_add_iff_pos_left, add_pos_iff, zero_lt_one, or_true] +/-- The sample space of a probability distribution is nonempty. -/ +theorem nonempty (P : Findist Ω) : Nonempty Ω := by + by_contra h + rw [not_nonempty_iff] at h + have := P.prob + simp_all only [Matrix.dotProduct_of_isEmpty, zero_ne_one] -theorem nonempty' (P : Findist 0) : False := by have h := P.nonempty; simp only [gt_iff_lt, lt_self_iff_false] at h +/-- A distribution over an empty sample space is impossible. -/ +theorem nonempty' [IsEmpty Ω] (P : Findist Ω) : False := + (not_nonempty_iff.mpr ‹_›) P.nonempty end Findist @@ -81,13 +77,15 @@ Main results section RandomVariable -/-- A finite random variable -/ -abbrev FinRV (n : ℕ) (ρ : Type) := Fin n → ρ +/-- A finite random variable: a bare function from the sample space `Ω` to `ρ`. + The `FinEnum` instance lives on `Ω` and is shared with the distribution, so + a random variable never carries its own enumeration. -/ +abbrev FinRV (Ω : Type) (ρ : Type) := Ω → ρ /- construct a random variable -/ --- def rvOf {n : ℕ} {ρ : Type} (f : Fin n → ρ) := f +-- def rvOf {Ω : Type} {ρ : Type} (f : Ω → ρ) := f -variable {n : ℕ} {ρ : Type} +variable {Ω : Type} [FinEnum Ω] {ρ : Type} namespace FinRV @@ -106,28 +104,28 @@ variable {A B : Bool} /-- Negates a random variable -/ -@[simp] def not (B : FinRV n Bool) : FinRV n Bool := +@[simp] def not (B : FinRV Ω Bool) : FinRV Ω Bool := fun ω ↦ (B ω).not /-- Negates a random variable -/ prefix:40 "¬ᵣ" => FinRV.not /-- Boolean random variable representing an quality condition -/ -@[simp] def eq [DecidableEq ρ] (Y : FinRV n ρ) (y : ρ) : FinRV n Bool := +@[simp] def eq [DecidableEq ρ] (Y : FinRV Ω ρ) (y : ρ) : FinRV Ω Bool := (fun ω ↦ decide (Y ω = y) ) /-- Boolean random variable representing an quality condition -/ infix:50 "=ᵣ" => FinRV.eq /-- 0/1 random variable representing an quality condition -/ -@[simp] def eqi [DecidableEq ρ] (Y : FinRV n ρ) (y : ρ) : FinRV n ℚ := +@[simp] def eqi [DecidableEq ρ] (Y : FinRV Ω ρ) (y : ρ) : FinRV Ω ℚ := (fun ω ↦ if Y ω = y then 1 else 0) /-- 0/1 random variable representing an quality condition -/ infix:50 "=ᵢ" => FinRV.eqi /-- Boolean random variable represening Y ≤ y inequality -/ -@[simp] def leq [LE ρ] [DecidableLE ρ] (Y : FinRV n ρ) (y : ρ) : FinRV n Bool := +@[simp] def leq [LE ρ] [DecidableLE ρ] (Y : FinRV Ω ρ) (y : ρ) : FinRV Ω Bool := (fun ω ↦ Y ω ≤ y) /-- Boolean random variable represening Y ≤ y inequality -/ @@ -135,21 +133,21 @@ infix:50 "≤ᵣ" => FinRV.leq /-- Boolean random variable represening Y ≤ y inequality -/ -@[simp] def lt [LT ρ] [DecidableLT ρ] (Y : FinRV n ρ) (y : ρ) : FinRV n Bool := +@[simp] def lt [LT ρ] [DecidableLT ρ] (Y : FinRV Ω ρ) (y : ρ) : FinRV Ω Bool := (fun ω ↦ Y ω < y) /-- Boolean random variable represening Y ≤ y inequality -/ infix:50 "<ᵣ" => FinRV.lt /-- Boolean random variable represening Y ≤ y inequality -/ -@[simp] def geq [LE ρ] [DecidableLE ρ] (Y : FinRV n ρ) (y : ρ) : FinRV n Bool := +@[simp] def geq [LE ρ] [DecidableLE ρ] (Y : FinRV Ω ρ) (y : ρ) : FinRV Ω Bool := (fun ω ↦ Y ω ≥ y) /-- Boolean random variable represening Y ≤ y inequality -/ infix:50 "≥ᵣ" => FinRV.geq /-- Boolean random variable represening Y > y inequality -/ -@[simp] def gt [LT ρ] [DecidableLT ρ] (Y : FinRV n ρ) (y : ρ) : FinRV n Bool := +@[simp] def gt [LT ρ] [DecidableLT ρ] (Y : FinRV Ω ρ) (y : ρ) : FinRV Ω Bool := fun ω ↦ Y ω > y /-- Boolean random variable represening Y > y inequality -/ @@ -161,7 +159,7 @@ lemma exclusion {a b : ℕ} (h : a > b + 1) : (a > b) ∧ ¬(a = b + 1) := ⟨ Nat.lt_of_succ_lt h, Ne.symm (Nat.ne_of_lt h) ⟩ /-- Equivalence when extending the random variable to another element. -/ -theorem le_of_le_eq (D : FinRV n ℕ) (m : ℕ) : ((D ≤ᵣ m) + (D =ᵣ m.succ)) = (D ≤ᵣ m.succ) := by +theorem le_of_le_eq (D : FinRV Ω ℕ) (m : ℕ) : ((D ≤ᵣ m) + (D =ᵣ m.succ)) = (D ≤ᵣ m.succ) := by funext x --extensionality principle for functions unfold FinRV.leq FinRV.eq instHAdd Add.add Pi.instAdd rw [Pi.add_apply, bool_sum_or] @@ -170,8 +168,8 @@ theorem le_of_le_eq (D : FinRV n ℕ) (m : ℕ) : ((D ≤ᵣ m) + (D =ᵣ m.succ · simp [h, exclusion (Nat.not_le.mp h) ] /-- Defines a preimage of an RV. This is a set with a decidable membership. -/ -def preimage (f : FinRV n ρ) : ρ → Set (Fin n) := - fun t => { m : Fin n | f m = t} +def preimage (f : FinRV Ω ρ) : ρ → Set Ω := + fun t => { m : Ω | f m = t} end FinRV @@ -182,12 +180,12 @@ def indicator [OfNat ℚ 0] [OfNat ℚ 1] (cond : Bool) : ℚ := cond.rec 0 1 abbrev 𝕀 [OfNat ℚ 0] [OfNat ℚ 1] : Bool → ℚ := indicator -variable {k : ℕ} {L : FinRV n (Fin k)} +variable {k : ℕ} {L : FinRV Ω (Fin k)} theorem indi_eq_indr : ∀i : Fin k, (𝕀 ∘ (L =ᵣ i)) = (L =ᵢ i) := by intro i; unfold FinRV.eq FinRV.eqi 𝕀 indicator; ext ω; by_cases h: L ω = i; repeat simp [h] -variable {B : FinRV n Bool} +variable {B : FinRV Ω Bool} /-- Indicator is 0 or 1 -/ theorem ind_zero_one : ∀ ω, (𝕀∘B) ω = 1 ∨ (𝕀∘B) ω = 0 := by intro ω @@ -196,27 +194,27 @@ theorem ind_zero_one : ∀ ω, (𝕀∘B) ω = 1 ∨ (𝕀∘B) ω = 0 := by · right; simp only [Function.comp_apply, h, indicator] /-- Indicator is 0 or 1 -/ -theorem ind_nneg : (0 : FinRV n ℚ) ≤ 𝕀∘B := by +theorem ind_nneg : (0 : FinRV Ω ℚ) ≤ 𝕀∘B := by intro ω; unfold 𝕀 indicator; by_cases h : B ω; repeat simp [h] -theorem ind_le_one : 𝕀∘B ≤ (1 : FinRV n ℚ) := +theorem ind_le_one : 𝕀∘B ≤ (1 : FinRV Ω ℚ) := by unfold 𝕀 indicator; intro ω; by_cases h : B ω; repeat simp [h] -variable {c : ℚ} {X : FinRV n ℚ} +variable {c : ℚ} {X : FinRV Ω ℚ} -theorem rv_const_fun_to_one : (fun _ ↦ c : FinRV n ℚ) = c • 1 := by ext; simp; +theorem rv_const_fun_to_one : (fun _ ↦ c : FinRV Ω ℚ) = c • 1 := by ext; simp; -theorem rv_decompose (X : FinRV n ℚ) (L : FinRV n (Fin k)) : X = ∑ i, X * (L =ᵢ i) := by ext ω; simp +theorem rv_decompose (X : FinRV Ω ℚ) (L : FinRV Ω (Fin k)) : X = ∑ i, X * (L =ᵢ i) := by ext ω; simp -theorem one_of_true : 𝕀 ∘ (1 : Fin n → Bool) = (1 : Fin n → ℚ) := by ext; simp [𝕀, indicator] +theorem one_of_true : 𝕀 ∘ (1 : Ω → Bool) = (1 : Ω → ℚ) := by ext; simp [𝕀, indicator] -theorem one_of_bool_or_not : B + (¬ᵣ B) = (1 : FinRV n Bool) := by ext ω; unfold FinRV.not; simp +theorem one_of_bool_or_not : B + (¬ᵣ B) = (1 : FinRV Ω Bool) := by ext ω; unfold FinRV.not; simp -theorem one_of_ind_bool_or_not : (𝕀∘B) + (𝕀∘(¬ᵣ B)) = (1 : FinRV n ℚ) := +theorem one_of_ind_bool_or_not : (𝕀∘B) + (𝕀∘(¬ᵣ B)) = (1 : FinRV Ω ℚ) := by ext ω; unfold FinRV.not 𝕀 indicator not by_cases h : B ω <;> simp [h] -variable {X Y: FinRV n ℚ} {Xs : Fin k → FinRV n ℚ} +variable {X Y: FinRV Ω ℚ} {Xs : Fin k → FinRV Ω ℚ} theorem rv_le_abs : X ≤ abs ∘ X := by intro i; simp [le_abs_self (X i)] @@ -230,22 +228,21 @@ theorem rv_prod_const : ∀i, (g ∘ L) * (L =ᵢ i) = (g i) • (L =ᵢ i) := variable {β : Type} -theorem rv_image_nonempty [DecidableEq β] [LinearOrder β] (P : Findist n) (X : FinRV n β) : +theorem rv_image_nonempty [DecidableEq β] [LinearOrder β] (P : Findist Ω) (X : FinRV Ω β) : (Finset.univ.image X).Nonempty := - match n with - | Nat.zero => P.nonempty' |> False.elim - | Nat.succ _ => Finset.image_nonempty.mpr Finset.univ_nonempty + have : Nonempty Ω := P.nonempty + Finset.image_nonempty.mpr Finset.univ_nonempty -def FinRV.min [DecidableEq β] [LinearOrder β] (P : Findist n) (X : FinRV n β) : β := +def FinRV.min [DecidableEq β] [LinearOrder β] (P : Findist Ω) (X : FinRV Ω β) : β := (Finset.univ.image X).min' (rv_image_nonempty P X) -def FinRV.max [DecidableEq β] [LinearOrder β] (P : Findist n) (X : FinRV n β) : β := +def FinRV.max [DecidableEq β] [LinearOrder β] (P : Findist Ω) (X : FinRV Ω β) : β := (Finset.univ.image X).max' (rv_image_nonempty P X) -variable {X : FinRV n ℚ} +variable {X : FinRV Ω ℚ} -theorem rv_omega_le_max (P : Findist n) : ∀ω, X ω ≤ (FinRV.max P X) := by +theorem rv_omega_le_max (P : Findist Ω) : ∀ω, X ω ≤ (FinRV.max P X) := by intro ω have h : X ω ∈ (Finset.image X Finset.univ) := Finset.mem_image_of_mem X (Finset.mem_univ ω) simpa using Finset.le_max' (Finset.image X Finset.univ) (X ω) h @@ -256,7 +253,7 @@ end RandomVariable ------------------------------ Probability --------------------------- section Probability -variable {n : ℕ} (P : Findist n) (B C : FinRV n Bool) +variable {Ω : Type} [FinEnum Ω] (P : Findist Ω) (B C : FinRV Ω Bool) /-- Probability of B -/ def probability : ℚ := P.p ⬝ᵥ (𝕀 ∘ B) @@ -280,7 +277,7 @@ theorem prob_one_of_true : ℙ[1 // P] = 1 := example {a b : ℚ} (h : 0 ≤ a) (h2 : 0 ≤ b) : 0 ≤ a * b := Rat.mul_nonneg h h2 -variable {P : Findist n} {B : FinRV n Bool} +variable {P : Findist Ω} {B : FinRV Ω Bool} theorem prod_zero_of_prob_zero : ℙ[B // P] = 0 → (P.p * (𝕀∘B) = 0) := by intro h; exact prod_eq_zero_of_nneg_dp_zero P.nneg ind_nneg h @@ -288,16 +285,16 @@ theorem prod_zero_of_prob_zero : ℙ[B // P] = 0 → (P.p * (𝕀∘B) = 0) := b ------------------------------ PMF --------------------------- /-- Proof that p is a the PMF of X on probability space P -/ -def PMF {K : ℕ} (pmf : Fin K → ℚ) (P : Findist n) (L : FinRV n (Fin K)) := +def PMF {K : ℕ} (pmf : Fin K → ℚ) (P : Findist Ω) (L : FinRV Ω (Fin K)) := ∀ k : Fin K, pmf k = ℙ[ L =ᵣ k // P] -variable {n : ℕ} {k : ℕ} {L : FinRV n (Fin k)} -variable {pmf : Fin k → ℚ} {P : Findist n} +variable {Ω : Type} [FinEnum Ω] {k : ℕ} {L : FinRV Ω (Fin k)} +variable {pmf : Fin k → ℚ} {P : Findist Ω} theorem pmf_rv_k_ge_1 (h : PMF pmf P L) : 0 < k := match k with - | Nat.zero => Fin.pos <| L ⟨0,P.nonempty⟩ + | Nat.zero => Fin.pos <| L P.nonempty.some | Nat.succ k₂ => Nat.zero_lt_succ k₂ end Probability @@ -306,11 +303,11 @@ end Probability section CDF -variable {n : ℕ} +variable {Ω : Type} [FinEnum Ω] -def cdf (P : Findist n) (X : FinRV n ℚ) (t : ℚ) : ℚ := ℙ[X ≤ᵣ t // P] +def cdf (P : Findist Ω) (X : FinRV Ω ℚ) (t : ℚ) : ℚ := ℙ[X ≤ᵣ t // P] -variable {P : Findist n} {X Y : FinRV n ℚ} {t t₁ t₂ : ℚ} +variable {P : Findist Ω} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} end CDF @@ -326,7 +323,7 @@ Main results - Decomposition with a discrete random variables, used in the proofs of LOTUS and TLE -/ -variable {n : ℕ} (P : Findist n) (X Y Z: FinRV n ℚ) (B : FinRV n Bool) +variable {Ω : Type} [FinEnum Ω] (P : Findist Ω) (X Y Z: FinRV Ω ℚ) (B : FinRV Ω Bool) /-- Standard expectation operator -/ def expect : ℚ := P.p ⬝ᵥ X @@ -344,10 +341,10 @@ def expect_cnd : ℚ := 𝔼[ X * (𝕀 ∘ B) // P] / ℙ[ B // P] /-- Conditional expectation operator -/ notation "𝔼[" X "|" B "//" P "]" => expect_cnd P X B -variable {k : ℕ} (L : FinRV n (Fin k)) +variable {k : ℕ} (L : FinRV Ω (Fin k)) /-- Expectation conditioned on a random variable. It creates a random variable -/ -def expect_cnd_rv : Fin n → ℚ := fun i ↦ 𝔼[ X | L =ᵣ (L i) // P ] +def expect_cnd_rv : Ω → ℚ := fun i ↦ 𝔼[ X | L =ᵣ (L i) // P ] /-- Expectation conditioned on a random variable. It creates a random variable -/ notation "𝔼[" X "|ᵣ" L "//" P "]" => expect_cnd_rv P X L @@ -355,7 +352,7 @@ notation "𝔼[" X "|ᵣ" L "//" P "]" => expect_cnd_rv P X L --- some basic properties section Expectation_properties -variable {P : Findist n} {X Y Z: FinRV n ℚ} {B : FinRV n Bool} +variable {P : Findist Ω} {X Y Z: FinRV Ω ℚ} {B : FinRV Ω Bool} theorem exp_congr : (X = Y) → 𝔼[X // P] = 𝔼[Y // P] := by intro h @@ -366,7 +363,7 @@ theorem exp_congr : (X = Y) → 𝔼[X // P] = 𝔼[Y // P] := theorem exp_mul_comm : 𝔼[X * Y // P] = 𝔼[Y * X // P] := exp_congr (CommMonoid.mul_comm X Y) -variable {c : ℚ} {p : Fin n → ℚ} +variable {c : ℚ} {p : Ω → ℚ} theorem exp_const : 𝔼[(fun _ ↦ c) // P] = c := by unfold expect @@ -396,7 +393,7 @@ theorem exp_indi_eq_exp_indr : ∀i : Fin k, 𝔼[L =ᵢ i // P] = 𝔼[𝕀 ∘ theorem exp_homogenous : 𝔼[c • X // P] = c * 𝔼[X // P] := by simp only [expect, dotProduct_smul, smul_eq_mul] /-- Additivity of expectation --/ -theorem exp_additive {m : ℕ} (Xs : Fin m → FinRV n ℚ) : 𝔼[∑ i : Fin m, Xs i // P] = ∑ i : Fin m, 𝔼[Xs i // P] := +theorem exp_additive {m : ℕ} (Xs : Fin m → FinRV Ω ℚ) : 𝔼[∑ i : Fin m, Xs i // P] = ∑ i : Fin m, 𝔼[Xs i // P] := by unfold expect; exact dotProduct_sum P.p Finset.univ Xs theorem exp_additive_two : 𝔼[X + Y // P] = 𝔼[X // P] + 𝔼[Y // P] := by simp [expect] @@ -406,7 +403,7 @@ theorem exp_monotone (h: X ≤ Y) : 𝔼[X // P] ≤ 𝔼[Y // P] := dotProduct ---- ** conditional expectation ----- -variable {k : ℕ} {g : Fin k → ℚ} {L : FinRV n (Fin k)} +variable {k : ℕ} {g : Fin k → ℚ} {L : FinRV Ω (Fin k)} theorem exp_decompose : 𝔼[X // P] = ∑ i, 𝔼[X * (L =ᵢ i) // P] := by nth_rewrite 1 [rv_decompose X L] @@ -425,7 +422,7 @@ end Expectation_properties -- Derived properties from the properties of expectation section Probability_properties -variable {n : ℕ} {P : Findist n} {A B : FinRV n Bool} +variable {n : ℕ} {P : Findist Ω} {A B : FinRV Ω Bool} theorem ind_monotone : (∀ ω, A ω → B ω) → (𝕀∘A) ≤ (𝕀∘B) := by intro h ω diff --git a/MDPLib/Probability/Matrix.lean b/MDPLib/Probability/Matrix.lean index e6942da..d0002a7 100644 --- a/MDPLib/Probability/Matrix.lean +++ b/MDPLib/Probability/Matrix.lean @@ -1,6 +1,6 @@ import MDPLib.Probability.Prelude +import MDPLib.Probability.Defs -import Probability.Probability.Defs import Mathlib.Data.Matrix.Mul import Mathlib.LinearAlgebra.Matrix.DotProduct @@ -8,13 +8,15 @@ namespace Matrix section ProbabilityMatrix -structure ProbabilityMatrix (n : ℕ) : Type where - --(n x n) Matrix where each row is a probability distribution - P : (Matrix (Fin n) (Fin n) ℚ) +variable {Ω : Type} [FinEnum Ω] + +structure ProbabilityMatrix (Ω : Type) [FinEnum Ω] : Type where + -- Square matrix over `Ω` where each row is a probability distribution + P : (Matrix Ω Ω ℚ) row_sum : P *ᵥ 1 = 1 - nneg : ∀ i j : Fin n, P i j ≥ 0 + nneg : ∀ i j : Ω, P i j ≥ 0 -variable (n : ℕ) (Prob : ProbabilityMatrix n) (μ : Findist n) (r : Fin n → ℚ) (γ : ℚ) +variable (Prob : ProbabilityMatrix Ω) (μ : Findist Ω) (r : Ω → ℚ) (γ : ℚ) theorem dist_prob_product_nneg : μ.p ᵥ* (Prob.P) ≥ 0 := by @@ -34,16 +36,18 @@ end ProbabilityMatrix section RewardProcess +variable {Ω : Type} [FinEnum Ω] + --Discounted Markov Reward Process Definition -structure DMRP (n : ℕ) : Type where - r : Fin n → ℚ --rewards - Prob : ProbabilityMatrix n --transitions +structure DMRP (Ω : Type) [FinEnum Ω] : Type where + r : Ω → ℚ --rewards + Prob : ProbabilityMatrix Ω --transitions γ : ℚ --discount discount_in_range : 0 ≤ γ ∧ γ < 1 -variable (n : ℕ) (Proc : DMRP n) (u : (Fin n) → ℚ) (v : (Fin n) → ℚ) +variable (Proc : DMRP Ω) (u : Ω → ℚ) (v : Ω → ℚ) -def bellman_backup (v : Fin n → ℚ) : Fin n → ℚ := Proc.r + Proc.γ • Proc.Prob.P *ᵥ v +def bellman_backup (v : Ω → ℚ) : Ω → ℚ := Proc.r + Proc.γ • Proc.Prob.P *ᵥ v notation "𝔹["v "//" Proc "]" => bellman_backup Proc v diff --git a/MDPLib/Probability/Prelude.lean b/MDPLib/Probability/Prelude.lean index 8ce1d28..728e349 100644 --- a/MDPLib/Probability/Prelude.lean +++ b/MDPLib/Probability/Prelude.lean @@ -51,7 +51,8 @@ end FunctionalAnalysis section dotProduct -variable {x y z : Fin n → ℚ} +variable {Ω : Type*} [Fintype Ω] +variable {x y z : Ω → ℚ} theorem dotProd_hadProd_rotate : x ⬝ᵥ (y * z) = z ⬝ᵥ (x * y) := by unfold dotProduct @@ -82,9 +83,9 @@ theorem prod_eq_zero_of_nneg_dp_zero (hx : 0 ≤ x) (hy : 0 ≤ y) : x ⬝ᵥ y theorem abs_pos_hom {a b : ℚ} (h : 0 ≤ a) : |a * b| = a * |b| := by rw [abs_mul, abs_of_nonneg h] -theorem abs_dotProd_le_dotProd_abs(p x : Fin n → ℚ) (hp : ∀ i, 0 ≤ p i) : |p ⬝ᵥ x| ≤ p ⬝ᵥ fun i => |x i| := by +theorem abs_dotProd_le_dotProd_abs(p x : Ω → ℚ) (hp : ∀ i, 0 ≤ p i) : |p ⬝ᵥ x| ≤ p ⬝ᵥ fun i => |x i| := by calc - |∑ i : Fin n, p i * x i| ≤ ∑ i, |p i * x i| := Finset.abs_sum_le_sum_abs (fun i ↦ p i * x i) Finset.univ + |∑ i : Ω, p i * x i| ≤ ∑ i, |p i * x i| := Finset.abs_sum_le_sum_abs (fun i ↦ p i * x i) Finset.univ _ = ∑ i, p i * |x i| := Finset.sum_congr rfl (fun i _ => abs_pos_hom (hp i)) _ = p ⬝ᵥ fun i => |x i| := rfl diff --git a/MDPLib/Probability/Quantile.lean b/MDPLib/Probability/Quantile.lean index d49ef34..e5aea19 100644 --- a/MDPLib/Probability/Quantile.lean +++ b/MDPLib/Probability/Quantile.lean @@ -12,7 +12,7 @@ section Definition --def UnitI := {α : ℚ // 0 ≤ α ∧ α ≤ 1} -variable {n : ℕ} (P : Findist n) (X Y : FinRV n ℚ) (α : ℚ) (q v : ℚ) +variable {Ω : Type} [FinEnum Ω] (P : Findist Ω) (X Y : FinRV Ω ℚ) (α : ℚ) (q v : ℚ) /-- Proof the `q` is an `α`-quantile of `X` --/ def IsQuantile : Prop := ℙ[X ≤ᵣ q // P ] ≥ α ∧ ℙ[X ≥ᵣ q // P] ≥ 1 - α @@ -34,7 +34,7 @@ def IsQuantMin : Prop := IsLeast (Quantile P X α) q end Definition -variable {n : ℕ} {P : Findist n} {X Y : FinRV n ℚ} {α : ℚ} {q v : ℚ} +variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {X Y : FinRV Ω ℚ} {α : ℚ} {q v : ℚ} theorem qset_lb : q ∈ Quantile P X α → ℙ[X ≤ᵣ q // P ] ≥ α := by simp_all [Quantile, IsQuantile] diff --git a/MDPLib/Risk/CVaR.lean b/MDPLib/Risk/CVaR.lean index 47c7ceb..3cd7722 100644 --- a/MDPLib/Risk/CVaR.lean +++ b/MDPLib/Risk/CVaR.lean @@ -4,20 +4,20 @@ import MDPLib.Risk.VaR namespace Risk /-- Tail indicator: 1 if X(ω) > t, else 0. -/ -def tailInd (X : FinRV n ℚ) (t : ℚ) : FinRV n ℚ := +def tailInd (X : FinRV Ω ℚ) (t : ℚ) : FinRV Ω ℚ := fun ω => if X ω > t then 1 else 0 -variable {n : ℕ} +variable {Ω : Type} [FinEnum Ω] /-- Conditional Value-at-Risk (CVaR) of X at level α under P. CVaR_α(X) = E[X * I[X > VaR] ] / P[X > VaR] If the tail probability is zero, CVaR is defined to be 0. -/ -def CVaR (P : Findist n) (X : FinRV n ℚ) (α : RiskLevel) : ℚ := +def CVaR (P : Findist Ω) (X : FinRV Ω ℚ) (α : RiskLevel) : ℚ := -- TODO: This CVaR computation is incorrect -- we could use the implementation from here: https://arxiv.org/abs/2607.05078 let v := Risk.FinVaR P X α - let B : FinRV n ℚ := tailInd X v + let B : FinRV Ω ℚ := tailInd X v let num := 𝔼[X * B // P] let den := ℙ[X >ᵣ v // P] if _ : den = 0 then diff --git a/MDPLib/Risk/VaR.lean b/MDPLib/Risk/VaR.lean index f134eb0..974d756 100644 --- a/MDPLib/Risk/VaR.lean +++ b/MDPLib/Risk/VaR.lean @@ -7,8 +7,8 @@ namespace Risk open Findist FinRV Statistic -variable {n : ℕ} -variable {P : Findist n} {X Y : FinRV n ℚ} {t t₁ t₂ : ℚ} +variable {Ω : Type} [FinEnum Ω] +variable {P : Findist Ω} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} def IsRiskLevel (α : ℚ) : Prop := 0 ≤ α ∧ α < 1 @@ -17,11 +17,11 @@ def RiskLevel := { α : ℚ // IsRiskLevel α} --instance instCoeRiskUnit : Coe RiskLevel UnitI where -- coe := fun ⟨v,c⟩ => ⟨v, ⟨c.1, le_of_lt c.2⟩ ⟩ -def FinVaRSet (P : Findist n) (X : FinRV n ℚ) (α : RiskLevel) : Finset ℚ := +def FinVaRSet (P : Findist Ω) (X : FinRV Ω ℚ) (α : RiskLevel) : Finset ℚ := let 𝓧 := Finset.univ.image X 𝓧.filter (fun t ↦ ℙ[X <ᵣ t // P] ≤ α.val) -theorem FinVarSet_nonempty (P : Findist n) (X : FinRV n ℚ) (α : RiskLevel) : (FinVaRSet (n := n) P X α).Nonempty := by +theorem FinVarSet_nonempty (P : Findist Ω) (X : FinRV Ω ℚ) (α : RiskLevel) : (FinVaRSet (Ω := Ω) P X α).Nonempty := by apply Finset.filter_nonempty_iff.mpr let xmin := (Finset.univ.image X).min' (rv_image_nonempty P X) use xmin @@ -33,7 +33,7 @@ theorem FinVarSet_nonempty (P : Findist n) (X : FinRV n ℚ) (α : RiskLevel) : /-- Value-at-Risk of X at level α: VaR_α(X) = min { t ∈ X(Ω) | P[X ≤ t] ≥ α }. If we assume 0 ≤ α < 1, then the "else 0" branch is never used. -/ -def FinVaR (P : Findist n) (X : FinRV n ℚ) (α : RiskLevel) : ℚ := +def FinVaR (P : Findist Ω) (X : FinRV Ω ℚ) (α : RiskLevel) : ℚ := let 𝓧 := Finset.univ.image X let 𝓢 := 𝓧.filter (fun t ↦ ℙ[X <ᵣ t // P] ≤ α.val) have h : 𝓢.Nonempty := FinVarSet_nonempty P X α @@ -56,11 +56,11 @@ theorem finvar_prob_cond : ℙ[X <ᵣ (FinVaR P X α) // P] ≤ α.val ∧ α.va rw [hqp] at hg have h2: q ∈ 𝓢 := Finset.mem_filter.mpr ⟨hqin, hg⟩ exact Finset.le_max' 𝓢 q h2 - exact false_of_lt_ge hqgt hqt + exact not_le_of_gt hqgt hqt notation "VaR[" X "//" P ", " α "]" => FinVaR P X α -variable {n : ℕ} (P : Findist n) (X Y : FinRV n ℚ) (α : RiskLevel) (q v : ℚ) +variable {Ω : Type} [FinEnum Ω] (P : Findist Ω) (X Y : FinRV Ω ℚ) (α : RiskLevel) (q v : ℚ) /-- Value `v` is the Value at Risk at `α` of `X` and probability `P` -/ def IsVaR_Q : Prop := IsGreatest (Quantile P X α.val) v @@ -68,7 +68,7 @@ def IsVaR_Q : Prop := IsGreatest (Quantile P X α.val) v /-- A simpler, equivalent definition of Value at Risk -/ def IsVaR : Prop := IsGreatest (QuantileLower P X α.val) v -variable {n : ℕ} {P : Findist n} {X Y : FinRV n ℚ} {α : RiskLevel} {q v q₁ q₂ : ℚ} +variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {X Y : FinRV Ω ℚ} {α : RiskLevel} {q v q₁ q₂ : ℚ} theorem var_prob_cond : IsVaR P X α v ↔ (ℙ[X <ᵣ v // P] ≤ α.val ∧ α.val < ℙ[X ≤ᵣ v // P]) := by constructor @@ -83,7 +83,7 @@ theorem var_prob_cond : IsVaR P X α v ↔ (ℙ[X <ᵣ v // P] ≤ α.val ∧ α rw [hq.2,prob_lt_of_ge] at hc suffices ℙ[X≥ᵣq//P] ≥ 1 - α.val from this linarith - exact false_of_le_gt (h.2 h3) hq.1 + exact not_lt_of_ge (h.2 h3) hq.1 · intro h constructor · exact qsetlower_of_cond_lt h.1 @@ -93,7 +93,7 @@ theorem var_prob_cond : IsVaR P X α v ↔ (ℙ[X <ᵣ v // P] ≤ α.val ∧ α have hu : ℙ[X ≤ᵣ v // P] ≤ α.val := calc ℙ[X ≤ᵣ v // P] ≤ ℙ[X <ᵣ q // P] := prob_lt_le_monotone hq.2 _ ≤ α.val := qsetlower_def_lt.mp hq.1 - exact false_of_lt_ge h.2 hu + exact not_le_of_gt h.2 hu -- This is the main correctness proof theorem finvar_correct : IsVaR P X α (FinVaR P X α) := var_prob_cond.mpr finvar_prob_cond @@ -130,8 +130,8 @@ theorem isquantilelower_le_isquantile : IsCofinalFor (QuantileLower P X α.val) constructor · exact hq₂ · by_contra! ine - exact ge_trans (prob_le_monotone (le_refl X) (le_of_lt ine)) (qset_lb hq₂) |> false_of_lt_ge h2l - · exfalso; exact false_of_lt_ge h2r h + exact ge_trans (prob_le_monotone (le_refl X) (le_of_lt ine)) (qset_lb hq₂) |> not_le_of_gt h2l + · exfalso; exact not_le_of_gt h2r h theorem isquantile_le_isquantilelower : IsCofinalFor (Quantile P X α.val) (QuantileLower P X α.val) := HasSubset.Subset.isCofinalFor quantile_subset_quantilelower @@ -147,7 +147,7 @@ theorem varq_prob_cond : IsVaR_Q P X α v ↔ (ℙ[X <ᵣ v // P] ≤ α.val ∧ section VaR_properties -variable {P : Findist n} {X Y : FinRV n ℚ} {q q₁ v₁ v₂ c : ℚ} {α : RiskLevel} {f : ℚ → ℚ} +variable {P : Findist Ω} {X Y : FinRV Ω ℚ} {q q₁ v₁ v₂ c : ℚ} {α : RiskLevel} {f : ℚ → ℚ} theorem var_monotone : X ≤ Y → IsVaR P X α v₁ → IsVaR P Y α v₂ → v₁ ≤ v₂ := fun hle hv1 hv2 => upperBounds_mono_of_isCofinalFor (quantile_le_monotone hle) hv2.2 hv1.1 From 2f35d9b0246b5bc98eeb594c776dfd838a3f5d80 Mon Sep 17 00:00:00 2001 From: Marek Petrik Date: Thu, 6 Aug 2026 10:58:43 -0400 Subject: [PATCH 2/9] updated mathlib version --- MDPLib/MDP/Histories.lean | 15 +++++----- MDPLib/Probability/Basic.lean | 6 ++-- MDPLib/Probability/Defs.lean | 2 +- lake-manifest.json | 56 +++++++++++++++++------------------ lean-toolchain | 2 +- 5 files changed, 41 insertions(+), 40 deletions(-) diff --git a/MDPLib/MDP/Histories.lean b/MDPLib/MDP/Histories.lean index 09a0835..2061e25 100644 --- a/MDPLib/MDP/Histories.lean +++ b/MDPLib/MDP/Histories.lean @@ -373,16 +373,17 @@ theorem hist_horiz_exact (t : ℕ) (h : Hist M) (hh : h ∈ M.HistoriesHorizon t unfold MDP.HistoriesHorizon state2hist_emb MDP.state2hist at hh rewrite [Finset.mem_map] at hh obtain ⟨s, sin, sf⟩ := hh - subst sf - simp only [Hist.length] + subst sf + rfl case succ t' ih => unfold MDP.HistoriesHorizon emb_tuple2hist MDP.tuple2hist at hh rw [Finset.mem_map] at hh - obtain ⟨has, hasi, em⟩ := hh - subst em - unfold Hist.length + obtain ⟨has, hasi, em⟩ := hh + subst em + obtain ⟨h', a, s⟩ := has rewrite [Finset.mem_product] at hasi - rewrite [ih has.1 hasi.1] + show 1 + h'.length = t' + 1 + rw [ih h' hasi.1] exact Nat.add_comm 1 t' def MDP.HistoriesHorizonT (M : MDP) (t : ℕ) : Finset (M.HistT t) := @@ -397,7 +398,7 @@ theorem hist_horiz_complete_t (t : ℕ) (h : M.HistT t) : h ∈ M.HistoriesHoriz extract_lets H f finj apply Finset.mem_map.mpr use ⟨h.1, hist_horiz_complete t h⟩ - simp [f] + exact ⟨Finset.mem_attach _ _, rfl⟩ instance (M : MDP) (t : ℕ) : Fintype (M.HistT t) where elems := M.HistoriesHorizonT t diff --git a/MDPLib/Probability/Basic.lean b/MDPLib/Probability/Basic.lean index f524e39..d98f4fb 100644 --- a/MDPLib/Probability/Basic.lean +++ b/MDPLib/Probability/Basic.lean @@ -84,7 +84,7 @@ theorem rvlt_monotone (h1 : X ≤ Y) (h2: t₁ ≤ t₂) : 𝕀 ∘ (Y <ᵣ t₁ theorem rv_le_max_one : (X ≤ᵣ (FinRV.max P X)) = 1 := by ext ω - unfold FinRV.leq FinRV.max + unfold FinRV.leq simpa using rv_omega_le_max P ω theorem rv_max_in_image : (FinRV.max P X) ∈ Finset.univ.image X := @@ -93,11 +93,11 @@ theorem rv_max_in_image : (FinRV.max P X) ∈ Finset.univ.image X := theorem rv_omega_ge_min (P : Findist Ω) : ∀ω, X ω ≥ (FinRV.min P X) := by intro ω have h : X ω ∈ (Finset.image X Finset.univ) := Finset.mem_image_of_mem X (Finset.mem_univ ω) - simpa using Finset.min'_le (Finset.image X Finset.univ) (X ω) h + exact Finset.min'_le (Finset.image X Finset.univ) (X ω) h theorem rv_ge_min_one : (X ≥ᵣ (FinRV.min P X)) = 1 := by ext ω - unfold FinRV.geq FinRV.min + unfold FinRV.geq simpa using rv_omega_ge_min P ω theorem rv_monotone_sharp {t₁ t₂ : ℚ} : t₁ < t₂ → ∀ ω, (X ≥ᵣ t₂) ω → (X >ᵣ t₁) ω := diff --git a/MDPLib/Probability/Defs.lean b/MDPLib/Probability/Defs.lean index 452040c..1569430 100644 --- a/MDPLib/Probability/Defs.lean +++ b/MDPLib/Probability/Defs.lean @@ -245,7 +245,7 @@ variable {X : FinRV Ω ℚ} theorem rv_omega_le_max (P : Findist Ω) : ∀ω, X ω ≤ (FinRV.max P X) := by intro ω have h : X ω ∈ (Finset.image X Finset.univ) := Finset.mem_image_of_mem X (Finset.mem_univ ω) - simpa using Finset.le_max' (Finset.image X Finset.univ) (X ω) h + exact Finset.le_max' (Finset.image X Finset.univ) (X ω) h end RandomVariable diff --git a/lake-manifest.json b/lake-manifest.json index b78750e..bc48131 100644 --- a/lake-manifest.json +++ b/lake-manifest.json @@ -5,7 +5,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "c7a9524911e4b02bb9c406800ad3bff568971837", + "rev": "5edf93c0f3d1f3d509c42c888931b49be5c06816", "name": "mathlib", "manifestFile": "lake-manifest.json", "inputRev": "master", @@ -15,7 +15,7 @@ "type": "git", "subDir": null, "scope": "", - "rev": "799154a6ee3cefd3c0c89a68a81911fb83b7755c", + "rev": "feb58b82ec2b9acfbaf71bda268acdbd0a44ccd4", "name": "«doc-gen4»", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -35,7 +35,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "293af9b2a383eed4d04d66b898d608d0a44b750f", + "rev": "123d15766ba49356c02ebad2a4462dfe12d79899", "name": "plausible", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -45,7 +45,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "c5d5b8fe6e5158def25cd28eb94e4141ad97c843", + "rev": "f5c090429dff3cf66cb65562526c9ea6e8edfbcb", "name": "LeanSearchClient", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -55,7 +55,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "cdab3938ccabbdb044be6896e251b5814bec932e", + "rev": "bb3469a87774349fe01898d8bf2fc6a1ce6411ca", "name": "importGraph", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -65,67 +65,67 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "2db6054a44326f8c0230ee0570e2ddb894816511", + "rev": "222c58dad7706a6e7cae46c0edd65ea881d3ee27", "name": "proofwidgets", "manifestFile": "lake-manifest.json", - "inputRev": "v0.0.98", + "inputRev": "main", "inherited": true, "configFile": "lakefile.lean"}, {"url": "https://github.com/leanprover-community/aesop", "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "f0c6e183ea26531e82773feb4b73ab6595ca17a5", + "rev": "7db8190085343afde2f5d2cdcc9bac719b6ec02c", "name": "aesop", "manifestFile": "lake-manifest.json", - "inputRev": "v4.30.0-rc2", + "inputRev": "master", "inherited": true, "configFile": "lakefile.toml"}, {"url": "https://github.com/leanprover-community/quote4", "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "1cc7e819b9b9bc1e87c9edcccb62e0269e00a809", + "rev": "ef42f8944eaf5b6cbfbe75d1917d824c7dd6cf33", "name": "Qq", "manifestFile": "lake-manifest.json", - "inputRev": "v4.30.0-rc2", + "inputRev": "master", "inherited": true, "configFile": "lakefile.toml"}, {"url": "https://github.com/leanprover-community/batteries", "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "5c57f3857ba81924a88b2cdf4f062e34ec04ff11", + "rev": "76e1c118b0700b4ceafe99532e887d6431625e1a", "name": "batteries", "manifestFile": "lake-manifest.json", - "inputRev": "v4.30.0-rc2", - "inherited": true, - "configFile": "lakefile.toml"}, - {"url": "https://github.com/leanprover/lean4-cli", - "type": "git", - "subDir": null, - "scope": "leanprover", - "rev": "13567aed1ac4f12aea9484178e07e51f8c9f7658", - "name": "Cli", - "manifestFile": "lake-manifest.json", - "inputRev": "v4.30.0-rc2", + "inputRev": "main", "inherited": true, "configFile": "lakefile.toml"}, {"url": "https://github.com/leanprover/leansqlite", "type": "git", "subDir": null, "scope": "", - "rev": "c7326b06de365308e008bdfe379909e44334a2e2", + "rev": "8daa8a16be0c62a75720fb61825edaadd188ac79", "name": "leansqlite", "manifestFile": "lake-manifest.json", "inputRev": "main", "inherited": true, "configFile": "lakefile.lean"}, + {"url": "https://github.com/leanprover/lean4-cli", + "type": "git", + "subDir": null, + "scope": "leanprover", + "rev": "1319485273bf87833fa472afbcefdedecb16b45f", + "name": "Cli", + "manifestFile": "lake-manifest.json", + "inputRev": "v4.33.0-rc2", + "inherited": true, + "configFile": "lakefile.toml"}, {"url": "https://github.com/fgdorais/lean4-unicode-basic", "type": "git", "subDir": null, "scope": "", - "rev": "05954ce1797e6bd6b414c916499fe6dda4a11702", + "rev": "7e46ff559eb084f0c4d16f847d7907c3fac78dd3", "name": "UnicodeBasic", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -135,17 +135,17 @@ "type": "git", "subDir": null, "scope": "", - "rev": "5d31b64fb703c5d77f6ef4d1fb958f9bdf1ea539", + "rev": "5706cc4847b6b8343c00bd4233bda65526b79e43", "name": "BibtexQuery", "manifestFile": "lake-manifest.json", - "inputRev": "nightly-testing", + "inputRev": "master", "inherited": true, "configFile": "lakefile.toml"}, {"url": "https://github.com/acmepjz/md4lean", "type": "git", "subDir": null, "scope": "", - "rev": "6a3fb240133bcb7e1a066fdc784b3fdc304e3fc5", + "rev": "31907cc18f48a95384f99cee5582c00fb39e0f67", "name": "MD4Lean", "manifestFile": "lake-manifest.json", "inputRev": "main", diff --git a/lean-toolchain b/lean-toolchain index 635bb95..cee6b00 100644 --- a/lean-toolchain +++ b/lean-toolchain @@ -1 +1 @@ -leanprover/lean4:v4.30.0-rc2 \ No newline at end of file +leanprover/lean4:v4.33.0-rc2 \ No newline at end of file From aaff9796fc88cf072fc9ff1fb1235fb4200bd001 Mon Sep 17 00:00:00 2001 From: Marek Petrik Date: Thu, 6 Aug 2026 18:08:02 -0400 Subject: [PATCH 3/9] Cleaned up some warnings regarding Finenum class and unnecessary lemmas --- MDPLib/Probability/Defs.lean | 35 +++++++++++++++++----------------- MDPLib/Probability/Matrix.lean | 3 --- MDPLib/Risk/VaR.lean | 8 ++++---- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/MDPLib/Probability/Defs.lean b/MDPLib/Probability/Defs.lean index 1569430..064545d 100644 --- a/MDPLib/Probability/Defs.lean +++ b/MDPLib/Probability/Defs.lean @@ -18,9 +18,12 @@ theorem bool_eq {a b : Bool} (h1 : a → b) (h2 : b → a) : a = b := Bool.le_an variable {Ω : Type} [FinEnum Ω] + +-- TODO: do we even need to assume that Ω is finitely enumerable when defining the probability space? + /-- Finite probability distribution over a finitely-enumerable sample space `Ω`. -/ structure Findist (Ω : Type) [FinEnum Ω] : Type where - /-- Probaiblity measure -/ + /-- Probability measure -/ p : Ω → ℚ prob : 1 ⬝ᵥ p = 1 nneg : 0 ≤ p @@ -78,14 +81,12 @@ Main results section RandomVariable /-- A finite random variable: a bare function from the sample space `Ω` to `ρ`. - The `FinEnum` instance lives on `Ω` and is shared with the distribution, so + The `FinEnum` instance lives on `Ω` and is shared with the distribution function, so a random variable never carries its own enumeration. -/ abbrev FinRV (Ω : Type) (ρ : Type) := Ω → ρ -/- construct a random variable -/ --- def rvOf {Ω : Type} {ρ : Type} (f : Ω → ρ) := f -variable {Ω : Type} [FinEnum Ω] {ρ : Type} +variable {Ω : Type} {ρ : Type} namespace FinRV @@ -153,14 +154,12 @@ infix:50 "≥ᵣ" => FinRV.geq /-- Boolean random variable represening Y > y inequality -/ infix:50 ">ᵣ" => FinRV.gt -example (a b : ℕ) (h : a ≤ b +1) : (a ≤ b) ∨ (a = b + 1) := by exact Nat.le_or_eq_of_le_succ h -lemma exclusion {a b : ℕ} (h : a > b + 1) : (a > b) ∧ ¬(a = b + 1) := - ⟨ Nat.lt_of_succ_lt h, Ne.symm (Nat.ne_of_lt h) ⟩ - -/-- Equivalence when extending the random variable to another element. -/ +/-- Equivalence when adding an element to integer comparison. -/ theorem le_of_le_eq (D : FinRV Ω ℕ) (m : ℕ) : ((D ≤ᵣ m) + (D =ᵣ m.succ)) = (D ≤ᵣ m.succ) := by - funext x --extensionality principle for functions + have exclusion {a b : ℕ} (h : a > b + 1) : (a > b) ∧ ¬(a = b + 1) := + ⟨ Nat.lt_of_succ_lt h, Ne.symm (Nat.ne_of_lt h) ⟩ + funext x unfold FinRV.leq FinRV.eq instHAdd Add.add Pi.instAdd rw [Pi.add_apply, bool_sum_or] by_cases h : D x ≤ m.succ @@ -228,6 +227,10 @@ theorem rv_prod_const : ∀i, (g ∘ L) * (L =ᵢ i) = (g i) • (L =ᵢ i) := variable {β : Type} + +-- assume enumerability of Ω from here because we need a probability space +variable [FinEnum Ω] + theorem rv_image_nonempty [DecidableEq β] [LinearOrder β] (P : Findist Ω) (X : FinRV Ω β) : (Finset.univ.image X).Nonempty := have : Nonempty Ω := P.nonempty @@ -261,9 +264,6 @@ def probability : ℚ := P.p ⬝ᵥ (𝕀 ∘ B) /-- Probability of B -/ notation "ℙ[" B "//" P "]" => probability P B --- helps to refold is when needed -lemma probability_def : P.p ⬝ᵥ (𝕀 ∘ B) = ℙ[B // P] := rfl - /-- Conditional probability of B on C -/ def probability_cnd : ℚ := ℙ[B * C // P] / ℙ[ C // P ] @@ -381,10 +381,9 @@ theorem exp_cond_eq_def : 𝔼[X | B // P] * ℙ[B // P] = 𝔼[X * (𝕀 ∘ B exact (dotProduct_zero X).symm · simp_all -lemma constant_mul_eq_smul : (fun ω ↦ c * X ω) = c • X := rfl - -theorem exp_prod_const_fun : 𝔼[(λ _ ↦ c) * X // P] = c * 𝔼[X // P] := - by simp only [expect, Pi.mul_def, constant_mul_eq_smul, dotProduct_smul, smul_eq_mul] +theorem exp_prod_const_fun : 𝔼[(λ _ ↦ c) * X // P] = c * 𝔼[X // P] := by + have constant_mul_eq_smul : (fun ω ↦ c * X ω) = c • X := rfl + simp only [expect, Pi.mul_def, constant_mul_eq_smul, dotProduct_smul, smul_eq_mul] theorem exp_indi_eq_exp_indr : ∀i : Fin k, 𝔼[L =ᵢ i // P] = 𝔼[𝕀 ∘ (L =ᵣ i) // P] := by intro i; rw [indi_eq_indr] diff --git a/MDPLib/Probability/Matrix.lean b/MDPLib/Probability/Matrix.lean index d0002a7..d7580be 100644 --- a/MDPLib/Probability/Matrix.lean +++ b/MDPLib/Probability/Matrix.lean @@ -51,7 +51,4 @@ def bellman_backup (v : Ω → ℚ) : Ω → ℚ := Proc.r + Proc.γ • Proc.Pr notation "𝔹["v "//" Proc "]" => bellman_backup Proc v --- Looking for norm in mathlib -theorem bellman_backup_contraction : 1 = 1 := by sorry - end RewardProcess diff --git a/MDPLib/Risk/VaR.lean b/MDPLib/Risk/VaR.lean index 974d756..691786e 100644 --- a/MDPLib/Risk/VaR.lean +++ b/MDPLib/Risk/VaR.lean @@ -99,13 +99,13 @@ theorem var_prob_cond : IsVaR P X α v ↔ (ℙ[X <ᵣ v // P] ≤ α.val ∧ α theorem finvar_correct : IsVaR P X α (FinVaR P X α) := var_prob_cond.mpr finvar_prob_cond theorem varq_is_quantile : IsVaR_Q P X α v → IsQuantile P X α.val v := - fun h => by simp_all only [Set.mem_setOf_eq,IsVaR_Q,Quantile,IsGreatest] + fun h => by simp_all only [Set.mem_ofPred_eq,IsVaR_Q,Quantile,IsGreatest] theorem varq_is_quantilelower : IsVaR_Q P X α v → IsQuantileLower P X α.val v := - fun h => by simp_all only [Set.mem_setOf_eq,IsVaR_Q,Quantile,IsGreatest,IsQuantileLower,IsQuantile] + fun h => by simp_all only [Set.mem_ofPred_eq,IsVaR_Q,Quantile,IsGreatest,IsQuantileLower,IsQuantile] theorem var_is_quantilelower : IsVaR P X α v → IsQuantileLower P X α.val v := - fun h => by simp_all only [Set.mem_setOf_eq,IsVaR,QuantileLower,IsGreatest,Set.mem_setOf_eq] + fun h => by simp_all only [Set.mem_ofPred_eq,IsVaR,QuantileLower,IsGreatest] theorem var_is_quantile : IsVaR P X α v → IsQuantile P X α.val v := by intro h @@ -134,7 +134,7 @@ theorem isquantilelower_le_isquantile : IsCofinalFor (QuantileLower P X α.val) · exfalso; exact not_le_of_gt h2r h theorem isquantile_le_isquantilelower : IsCofinalFor (Quantile P X α.val) (QuantileLower P X α.val) := - HasSubset.Subset.isCofinalFor quantile_subset_quantilelower + LE.le.isCofinalFor quantile_subset_quantilelower theorem varq_eq_var : IsVaR_Q P X α v ↔ IsVaR P X α v := ⟨fun h => ⟨varq_is_quantilelower h, (upperBounds_mono_of_isCofinalFor isquantilelower_le_isquantile) h.2⟩, From e8978e553d200c33fd973b51a3c540c09dca0127 Mon Sep 17 00:00:00 2001 From: Marek Petrik Date: Thu, 20 Aug 2026 15:51:38 +0200 Subject: [PATCH 4/9] simplified claude's proof --- MDPLib/Probability/Basic.lean | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/MDPLib/Probability/Basic.lean b/MDPLib/Probability/Basic.lean index d98f4fb..eb3e990 100644 --- a/MDPLib/Probability/Basic.lean +++ b/MDPLib/Probability/Basic.lean @@ -23,17 +23,15 @@ open Matrix variable {Ω : Type} [Fintype Ω] {p x : Ω → ℚ} -/-- If a dot product of nonnegative vectors is positive, some coordinate of the - second vector is positive. Proved over an arbitrary finite index. -/ -theorem nneg_dotProd_pos_ex_pos (h1 : ∀ ω, p ω ≥ 0) (h2 : ∀ ω, x ω ≥ 0) (h : p ⬝ᵥ x > 0) : ∃ ω, x ω > 0 := by - by_contra hcon - push Not at hcon - have hle : p ⬝ᵥ x ≤ 0 := by - unfold dotProduct - apply Finset.sum_nonpos - intro ω _ - nlinarith [h1 ω, hcon ω] - linarith + +/-- If a dot product with a nonnegative vector is positive, some coordinate of the + vector is positive. -/ +theorem nneg_dotProd_pos_ex_pos (h1 : p ≥ 0) (h : p ⬝ᵥ x > 0) : ∃ ω, x ω > 0 := by + by_contra! hcon + have h2 := dotProduct_le_dotProduct_of_nonneg_left hcon h1 + rw [dotProduct_zero'] at h2 + order + end General @@ -61,7 +59,7 @@ end Findist section RandomVariables -variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {A B : FinRV Ω Bool} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} +variable {Ω : Type} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} theorem rvle_monotone (h1 : X ≤ Y) (h2: t₁ ≤ t₂) : 𝕀 ∘ (Y ≤ᵣ t₁) ≤ 𝕀 ∘ (X ≤ᵣ t₂) := by intro ω @@ -82,6 +80,8 @@ theorem rvlt_monotone (h1 : X ≤ Y) (h2: t₁ ≤ t₂) : 𝕀 ∘ (Y <ᵣ t₁ · by_cases h5 : X ω < t₂ repeat simp [h3, h5, 𝕀, indicator] +variable [FinEnum Ω] {P : Findist Ω} {A B : FinRV Ω Bool} + theorem rv_le_max_one : (X ≤ᵣ (FinRV.max P X)) = 1 := by ext ω unfold FinRV.leq @@ -112,7 +112,7 @@ variable (P : Findist Ω) (X : FinRV Ω ℚ) (t : ℚ) theorem prob_atomic_omega {b : ℚ} (h : ℙ[X =ᵣ b // P] > 0) : ∃ω, X ω = b := by - obtain ⟨ω, hω⟩ : ∃ω, (𝕀 ∘ (X=ᵣb)) ω > 0 := nneg_dotProd_pos_ex_pos (P.nneg) (ind_nneg) h + obtain ⟨ω, hω⟩ : ∃ω, (𝕀 ∘ (X=ᵣb)) ω > 0 := nneg_dotProd_pos_ex_pos (P.nneg) h use ω by_contra! simp_all [𝕀, indicator] From a54fa581c601bc8eccfa4f1bfcdfd338a84e2b71 Mon Sep 17 00:00:00 2001 From: Marek Petrik Date: Mon, 24 Aug 2026 11:41:13 +0200 Subject: [PATCH 5/9] added a nonempty attribute to Omega --- MDPLib/Probability/Basic.lean | 46 +++++++++++++++++------------------ MDPLib/Probability/Defs.lean | 29 ++++++++++------------ 2 files changed, 35 insertions(+), 40 deletions(-) diff --git a/MDPLib/Probability/Basic.lean b/MDPLib/Probability/Basic.lean index eb3e990..17e8379 100644 --- a/MDPLib/Probability/Basic.lean +++ b/MDPLib/Probability/Basic.lean @@ -64,8 +64,7 @@ variable {Ω : Type} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} theorem rvle_monotone (h1 : X ≤ Y) (h2: t₁ ≤ t₂) : 𝕀 ∘ (Y ≤ᵣ t₁) ≤ 𝕀 ∘ (X ≤ᵣ t₂) := by intro ω by_cases h3 : Y ω ≤ t₁ - · have h4 : X ω ≤ t₂ := le_trans (le_trans (h1 ω) h3) h2 - simp [FinRV.leq, 𝕀, indicator, h3, h4] + · simp [FinRV.leq, 𝕀, indicator, h3, (le_trans (le_trans (h1 ω) h3) h2)] · by_cases h5 : X ω ≤ t₂ repeat simp [h3, h5, 𝕀, indicator] @@ -80,64 +79,58 @@ theorem rvlt_monotone (h1 : X ≤ Y) (h2: t₁ ≤ t₂) : 𝕀 ∘ (Y <ᵣ t₁ · by_cases h5 : X ω < t₂ repeat simp [h3, h5, 𝕀, indicator] +theorem rv_monotone_sharp {t₁ t₂ : ℚ} (h : t₁ < t₂) (ω) (hω : (X ≥ᵣ t₂) ω ) : (X >ᵣ t₁) ω := + by simp [FinRV.gt, FinRV.geq] at hω ⊢ + order + variable [FinEnum Ω] {P : Findist Ω} {A B : FinRV Ω Bool} theorem rv_le_max_one : (X ≤ᵣ (FinRV.max P X)) = 1 := - by ext ω - unfold FinRV.leq - simpa using rv_omega_le_max P ω + by ext ω; simpa using rv_omega_le_max P ω theorem rv_max_in_image : (FinRV.max P X) ∈ Finset.univ.image X := Finset.max'_mem (Finset.image X Finset.univ) (rv_image_nonempty P X) -theorem rv_omega_ge_min (P : Findist Ω) : ∀ω, X ω ≥ (FinRV.min P X) := - by intro ω - have h : X ω ∈ (Finset.image X Finset.univ) := Finset.mem_image_of_mem X (Finset.mem_univ ω) - exact Finset.min'_le (Finset.image X Finset.univ) (X ω) h +theorem rv_omega_ge_min (P : Findist Ω) (ω) : X ω ≥ (FinRV.min P X) := + Finset.min'_le (Finset.image X Finset.univ) (X ω) (Finset.mem_image_of_mem X (Finset.mem_univ ω)) theorem rv_ge_min_one : (X ≥ᵣ (FinRV.min P X)) = 1 := - by ext ω - unfold FinRV.geq - simpa using rv_omega_ge_min P ω - -theorem rv_monotone_sharp {t₁ t₂ : ℚ} : t₁ < t₂ → ∀ ω, (X ≥ᵣ t₂) ω → (X >ᵣ t₁) ω := - by intro h ω pre - simp [FinRV.gt, FinRV.geq] at pre ⊢ - linarith + by ext ω; simpa using rv_omega_ge_min P ω -- results for discrete probability distributions section Atomic variable (P : Findist Ω) (X : FinRV Ω ℚ) (t : ℚ) - theorem prob_atomic_omega {b : ℚ} (h : ℙ[X =ᵣ b // P] > 0) : ∃ω, X ω = b := by obtain ⟨ω, hω⟩ : ∃ω, (𝕀 ∘ (X=ᵣb)) ω > 0 := nneg_dotProd_pos_ex_pos (P.nneg) h use ω by_contra! simp_all [𝕀, indicator] +#check Finset.max' + theorem rv_le_step_lt_max (h0 : t < (FinRV.max P X)) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) ∧ q ∈ (Finset.univ.image X) := by let 𝓧 := Finset.univ.image X let 𝓨 := 𝓧.filter (fun x ↦ x > t) have hnonempty : 𝓨.Nonempty := Finset.filter_nonempty_iff.mpr ⟨FinRV.max P X, ⟨rv_max_in_image, h0⟩⟩ let q := 𝓨.min' hnonempty - have hq_Y : q > t := (Finset.mem_filter.mp (Finset.min'_mem 𝓨 hnonempty)).right + have q_ge_t : q > t := (Finset.mem_filter.mp (Finset.min'_mem 𝓨 hnonempty)).right use q constructor - · exact hq_Y - · constructor; swap - · exact Finset.mem_of_mem_filter q (Finset.min'_mem 𝓨 hnonempty) + · exact q_ge_t + · constructor · ext ω rw [FinRV.leq,FinRV.lt,decide_eq_decide] constructor - · exact fun h2 => lt_of_le_of_lt h2 hq_Y + · exact fun h2 => lt_of_le_of_lt h2 q_ge_t · intro h2 have hxω : X ω ∉ 𝓨 := by by_contra! inY; exact not_lt_of_ge (Finset.min'_le 𝓨 (X ω) inY) h2 rw [Finset.mem_filter] at hxω push Not at hxω exact hxω (Finset.mem_image_of_mem X (Finset.mem_univ ω)) + · exact Finset.mem_of_mem_filter q (Finset.min'_mem 𝓨 hnonempty) theorem rv_le_step_lt (P : Findist Ω) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) := by cases' lt_or_ge t (FinRV.max P X) with hlt hge @@ -150,9 +143,14 @@ theorem rv_le_step_lt (P : Findist Ω) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) := have ab : (X ≤ᵣ t) = (X <ᵣ q) := by ext ω; simp_all [FinRV.leq, FinRV.lt] exact ⟨q, ⟨lt_add_one t, ab⟩⟩ + + theorem rv_ge_step_lt_min (h0 : t > (FinRV.min P X)) : ∃q < t, (X ≥ᵣ t) = (X >ᵣ q) ∧ q ∈ (Finset.univ.image X) := by - sorry + sorry + +#help tactic + end Atomic diff --git a/MDPLib/Probability/Defs.lean b/MDPLib/Probability/Defs.lean index 064545d..25749c4 100644 --- a/MDPLib/Probability/Defs.lean +++ b/MDPLib/Probability/Defs.lean @@ -81,12 +81,10 @@ Main results section RandomVariable /-- A finite random variable: a bare function from the sample space `Ω` to `ρ`. - The `FinEnum` instance lives on `Ω` and is shared with the distribution function, so - a random variable never carries its own enumeration. -/ -abbrev FinRV (Ω : Type) (ρ : Type) := Ω → ρ + The `FinEnum` instance lives on `Ω` and is shared with the distribution function. -/ +abbrev FinRV (Ω : Type) [Nonempty Ω] (ρ : Type) := Ω → ρ - -variable {Ω : Type} {ρ : Type} +variable {Ω : Type} [Nonempty Ω] {ρ : Type} namespace FinRV @@ -231,21 +229,20 @@ variable {β : Type} -- assume enumerability of Ω from here because we need a probability space variable [FinEnum Ω] -theorem rv_image_nonempty [DecidableEq β] [LinearOrder β] (P : Findist Ω) (X : FinRV Ω β) : +theorem rv_image_nonempty [DecidableEq β] [LinearOrder β] (X : FinRV Ω β) : (Finset.univ.image X).Nonempty := - have : Nonempty Ω := P.nonempty Finset.image_nonempty.mpr Finset.univ_nonempty -def FinRV.min [DecidableEq β] [LinearOrder β] (P : Findist Ω) (X : FinRV Ω β) : β := - (Finset.univ.image X).min' (rv_image_nonempty P X) +def FinRV.min [DecidableEq β] [LinearOrder β] (X : FinRV Ω β) : β := + (Finset.univ.image X).min' (rv_image_nonempty X) -def FinRV.max [DecidableEq β] [LinearOrder β] (P : Findist Ω) (X : FinRV Ω β) : β := - (Finset.univ.image X).max' (rv_image_nonempty P X) +def FinRV.max [DecidableEq β] [LinearOrder β] (X : FinRV Ω β) : β := + (Finset.univ.image X).max' (rv_image_nonempty X) variable {X : FinRV Ω ℚ} -theorem rv_omega_le_max (P : Findist Ω) : ∀ω, X ω ≤ (FinRV.max P X) := by +theorem rv_omega_le_max (P : Findist Ω) : ∀ω, X ω ≤ (FinRV.max X) := by intro ω have h : X ω ∈ (Finset.image X Finset.univ) := Finset.mem_image_of_mem X (Finset.mem_univ ω) exact Finset.le_max' (Finset.image X Finset.univ) (X ω) h @@ -256,7 +253,7 @@ end RandomVariable ------------------------------ Probability --------------------------- section Probability -variable {Ω : Type} [FinEnum Ω] (P : Findist Ω) (B C : FinRV Ω Bool) +variable {Ω : Type} [Nonempty Ω] [FinEnum Ω] (P : Findist Ω) (B C : FinRV Ω Bool) /-- Probability of B -/ def probability : ℚ := P.p ⬝ᵥ (𝕀 ∘ B) @@ -289,7 +286,7 @@ def PMF {K : ℕ} (pmf : Fin K → ℚ) (P : Findist Ω) (L : FinRV Ω (Fin K)) ∀ k : Fin K, pmf k = ℙ[ L =ᵣ k // P] -variable {Ω : Type} [FinEnum Ω] {k : ℕ} {L : FinRV Ω (Fin k)} +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {k : ℕ} {L : FinRV Ω (Fin k)} variable {pmf : Fin k → ℚ} {P : Findist Ω} theorem pmf_rv_k_ge_1 (h : PMF pmf P L) : 0 < k := @@ -303,7 +300,7 @@ end Probability section CDF -variable {Ω : Type} [FinEnum Ω] +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] def cdf (P : Findist Ω) (X : FinRV Ω ℚ) (t : ℚ) : ℚ := ℙ[X ≤ᵣ t // P] @@ -323,7 +320,7 @@ Main results - Decomposition with a discrete random variables, used in the proofs of LOTUS and TLE -/ -variable {Ω : Type} [FinEnum Ω] (P : Findist Ω) (X Y Z: FinRV Ω ℚ) (B : FinRV Ω Bool) +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] (P : Findist Ω) (X Y Z: FinRV Ω ℚ) (B : FinRV Ω Bool) /-- Standard expectation operator -/ def expect : ℚ := P.p ⬝ᵥ X From 22c0c89f2680dcb275aa21fbe0335f9eefc311a1 Mon Sep 17 00:00:00 2001 From: Marek Petrik Date: Tue, 25 Aug 2026 18:24:26 +0200 Subject: [PATCH 6/9] simplified and cleaned up --- MDPLib/Probability/Basic.lean | 21 +++++----- MDPLib/Probability/Defs.lean | 68 ++++++++++++++------------------- MDPLib/Probability/Prelude.lean | 26 +++++++++---- 3 files changed, 56 insertions(+), 59 deletions(-) diff --git a/MDPLib/Probability/Basic.lean b/MDPLib/Probability/Basic.lean index 17e8379..1ec2118 100644 --- a/MDPLib/Probability/Basic.lean +++ b/MDPLib/Probability/Basic.lean @@ -37,7 +37,7 @@ end General namespace Findist -variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {B : FinRV Ω Bool} +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {P : Findist Ω} {B : FinRV Ω Bool} theorem ge_zero : 0 ≤ ℙ[B // P] := by rw [prob_eq_exp_ind] @@ -59,7 +59,7 @@ end Findist section RandomVariables -variable {Ω : Type} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} +variable {Ω : Type} [Nonempty Ω] {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} theorem rvle_monotone (h1 : X ≤ Y) (h2: t₁ ≤ t₂) : 𝕀 ∘ (Y ≤ᵣ t₁) ≤ 𝕀 ∘ (X ≤ᵣ t₂) := by intro ω @@ -83,18 +83,18 @@ theorem rv_monotone_sharp {t₁ t₂ : ℚ} (h : t₁ < t₂) (ω) (hω : (X ≥ by simp [FinRV.gt, FinRV.geq] at hω ⊢ order -variable [FinEnum Ω] {P : Findist Ω} {A B : FinRV Ω Bool} +variable [FinEnum Ω] [Nonempty Ω] {P : Findist Ω} {A B : FinRV Ω Bool} -theorem rv_le_max_one : (X ≤ᵣ (FinRV.max P X)) = 1 := - by ext ω; simpa using rv_omega_le_max P ω +theorem rv_le_max_one : (X ≤ᵣ (FinRV.max X)) = 1 := + by ext ω; simpa using rv_omega_le_max ω -theorem rv_max_in_image : (FinRV.max P X) ∈ Finset.univ.image X := +theorem rv_max_in_image : (FinRV.max X) ∈ Finset.univ.image X := Finset.max'_mem (Finset.image X Finset.univ) (rv_image_nonempty P X) -theorem rv_omega_ge_min (P : Findist Ω) (ω) : X ω ≥ (FinRV.min P X) := +theorem rv_omega_ge_min (P : Findist Ω) (ω) : X ω ≥ (FinRV.min X) := Finset.min'_le (Finset.image X Finset.univ) (X ω) (Finset.mem_image_of_mem X (Finset.mem_univ ω)) -theorem rv_ge_min_one : (X ≥ᵣ (FinRV.min P X)) = 1 := +theorem rv_ge_min_one : (X ≥ᵣ (FinRV.min X)) = 1 := by ext ω; simpa using rv_omega_ge_min P ω -- results for discrete probability distributions @@ -108,7 +108,6 @@ theorem prob_atomic_omega {b : ℚ} (h : ℙ[X =ᵣ b // P] > 0) : ∃ω, X ω = by_contra! simp_all [𝕀, indicator] -#check Finset.max' theorem rv_le_step_lt_max (h0 : t < (FinRV.max P X)) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) ∧ q ∈ (Finset.univ.image X) := by let 𝓧 := Finset.univ.image X @@ -133,10 +132,10 @@ theorem rv_le_step_lt_max (h0 : t < (FinRV.max P X)) : ∃q > t, (X ≤ᵣ t) = · exact Finset.mem_of_mem_filter q (Finset.min'_mem 𝓨 hnonempty) theorem rv_le_step_lt (P : Findist Ω) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) := - by cases' lt_or_ge t (FinRV.max P X) with hlt hge + by cases' lt_or_ge t (FinRV.max X) with hlt hge · obtain ⟨q, h⟩ := rv_le_step_lt_max P X t hlt exact ⟨q, ⟨h.1, h.2.1⟩⟩ - · have h := rv_omega_le_max P (X:=X) + · have h := rv_omega_le_max (X:=X) grw [hge] at h let q := t + 1 have b : ∀ω, X ω < q := fun ω => lt_add_of_le_of_pos (h ω) rfl diff --git a/MDPLib/Probability/Defs.lean b/MDPLib/Probability/Defs.lean index 25749c4..2420060 100644 --- a/MDPLib/Probability/Defs.lean +++ b/MDPLib/Probability/Defs.lean @@ -220,34 +220,31 @@ theorem rv_prod_sum_additive : ∑ i, Y * (Xs i) = Y * (∑ i, Xs i) := variable {g : Fin k → ℚ} -theorem rv_prod_const : ∀i, (g ∘ L) * (L =ᵢ i) = (g i) • (L =ᵢ i) := - by intro i; ext ω; by_cases h : L ω = i <;> simp [h] +theorem rv_prod_const (i) : (g ∘ L) * (L =ᵢ i) = (g i) • (L =ᵢ i) := + by ext ω; by_cases h : L ω = i <;> simp [h] variable {β : Type} -- assume enumerability of Ω from here because we need a probability space -variable [FinEnum Ω] +variable [FinEnum Ω] [DecidableEq β] -theorem rv_image_nonempty [DecidableEq β] [LinearOrder β] (X : FinRV Ω β) : +theorem rv_image_nonempty (X : FinRV Ω β) : (Finset.univ.image X).Nonempty := Finset.image_nonempty.mpr Finset.univ_nonempty -def FinRV.min [DecidableEq β] [LinearOrder β] (X : FinRV Ω β) : β := +def FinRV.min [LinearOrder β] (X : FinRV Ω β) : β := (Finset.univ.image X).min' (rv_image_nonempty X) -def FinRV.max [DecidableEq β] [LinearOrder β] (X : FinRV Ω β) : β := +def FinRV.max [LinearOrder β] (X : FinRV Ω β) : β := (Finset.univ.image X).max' (rv_image_nonempty X) variable {X : FinRV Ω ℚ} - -theorem rv_omega_le_max (P : Findist Ω) : ∀ω, X ω ≤ (FinRV.max X) := by - intro ω +theorem rv_omega_le_max (ω) : X ω ≤ (FinRV.max X) := by have h : X ω ∈ (Finset.image X Finset.univ) := Finset.mem_image_of_mem X (Finset.mem_univ ω) exact Finset.le_max' (Finset.image X Finset.univ) (X ω) h - end RandomVariable ------------------------------ Probability --------------------------- @@ -276,8 +273,8 @@ example {a b : ℚ} (h : 0 ≤ a) (h2 : 0 ≤ b) : 0 ≤ a * b := Rat.mul_nonne variable {P : Findist Ω} {B : FinRV Ω Bool} -theorem prod_zero_of_prob_zero : ℙ[B // P] = 0 → (P.p * (𝕀∘B) = 0) := by - intro h; exact prod_eq_zero_of_nneg_dp_zero P.nneg ind_nneg h +theorem prod_zero_of_prob_zero (h : ℙ[B // P] = 0) : (P.p * (𝕀∘B) = 0) := by + exact prod_eq_zero_of_nneg_dp_zero P.nneg ind_nneg h ------------------------------ PMF --------------------------- @@ -351,8 +348,7 @@ notation "𝔼[" X "|ᵣ" L "//" P "]" => expect_cnd_rv P X L section Expectation_properties variable {P : Findist Ω} {X Y Z: FinRV Ω ℚ} {B : FinRV Ω Bool} -theorem exp_congr : (X = Y) → 𝔼[X // P] = 𝔼[Y // P] := - by intro h +theorem exp_congr (h : X = Y) : 𝔼[X // P] = 𝔼[Y // P] := by unfold expect dotProduct apply Fintype.sum_congr simp_all @@ -362,14 +358,10 @@ theorem exp_mul_comm : 𝔼[X * Y // P] = 𝔼[Y * X // P] := exp_congr (CommMon variable {c : ℚ} {p : Ω → ℚ} -theorem exp_const : 𝔼[(fun _ ↦ c) // P] = c := - by unfold expect - rw [rv_const_fun_to_one] - simp only [dotProduct_smul, smul_eq_mul] - rw [dotProduct_comm, P.prob] - simp +theorem exp_const : 𝔼[(fun _ ↦ c) // P] = c := by + rw [rv_const_fun_to_one,expect, dotProduct_smul,smul_eq_mul,dotProduct_comm,P.prob,Rat.mul_one] -theorem exp_one : 𝔼[ 1 // P] = 1 := exp_const +theorem exp_one : 𝔼[ 1 // P] = 1 := exp_const theorem exp_cond_eq_def : 𝔼[X | B // P] * ℙ[B // P] = 𝔼[X * (𝕀 ∘ B) // P] := by unfold expect_cnd @@ -377,20 +369,18 @@ theorem exp_cond_eq_def : 𝔼[X | B // P] * ℙ[B // P] = 𝔼[X * (𝕀 ∘ B · rw [h, Rat.mul_zero, expect,dotProd_hadProd_comm, dotProd_hadProd_rotate, prod_zero_of_prob_zero h] exact (dotProduct_zero X).symm · simp_all + +theorem exp_prod_const_fun : 𝔼[(fun _ ↦ c) * X // P] = c * 𝔼[X // P] := by + rw [funmul_eq_smul, expect, expect, dotProd_smul_homogeneous] -theorem exp_prod_const_fun : 𝔼[(λ _ ↦ c) * X // P] = c * 𝔼[X // P] := by - have constant_mul_eq_smul : (fun ω ↦ c * X ω) = c • X := rfl - simp only [expect, Pi.mul_def, constant_mul_eq_smul, dotProduct_smul, smul_eq_mul] - -theorem exp_indi_eq_exp_indr : ∀i : Fin k, 𝔼[L =ᵢ i // P] = 𝔼[𝕀 ∘ (L =ᵣ i) // P] := by - intro i; rw [indi_eq_indr] +theorem exp_indi_eq_exp_indr (i) : 𝔼[L =ᵢ i // P] = 𝔼[𝕀 ∘ (L =ᵣ i) // P] := by rw [indi_eq_indr] /-- Expectation is homogeneous under product -/ theorem exp_homogenous : 𝔼[c • X // P] = c * 𝔼[X // P] := by simp only [expect, dotProduct_smul, smul_eq_mul] /-- Additivity of expectation --/ -theorem exp_additive {m : ℕ} (Xs : Fin m → FinRV Ω ℚ) : 𝔼[∑ i : Fin m, Xs i // P] = ∑ i : Fin m, 𝔼[Xs i // P] := - by unfold expect; exact dotProduct_sum P.p Finset.univ Xs +theorem exp_additive {m : ℕ} (Xs : Fin m → FinRV Ω ℚ) : + 𝔼[∑ i : Fin m, Xs i // P] = ∑ i : Fin m, 𝔼[Xs i // P] := dotProduct_sum P.p Finset.univ Xs theorem exp_additive_two : 𝔼[X + Y // P] = 𝔼[X // P] + 𝔼[Y // P] := by simp [expect] @@ -401,17 +391,15 @@ theorem exp_monotone (h: X ≤ Y) : 𝔼[X // P] ≤ 𝔼[Y // P] := dotProduct variable {k : ℕ} {g : Fin k → ℚ} {L : FinRV Ω (Fin k)} -theorem exp_decompose : 𝔼[X // P] = ∑ i, 𝔼[X * (L =ᵢ i) // P] := - by nth_rewrite 1 [rv_decompose X L] - rw [exp_additive] +theorem exp_decompose : 𝔼[X // P] = ∑ i, 𝔼[X * (L =ᵢ i) // P] := by + nth_rewrite 1 [rv_decompose X L] + rw [exp_additive] /-- Expectation of a conditional constant. Only when probability is positive. -/ -theorem exp_cond_const : ∀ i, ℙ[L =ᵣ i // P] ≠ 0 → 𝔼[g ∘ L | L =ᵣ i // P] = g i := - by intro i h - unfold expect_cnd - rw [indi_eq_indr, rv_prod_const i, exp_homogenous] - rw [←indi_eq_indr, ←prob_eq_exp_ind] - simp only [h, ne_eq, isUnit_iff_ne_zero, not_false_eq_true, IsUnit.mul_div_cancel_right] +theorem exp_cond_const (i) (h : ℙ[L =ᵣ i // P] ≠ 0) : 𝔼[g ∘ L | L =ᵣ i // P] = g i := by + unfold expect_cnd + rw [indi_eq_indr, rv_prod_const i, exp_homogenous, ←indi_eq_indr, ←prob_eq_exp_ind] + simp [h, ne_eq, not_false_eq_true] end Expectation_properties @@ -420,8 +408,8 @@ section Probability_properties variable {n : ℕ} {P : Findist Ω} {A B : FinRV Ω Bool} -theorem ind_monotone : (∀ ω, A ω → B ω) → (𝕀∘A) ≤ (𝕀∘B) := by - intro h ω +theorem ind_monotone (h : ∀ ω, A ω → B ω) : (𝕀∘A) ≤ (𝕀∘B) := by + intro ω specialize h ω by_cases h1 : A ω · simp_all [indicator] diff --git a/MDPLib/Probability/Prelude.lean b/MDPLib/Probability/Prelude.lean index 728e349..5f60840 100644 --- a/MDPLib/Probability/Prelude.lean +++ b/MDPLib/Probability/Prelude.lean @@ -52,26 +52,36 @@ end FunctionalAnalysis section dotProduct variable {Ω : Type*} [Fintype Ω] -variable {x y z : Ω → ℚ} +variable {x y z : Ω → ℚ} {c : ℚ} + + theorem dotProd_hadProd_rotate : x ⬝ᵥ (y * z) = z ⬝ᵥ (x * y) := by - unfold dotProduct apply Fintype.sum_congr intro i - simp + rewrite [Pi.mul_apply y z i, Pi.mul_apply] ring +example : z * x = x * z := by apply? + theorem dotProd_hadProd_comm : x ⬝ᵥ (y * z) = x ⬝ᵥ (z * y) := by - unfold dotProduct apply Fintype.sum_congr intro i - simp - left + rewrite [mul_comm' z y] ring -theorem dotProduct_eq_one_had : x ⬝ᵥ y = 1 ⬝ᵥ (x * y) := by simp [dotProduct] +example : (c • x) i = c * x i := by rw [Pi.smul_apply, smul_eq_mul] + +theorem funmul_eq_smul : (fun _ ↦ c) * X = c • X := rfl -example (hx : 0 ≤ x) (hy : 0 ≤ y) : 0 ≤ x * y := Left.mul_nonneg hx hy +theorem dotProd_smul_homogeneous : x ⬝ᵥ (c • y) = c * x ⬝ᵥ y := by + rewrite [dotProduct, dotProduct, Finset.mul_sum] + apply Fintype.sum_congr + intro i + rw [Pi.smul_apply, smul_eq_mul] + ring + +theorem dotProduct_eq_one_had : x ⬝ᵥ y = 1 ⬝ᵥ (x * y) := by simp [dotProduct] theorem prod_eq_zero_of_nneg_dp_zero (hx : 0 ≤ x) (hy : 0 ≤ y) : x ⬝ᵥ y = 0 → x * y = 0 := by intro h From e6a2d8d762ac787792634730c3d4a05d99492cd2 Mon Sep 17 00:00:00 2001 From: Marek Petrik Date: Fri, 4 Sep 2026 16:30:57 -0400 Subject: [PATCH 7/9] simplified maxima and minima --- MDPLib/Probability/Basic.lean | 62 ++++++++++++-------------------- MDPLib/Probability/Defs.lean | 55 +++++++++++++++------------- MDPLib/Probability/Quantile.lean | 4 +-- MDPLib/Risk/CVaR.lean | 34 ------------------ MDPLib/Risk/VaR.lean | 12 +++---- lake-manifest.json | 28 +++++++-------- lean-toolchain | 2 +- 7 files changed, 76 insertions(+), 121 deletions(-) diff --git a/MDPLib/Probability/Basic.lean b/MDPLib/Probability/Basic.lean index 1ec2118..05fab87 100644 --- a/MDPLib/Probability/Basic.lean +++ b/MDPLib/Probability/Basic.lean @@ -23,7 +23,6 @@ open Matrix variable {Ω : Type} [Fintype Ω] {p x : Ω → ℚ} - /-- If a dot product with a nonnegative vector is positive, some coordinate of the vector is positive. -/ theorem nneg_dotProd_pos_ex_pos (h1 : p ≥ 0) (h : p ⬝ᵥ x > 0) : ∃ ω, x ω > 0 := by @@ -83,19 +82,19 @@ theorem rv_monotone_sharp {t₁ t₂ : ℚ} (h : t₁ < t₂) (ω) (hω : (X ≥ by simp [FinRV.gt, FinRV.geq] at hω ⊢ order -variable [FinEnum Ω] [Nonempty Ω] {P : Findist Ω} {A B : FinRV Ω Bool} +variable [FinEnum Ω] {P : Findist Ω} {A B : FinRV Ω Bool} theorem rv_le_max_one : (X ≤ᵣ (FinRV.max X)) = 1 := by ext ω; simpa using rv_omega_le_max ω theorem rv_max_in_image : (FinRV.max X) ∈ Finset.univ.image X := - Finset.max'_mem (Finset.image X Finset.univ) (rv_image_nonempty P X) + Finset.max'_mem (Finset.image X Finset.univ) (rv_image_nonempty X) -theorem rv_omega_ge_min (P : Findist Ω) (ω) : X ω ≥ (FinRV.min X) := +theorem rv_omega_ge_min (ω) : X ω ≥ (FinRV.min X) := Finset.min'_le (Finset.image X Finset.univ) (X ω) (Finset.mem_image_of_mem X (Finset.mem_univ ω)) theorem rv_ge_min_one : (X ≥ᵣ (FinRV.min X)) = 1 := - by ext ω; simpa using rv_omega_ge_min P ω + by ext ω; simpa using rv_omega_ge_min ω -- results for discrete probability distributions section Atomic @@ -109,10 +108,10 @@ theorem prob_atomic_omega {b : ℚ} (h : ℙ[X =ᵣ b // P] > 0) : ∃ω, X ω = simp_all [𝕀, indicator] -theorem rv_le_step_lt_max (h0 : t < (FinRV.max P X)) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) ∧ q ∈ (Finset.univ.image X) := by +theorem rv_le_step_lt_max (h0 : t < (FinRV.max X)) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) ∧ q ∈ (Finset.univ.image X) := by let 𝓧 := Finset.univ.image X let 𝓨 := 𝓧.filter (fun x ↦ x > t) - have hnonempty : 𝓨.Nonempty := Finset.filter_nonempty_iff.mpr ⟨FinRV.max P X, ⟨rv_max_in_image, h0⟩⟩ + have hnonempty : 𝓨.Nonempty := Finset.filter_nonempty_iff.mpr ⟨FinRV.max X, ⟨rv_max_in_image, h0⟩⟩ let q := 𝓨.min' hnonempty have q_ge_t : q > t := (Finset.mem_filter.mp (Finset.min'_mem 𝓨 hnonempty)).right use q @@ -133,7 +132,7 @@ theorem rv_le_step_lt_max (h0 : t < (FinRV.max P X)) : ∃q > t, (X ≤ᵣ t) = theorem rv_le_step_lt (P : Findist Ω) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) := by cases' lt_or_ge t (FinRV.max X) with hlt hge - · obtain ⟨q, h⟩ := rv_le_step_lt_max P X t hlt + · obtain ⟨q, h⟩ := rv_le_step_lt_max X t hlt exact ⟨q, ⟨h.1, h.2.1⟩⟩ · have h := rv_omega_le_max (X:=X) grw [hge] at h @@ -142,18 +141,11 @@ theorem rv_le_step_lt (P : Findist Ω) : ∃q > t, (X ≤ᵣ t) = (X <ᵣ q) := have ab : (X ≤ᵣ t) = (X <ᵣ q) := by ext ω; simp_all [FinRV.leq, FinRV.lt] exact ⟨q, ⟨lt_add_one t, ab⟩⟩ - - - -theorem rv_ge_step_lt_min (h0 : t > (FinRV.min P X)) : ∃q < t, (X ≥ᵣ t) = (X >ᵣ q) ∧ q ∈ (Finset.univ.image X) := by +theorem rv_ge_step_lt_min (h0 : t > (FinRV.min X)) : ∃q < t, (X ≥ᵣ t) = (X >ᵣ q) ∧ q ∈ (Finset.univ.image X) := by sorry -#help tactic - - end Atomic - section Transformations -- Monotone transformation of the random variable @@ -256,7 +248,7 @@ end RandomVariables section Probability -variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {A B C : FinRV Ω Bool} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {P : Findist Ω} {A B C : FinRV Ω Bool} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} theorem prob_compl_sums_to_one : ℙ[B // P] + ℙ[¬ᵣB // P] = 1 := @@ -348,14 +340,14 @@ theorem prob_lt_le_monotone {q : ℚ} (h : q > t) : ℙ[X <ᵣ q // P] ≥ ℙ[X by_cases h5 : X ω < q <;> simp [h5] exact mul_le_mul_of_nonneg_left h2 (P.nneg ω) -theorem prob_le_eq_one : ℙ[X ≤ᵣ (FinRV.max P X) // P] = 1 := by rw [rv_le_max_one]; exact prob_one_of_true P +theorem prob_le_eq_one : ℙ[X ≤ᵣ (FinRV.max X) // P] = 1 := by rw [rv_le_max_one]; exact prob_one_of_true P -theorem prob_ge_eq_one : ℙ[X ≥ᵣ (FinRV.min P X) // P] = 1 := by rw [rv_ge_min_one]; exact prob_one_of_true P +theorem prob_ge_eq_one : ℙ[X ≥ᵣ (FinRV.min X) // P] = 1 := by rw [rv_ge_min_one]; exact prob_one_of_true P -theorem prob_lt_min_eq_zero : ℙ[X <ᵣ (FinRV.min P X) // P] = 0 := by +theorem prob_lt_min_eq_zero : ℙ[X <ᵣ (FinRV.min X) // P] = 0 := by rw [prob_lt_of_ge, prob_ge_eq_one]; exact sub_self 1 -theorem prob_le_max_of_le_1 {t : ℚ} (h : ℙ[X ≤ᵣ t // P] < 1) : t < FinRV.max P X := by +theorem prob_le_max_of_le_1 {t : ℚ} (h : ℙ[X ≤ᵣ t // P] < 1) : t < FinRV.max X := by by_contra! hcontra have h1 := prob_le_monotone (P := P) (le_refl X) hcontra rw [prob_le_eq_one] at h1 @@ -365,10 +357,10 @@ section Rounding ---results for discrete probability distributions variable (P : Findist Ω) (X : FinRV Ω ℚ) (t : ℚ) -theorem prob_le_step_lt_max (h: t < (FinRV.max P X)) : - ∃q > t, ℙ[X ≤ᵣ t // P] = ℙ[X <ᵣ q // P] ∧ q ∈ (Finset.univ.image X) := - let ⟨q, hq⟩ := rv_le_step_lt_max P X t h - Exists.intro q ⟨hq.1, ⟨congrArg (probability P) hq.2.1, hq.2.2 ⟩⟩ +theorem prob_le_step_lt_max (h: t < (FinRV.max X)) : + ∃q > t, ℙ[X ≤ᵣ t // P] = ℙ[X <ᵣ q // P] ∧ q ∈ (Finset.univ.image X) := sorry + --let ⟨q, hq⟩ := rv_le_step_lt_max P t h + --Exists.intro q ⟨hq.1, ⟨congrArg (probability P) hq.2.1, hq.2.2 ⟩⟩ /-- similar to `prob_le_step_lt_max` but no precondition -/ theorem prob_le_step_lt : ∃q > t, ℙ[X ≤ᵣ t // P] = ℙ[X <ᵣ q // P] := @@ -451,7 +443,7 @@ end Probability section CDF -variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {P : Findist Ω} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} /-- shows CDF is non-decreasing -/ theorem cdf_nondecreasing : t₁ ≤ t₂ → cdf P X t₁ ≤ cdf P X t₂ := by @@ -469,7 +461,7 @@ end CDF section Expectation -variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {P : Findist Ω} variable {k : ℕ} {X : FinRV Ω ℚ} {B : FinRV Ω Bool} {L : FinRV Ω (Fin k)} variable (g : Fin k → ℚ) @@ -503,10 +495,6 @@ lemma finset_image_eq_list_map_dedup : ∀x, x ∈ Finset.univ.image X ↔ x ∈ lemma finset_list_eq_list_dedup (l : List ℚ) : l.toFinset = l.dedup.toFinset := List.toFinset.ext (fun _ => List.mem_dedup.symm) - -example (f : ℚ → ℚ) (l : List ℚ) (h : l.Nodup) : ∑ y ∈ l.toFinset, f y = (l.map f).sum := - List.sum_toFinset (fun y => f y) h - section RV_Unique_Values variable {τ:Type} [DecidableEq τ] @@ -528,9 +516,7 @@ theorem sum_finset_eq_sum_image (f : ℚ → ℚ) : exact List.sum_toFinset f (List.nodup_dedup _) - - -section generic +section Generic variable {X : FinRV Ω τ} @@ -567,9 +553,7 @@ theorem finrv_image_exact {ω i} : X ω = X.imageList[i] ↔ X.imageIdxOf ω = i ⟨finrv_image_unique, fun h => by rw[←h]; exact Eq.symm (finrv_image_inverse ω)⟩ - - -end generic +end Generic theorem sum_eq_sum_image (f : ℚ → ℚ) : ∑ y ∈ (Finset.univ.image X), f y = ∑ i : Fin X.imageList.length, f (X.imageList[i]) := by @@ -597,7 +581,7 @@ end Expectation section Probability -variable {Ω : Type} [FinEnum Ω] {k : ℕ} {L : FinRV Ω (Fin k)} +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {k : ℕ} {L : FinRV Ω (Fin k)} /-- The law of total probabilities -/ theorem law_of_total_probs : ℙ[B // P] = ∑ i, ℙ[B * (L =ᵣ i) // P] := by @@ -616,7 +600,7 @@ end Probability section Probability_Permutation -variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {A B : FinRV Ω Bool} {X Y : FinRV Ω ℚ} {t : ℚ} +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {P : Findist Ω} {A B : FinRV Ω Bool} {X Y : FinRV Ω ℚ} {t : ℚ} example (σ : Equiv.Perm (Ω)) (f g : Ω → ℚ) : f ⬝ᵥ g = (f ∘ σ) ⬝ᵥ (g ∘ σ) := by exact Eq.symm (comp_equiv_dotProduct_comp_equiv f g σ) diff --git a/MDPLib/Probability/Defs.lean b/MDPLib/Probability/Defs.lean index 2420060..c1e2492 100644 --- a/MDPLib/Probability/Defs.lean +++ b/MDPLib/Probability/Defs.lean @@ -16,7 +16,6 @@ theorem bool_eq {a b : Bool} (h1 : a → b) (h2 : b → a) : a = b := Bool.le_an --------------------------- Findist --------------------------------------------------------------- -variable {Ω : Type} [FinEnum Ω] -- TODO: do we even need to assume that Ω is finitely enumerable when defining the probability space? @@ -31,6 +30,7 @@ structure Findist (Ω : Type) [FinEnum Ω] : Type where namespace Findist + /-- Finite probability distribution -/ abbrev Delta (Ω : Type) [FinEnum Ω] : Type := Findist Ω @@ -38,11 +38,15 @@ abbrev Delta (Ω : Type) [FinEnum Ω] : Type := Findist Ω abbrev Δ (Ω : Type) [FinEnum Ω] : Type := Delta Ω /-- Dirac (point mass) distribution concentrated at `ω₀`. -/ -def dirac (ω₀ : Ω) : Findist Ω where +def dirac {Ω : Type} [FinEnum Ω] (ω₀ : Ω) : Findist Ω where p := fun ω => if ω = ω₀ then 1 else 0 prob := by simp [dotProduct] nneg := by intro ω; by_cases h : ω = ω₀ <;> simp [h] +section General + +variable {Ω : Type} [FinEnum Ω] + /-- The sample space of a probability distribution is nonempty. -/ theorem nonempty (P : Findist Ω) : Nonempty Ω := by by_contra h @@ -54,6 +58,8 @@ theorem nonempty (P : Findist Ω) : Nonempty Ω := by theorem nonempty' [IsEmpty Ω] (P : Findist Ω) : False := (not_nonempty_iff.mpr ‹_›) P.nonempty +end General + end Findist ------------------------ Random Variable -------------------------------------------------- @@ -183,9 +189,8 @@ theorem indi_eq_indr : ∀i : Fin k, (𝕀 ∘ (L =ᵣ i)) = (L =ᵢ i) := by intro i; unfold FinRV.eq FinRV.eqi 𝕀 indicator; ext ω; by_cases h: L ω = i; repeat simp [h] variable {B : FinRV Ω Bool} -/-- Indicator is 0 or 1 -/ -theorem ind_zero_one : ∀ ω, (𝕀∘B) ω = 1 ∨ (𝕀∘B) ω = 0 := by - intro ω + +theorem ind_zero_one : (𝕀∘B) ω = 1 ∨ (𝕀∘B) ω = 0 := by by_cases h : B ω · left; simp only [Function.comp_apply, h, indicator] · right; simp only [Function.comp_apply, h, indicator] @@ -199,10 +204,12 @@ theorem ind_le_one : 𝕀∘B ≤ (1 : FinRV Ω ℚ) := variable {c : ℚ} {X : FinRV Ω ℚ} +omit [Nonempty Ω] in theorem rv_const_fun_to_one : (fun _ ↦ c : FinRV Ω ℚ) = c • 1 := by ext; simp; theorem rv_decompose (X : FinRV Ω ℚ) (L : FinRV Ω (Fin k)) : X = ∑ i, X * (L =ᵢ i) := by ext ω; simp +omit [Nonempty Ω] in theorem one_of_true : 𝕀 ∘ (1 : Ω → Bool) = (1 : Ω → ℚ) := by ext; simp [𝕀, indicator] theorem one_of_bool_or_not : B + (¬ᵣ B) = (1 : FinRV Ω Bool) := by ext ω; unfold FinRV.not; simp @@ -229,8 +236,7 @@ variable {β : Type} -- assume enumerability of Ω from here because we need a probability space variable [FinEnum Ω] [DecidableEq β] -theorem rv_image_nonempty (X : FinRV Ω β) : - (Finset.univ.image X).Nonempty := +theorem rv_image_nonempty (X : FinRV Ω β) : (Finset.univ.image X).Nonempty := Finset.image_nonempty.mpr Finset.univ_nonempty def FinRV.min [LinearOrder β] (X : FinRV Ω β) : β := @@ -282,7 +288,6 @@ theorem prod_zero_of_prob_zero (h : ℙ[B // P] = 0) : (P.p * (𝕀∘B) = 0) := def PMF {K : ℕ} (pmf : Fin K → ℚ) (P : Findist Ω) (L : FinRV Ω (Fin K)) := ∀ k : Fin K, pmf k = ℙ[ L =ᵣ k // P] - variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {k : ℕ} {L : FinRV Ω (Fin k)} variable {pmf : Fin k → ℚ} {P : Findist Ω} @@ -317,6 +322,8 @@ Main results - Decomposition with a discrete random variables, used in the proofs of LOTUS and TLE -/ +section Expectation_properties + variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] (P : Findist Ω) (X Y Z: FinRV Ω ℚ) (B : FinRV Ω Bool) /-- Standard expectation operator -/ @@ -345,8 +352,7 @@ notation "𝔼[" X "|ᵣ" L "//" P "]" => expect_cnd_rv P X L --- some basic properties -section Expectation_properties -variable {P : Findist Ω} {X Y Z: FinRV Ω ℚ} {B : FinRV Ω Bool} +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {P : Findist Ω} {X Y Z: FinRV Ω ℚ} {B : FinRV Ω Bool} theorem exp_congr (h : X = Y) : 𝔼[X // P] = 𝔼[Y // P] := by unfold expect dotProduct @@ -362,21 +368,16 @@ theorem exp_const : 𝔼[(fun _ ↦ c) // P] = c := by rw [rv_const_fun_to_one,expect, dotProduct_smul,smul_eq_mul,dotProduct_comm,P.prob,Rat.mul_one] theorem exp_one : 𝔼[ 1 // P] = 1 := exp_const - -theorem exp_cond_eq_def : 𝔼[X | B // P] * ℙ[B // P] = 𝔼[X * (𝕀 ∘ B) // P] := - by unfold expect_cnd - by_cases h: ℙ[B//P] = 0 - · rw [h, Rat.mul_zero, expect,dotProd_hadProd_comm, dotProd_hadProd_rotate, prod_zero_of_prob_zero h] - exact (dotProduct_zero X).symm - · simp_all -theorem exp_prod_const_fun : 𝔼[(fun _ ↦ c) * X // P] = c * 𝔼[X // P] := by - rw [funmul_eq_smul, expect, expect, dotProd_smul_homogeneous] +/-- Expectation is homogeneous under product -/ +theorem exp_homogenous : 𝔼[c • X // P] = c * 𝔼[X // P] := by rw [expect, expect, dotProd_smul_homogeneous] -theorem exp_indi_eq_exp_indr (i) : 𝔼[L =ᵢ i // P] = 𝔼[𝕀 ∘ (L =ᵣ i) // P] := by rw [indi_eq_indr] +-- TODO: rename to exp_homogenous' +theorem exp_prod_const_fun : 𝔼[(fun _ ↦ c) * X // P] = c * 𝔼[X // P] := by rw [funmul_eq_smul,exp_homogenous] -/-- Expectation is homogeneous under product -/ -theorem exp_homogenous : 𝔼[c • X // P] = c * 𝔼[X // P] := by simp only [expect, dotProduct_smul, smul_eq_mul] +variable {k : ℕ} {g : Fin k → ℚ} {L : FinRV Ω (Fin k)} + +theorem exp_indi_eq_exp_indr (i) : 𝔼[L =ᵢ i // P] = 𝔼[𝕀 ∘ (L =ᵣ i) // P] := by rw [indi_eq_indr] /-- Additivity of expectation --/ theorem exp_additive {m : ℕ} (Xs : Fin m → FinRV Ω ℚ) : @@ -389,7 +390,6 @@ theorem exp_monotone (h: X ≤ Y) : 𝔼[X // P] ≤ 𝔼[Y // P] := dotProduct ---- ** conditional expectation ----- -variable {k : ℕ} {g : Fin k → ℚ} {L : FinRV Ω (Fin k)} theorem exp_decompose : 𝔼[X // P] = ∑ i, 𝔼[X * (L =ᵢ i) // P] := by nth_rewrite 1 [rv_decompose X L] @@ -401,13 +401,18 @@ theorem exp_cond_const (i) (h : ℙ[L =ᵣ i // P] ≠ 0) : 𝔼[g ∘ L | L = rw [indi_eq_indr, rv_prod_const i, exp_homogenous, ←indi_eq_indr, ←prob_eq_exp_ind] simp [h, ne_eq, not_false_eq_true] +theorem exp_cond_eq_def : 𝔼[X | B // P] * ℙ[B // P] = 𝔼[X * (𝕀 ∘ B) // P] := + by unfold expect_cnd + by_cases h: ℙ[B//P] = 0 + · rw [h, Rat.mul_zero, expect,dotProd_hadProd_comm, dotProd_hadProd_rotate, prod_zero_of_prob_zero h] + exact (dotProduct_zero X).symm + · simp_all + end Expectation_properties -- Derived properties from the properties of expectation section Probability_properties -variable {n : ℕ} {P : Findist Ω} {A B : FinRV Ω Bool} - theorem ind_monotone (h : ∀ ω, A ω → B ω) : (𝕀∘A) ≤ (𝕀∘B) := by intro ω specialize h ω diff --git a/MDPLib/Probability/Quantile.lean b/MDPLib/Probability/Quantile.lean index e5aea19..e1d4de9 100644 --- a/MDPLib/Probability/Quantile.lean +++ b/MDPLib/Probability/Quantile.lean @@ -12,7 +12,7 @@ section Definition --def UnitI := {α : ℚ // 0 ≤ α ∧ α ≤ 1} -variable {Ω : Type} [FinEnum Ω] (P : Findist Ω) (X Y : FinRV Ω ℚ) (α : ℚ) (q v : ℚ) +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] (P : Findist Ω) (X Y : FinRV Ω ℚ) (α : ℚ) (q v : ℚ) /-- Proof the `q` is an `α`-quantile of `X` --/ def IsQuantile : Prop := ℙ[X ≤ᵣ q // P ] ≥ α ∧ ℙ[X ≥ᵣ q // P] ≥ 1 - α @@ -34,7 +34,7 @@ def IsQuantMin : Prop := IsLeast (Quantile P X α) q end Definition -variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {X Y : FinRV Ω ℚ} {α : ℚ} {q v : ℚ} +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {P : Findist Ω} {X Y : FinRV Ω ℚ} {α : ℚ} {q v : ℚ} theorem qset_lb : q ∈ Quantile P X α → ℙ[X ≤ᵣ q // P ] ≥ α := by simp_all [Quantile, IsQuantile] diff --git a/MDPLib/Risk/CVaR.lean b/MDPLib/Risk/CVaR.lean index 3cd7722..7daa1e8 100644 --- a/MDPLib/Risk/CVaR.lean +++ b/MDPLib/Risk/CVaR.lean @@ -3,40 +3,6 @@ import MDPLib.Risk.VaR namespace Risk -/-- Tail indicator: 1 if X(ω) > t, else 0. -/ -def tailInd (X : FinRV Ω ℚ) (t : ℚ) : FinRV Ω ℚ := - fun ω => if X ω > t then 1 else 0 - -variable {Ω : Type} [FinEnum Ω] - -/-- Conditional Value-at-Risk (CVaR) of X at level α under P. -CVaR_α(X) = E[X * I[X > VaR] ] / P[X > VaR] -If the tail probability is zero, CVaR is defined to be 0. --/ -def CVaR (P : Findist Ω) (X : FinRV Ω ℚ) (α : RiskLevel) : ℚ := - -- TODO: This CVaR computation is incorrect - -- we could use the implementation from here: https://arxiv.org/abs/2607.05078 - let v := Risk.FinVaR P X α - let B : FinRV Ω ℚ := tailInd X v - let num := 𝔼[X * B // P] - let den := ℙ[X >ᵣ v // P] - if _ : den = 0 then - 0 - else - num / den - --- NOTE (Marek): The CVaR, as defined above is not convex/concave. --- See Page 14 at https://www.cs.unh.edu/~mpetrik/pub/tutorials/risk2/dlrl2023.pdf --- NOTE (Marek): The CVaR above is defined for costs and not rewards - -notation "CVaR[" X "//" P ", " α "]" => CVaR P X α - ---TODO: prove... --- monotonicity: (∀ ω, X ω ≤ Y ω) → CVaR[α, X // P] ≤ CVaR[α, Y // P] --- translation: CVaR[α, (fun ω => X ω + c) // P] = CVaR[α, X // P] + c --- positive homogeneity: c > 0 → CVaR[α, (fun ω => c * X ω) // P] = c * CVaR[α, X // P] --- convexity --- CVaR ≥ VaR: CVaR[α, X // P] ≥ VaR[α, X // P] end Risk diff --git a/MDPLib/Risk/VaR.lean b/MDPLib/Risk/VaR.lean index 691786e..e31370d 100644 --- a/MDPLib/Risk/VaR.lean +++ b/MDPLib/Risk/VaR.lean @@ -7,7 +7,7 @@ namespace Risk open Findist FinRV Statistic -variable {Ω : Type} [FinEnum Ω] +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] variable {P : Findist Ω} {X Y : FinRV Ω ℚ} {t t₁ t₂ : ℚ} def IsRiskLevel (α : ℚ) : Prop := 0 ≤ α ∧ α < 1 @@ -23,10 +23,10 @@ def FinVaRSet (P : Findist Ω) (X : FinRV Ω ℚ) (α : RiskLevel) : Finset ℚ theorem FinVarSet_nonempty (P : Findist Ω) (X : FinRV Ω ℚ) (α : RiskLevel) : (FinVaRSet (Ω := Ω) P X α).Nonempty := by apply Finset.filter_nonempty_iff.mpr - let xmin := (Finset.univ.image X).min' (rv_image_nonempty P X) + let xmin := (Finset.univ.image X).min' (rv_image_nonempty X) use xmin constructor - · exact Finset.min'_mem (Finset.univ.image X) (rv_image_nonempty P X) + · exact Finset.min'_mem (Finset.univ.image X) (rv_image_nonempty X) · have h : ℙ[X <ᵣ xmin // P] = 0 := prob_lt_min_eq_zero rewrite [h] exact α.2.1 @@ -48,7 +48,7 @@ theorem finvar_prob_cond : ℙ[X <ᵣ (FinVaR P X α) // P] ≤ α.val ∧ α.va exact (Finset.mem_filter.mp (Finset.max'_mem 𝓢 ne𝓢)).right · generalize h : (FinVaR P X α) = t by_contra! hg - have hlt : t < (FinRV.max P X) := prob_le_max_of_le_1 (lt_of_le_of_lt hg (Set.Ico.coe_lt_one α)) + have hlt : t < (FinRV.max X) := prob_le_max_of_le_1 (lt_of_le_of_lt hg (Set.Ico.coe_lt_one α)) obtain ⟨q, ⟨hqgt, hqp, hqin⟩⟩ := prob_le_step_lt_max P X t hlt have hqt : t ≥ q := by unfold FinVaR at h; extract_lets 𝓧 𝓢 ne𝓢 at h; @@ -60,7 +60,7 @@ theorem finvar_prob_cond : ℙ[X <ᵣ (FinVaR P X α) // P] ≤ α.val ∧ α.va notation "VaR[" X "//" P ", " α "]" => FinVaR P X α -variable {Ω : Type} [FinEnum Ω] (P : Findist Ω) (X Y : FinRV Ω ℚ) (α : RiskLevel) (q v : ℚ) +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] (P : Findist Ω) (X Y : FinRV Ω ℚ) (α : RiskLevel) (q v : ℚ) /-- Value `v` is the Value at Risk at `α` of `X` and probability `P` -/ def IsVaR_Q : Prop := IsGreatest (Quantile P X α.val) v @@ -68,7 +68,7 @@ def IsVaR_Q : Prop := IsGreatest (Quantile P X α.val) v /-- A simpler, equivalent definition of Value at Risk -/ def IsVaR : Prop := IsGreatest (QuantileLower P X α.val) v -variable {Ω : Type} [FinEnum Ω] {P : Findist Ω} {X Y : FinRV Ω ℚ} {α : RiskLevel} {q v q₁ q₂ : ℚ} +variable {Ω : Type} [FinEnum Ω] [Nonempty Ω] {P : Findist Ω} {X Y : FinRV Ω ℚ} {α : RiskLevel} {q v q₁ q₂ : ℚ} theorem var_prob_cond : IsVaR P X α v ↔ (ℙ[X <ᵣ v // P] ≤ α.val ∧ α.val < ℙ[X ≤ᵣ v // P]) := by constructor diff --git a/lake-manifest.json b/lake-manifest.json index bc48131..b4e70f0 100644 --- a/lake-manifest.json +++ b/lake-manifest.json @@ -5,7 +5,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "5edf93c0f3d1f3d509c42c888931b49be5c06816", + "rev": "5437b89b97581f1d7dfbc4fdfa48e593a8a8dede", "name": "mathlib", "manifestFile": "lake-manifest.json", "inputRev": "master", @@ -15,7 +15,7 @@ "type": "git", "subDir": null, "scope": "", - "rev": "feb58b82ec2b9acfbaf71bda268acdbd0a44ccd4", + "rev": "97d4ecdfc8e09e7f511724c25e303d448de6a3db", "name": "«doc-gen4»", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -35,7 +35,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "123d15766ba49356c02ebad2a4462dfe12d79899", + "rev": "d9598f07b1bc701f1e3aae163d2681c1fd978793", "name": "plausible", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -45,7 +45,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "f5c090429dff3cf66cb65562526c9ea6e8edfbcb", + "rev": "ba67e212be1197b84c1f1f6299488a10a3002713", "name": "LeanSearchClient", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -55,7 +55,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "bb3469a87774349fe01898d8bf2fc6a1ce6411ca", + "rev": "5020974361be4f90652801a0368827bc8ed70c42", "name": "importGraph", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -65,7 +65,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "222c58dad7706a6e7cae46c0edd65ea881d3ee27", + "rev": "a8acbfd87375ff4abe14ce09db5b7664d383bc7f", "name": "proofwidgets", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -75,7 +75,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "7db8190085343afde2f5d2cdcc9bac719b6ec02c", + "rev": "18889deb9e83ea7420ef51c160d6f88552e744e3", "name": "aesop", "manifestFile": "lake-manifest.json", "inputRev": "master", @@ -85,7 +85,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "ef42f8944eaf5b6cbfbe75d1917d824c7dd6cf33", + "rev": "507746ab8f4b643ccdacb2ec4cdb5853fa9f8ab3", "name": "Qq", "manifestFile": "lake-manifest.json", "inputRev": "master", @@ -95,7 +95,7 @@ "type": "git", "subDir": null, "scope": "leanprover-community", - "rev": "76e1c118b0700b4ceafe99532e887d6431625e1a", + "rev": "4cac2177c37f5530c4da76aa8e4307f3fc9e4dcb", "name": "batteries", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -105,7 +105,7 @@ "type": "git", "subDir": null, "scope": "", - "rev": "8daa8a16be0c62a75720fb61825edaadd188ac79", + "rev": "fdb62324e6f06eeb31ff4b81e46e030f40045250", "name": "leansqlite", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -115,17 +115,17 @@ "type": "git", "subDir": null, "scope": "leanprover", - "rev": "1319485273bf87833fa472afbcefdedecb16b45f", + "rev": "ab3a82db9fea14cf0fd7f5a2de650f4b534640af", "name": "Cli", "manifestFile": "lake-manifest.json", - "inputRev": "v4.33.0-rc2", + "inputRev": "v4.34.0-rc2", "inherited": true, "configFile": "lakefile.toml"}, {"url": "https://github.com/fgdorais/lean4-unicode-basic", "type": "git", "subDir": null, "scope": "", - "rev": "7e46ff559eb084f0c4d16f847d7907c3fac78dd3", + "rev": "bbb75c5c9b7f30d5e397f27bd099655aed7db410", "name": "UnicodeBasic", "manifestFile": "lake-manifest.json", "inputRev": "main", @@ -135,7 +135,7 @@ "type": "git", "subDir": null, "scope": "", - "rev": "5706cc4847b6b8343c00bd4233bda65526b79e43", + "rev": "4cd1575ab202c7b2b59d8c3eb4b792744e9988fc", "name": "BibtexQuery", "manifestFile": "lake-manifest.json", "inputRev": "master", diff --git a/lean-toolchain b/lean-toolchain index cee6b00..d5ae4e3 100644 --- a/lean-toolchain +++ b/lean-toolchain @@ -1 +1 @@ -leanprover/lean4:v4.33.0-rc2 \ No newline at end of file +leanprover/lean4:v4.34.0-rc2 \ No newline at end of file From 0a3c2d772189d395d3e22337f5f35a25ad860bbd Mon Sep 17 00:00:00 2001 From: Marek Petrik Date: Fri, 4 Sep 2026 16:35:02 -0400 Subject: [PATCH 8/9] added the latex file back into the project --- blueprint/src/blueprint.sty | 4 - blueprint/src/content.tex | 1368 -------------------------- blueprint/src/extra_styles.css | 25 - blueprint/src/latexmkrc | 5 - blueprint/src/macros/common.tex | 41 - blueprint/src/macros/print.tex | 53 - blueprint/src/macros/web.tex | 9 - blueprint/src/plastex.cfg | 17 - blueprint/src/print.tex | 40 - blueprint/src/web.tex | 28 - home_page/404.html | 25 - home_page/Gemfile | 25 - home_page/Gemfile.lock | 287 ------ home_page/_config.yml | 34 - home_page/_include/mathjax.html | 17 - home_page/_layouts/default.html | 50 - home_page/assets/css/style.scss | 9 - home_page/index.md | 15 - latex/main.tex | 1605 +++++++++++++++++++++++++++++++ 19 files changed, 1605 insertions(+), 2052 deletions(-) delete mode 100644 blueprint/src/blueprint.sty delete mode 100644 blueprint/src/content.tex delete mode 100644 blueprint/src/extra_styles.css delete mode 100644 blueprint/src/latexmkrc delete mode 100644 blueprint/src/macros/common.tex delete mode 100644 blueprint/src/macros/print.tex delete mode 100644 blueprint/src/macros/web.tex delete mode 100644 blueprint/src/plastex.cfg delete mode 100644 blueprint/src/print.tex delete mode 100644 blueprint/src/web.tex delete mode 100644 home_page/404.html delete mode 100644 home_page/Gemfile delete mode 100644 home_page/Gemfile.lock delete mode 100644 home_page/_config.yml delete mode 100644 home_page/_include/mathjax.html delete mode 100644 home_page/_layouts/default.html delete mode 100644 home_page/assets/css/style.scss delete mode 100644 home_page/index.md create mode 100644 latex/main.tex diff --git a/blueprint/src/blueprint.sty b/blueprint/src/blueprint.sty deleted file mode 100644 index 306b18c..0000000 --- a/blueprint/src/blueprint.sty +++ /dev/null @@ -1,4 +0,0 @@ -\DeclareOption*{} -\ProcessOptions - -\newcommand{\graphcolor}[3]{} \ No newline at end of file diff --git a/blueprint/src/content.tex b/blueprint/src/content.tex deleted file mode 100644 index 57ad6ec..0000000 --- a/blueprint/src/content.tex +++ /dev/null @@ -1,1368 +0,0 @@ -% In this file you should put the actual content of the blueprint. -% It will be used both by the web and the print version. -% It should *not* include the \begin{document} -% -% If you want to split the blueprint content into several files then -% the current file can be a simple sequence of \input. Otherwise It -% can start with a \section or \chapter for instance. -\newcommand{\id}{\operatorname{id}} - - -\begin{remark} -This document omits basic definitions, such as the comparison or arithmetic operations on random variables. Arithmetic operations on random variables are performed element-wise for each element of the sample set. Please see the Lean file for complete details. -\end{remark} - - -\section{Probability} - -The treatment of the probability follows mostly the treatment in -\begin{quote} - Deisenroth, M.P., Faisal, A.A. and Ong, C.S. (2021) Mathematics for machine learning. -\end{quote} - -Although this document states that the values are in $\Real$, the lean file is all defined in terms of rational numbers $\mathbb{Q}$ for computability reasons. - -\subsection{Prelude} - -This section sets up some basic tools for working with probabilities. See \texttt{Probability/Prelude.lean}. - -\begin{definition} \label{def:probablity-value} -A $p\in \Real$ is a \emph{probability value} when $p \ge 0 \wedge p \le 1$. -\lean{Prob}\leanok -\end{definition} - -The lean file also defines several basic properties for \cref{def:probablity-value} which we do not summarize here. - -\begin{definition} -A finite set of natural numbers is defined as $\Nats_n := \left\{ 0, \dots , n-1 \right\}$. -%\lean{Fin} \leanok -\end{definition} - -\begin{definition} -A list $\mathbb{L}_n \colon \Nats_n \to \tau$ of type $\tau$ and length $n$ is a function defined for $\Nats_{n} := \left\{ 0, \dots , n-1 \right\}$ of type $\tau$. -%\lean{List} \leanok -\end{definition} - -The following definition generalizes the inverse of a function to non-injective function which do not have an inverse. -\begin{definition}[Preimage] -The \emph{preimage} $f^{-1}\colon \mathcal{Y} \to \mathcal{X}$ of a function $f\colon \mathcal{X} \to \mathcal{Y}$ is defined as -\begin{equation} \label{eq:preimage-set} -f^{-1}(y) := \left\{ x \in \mathcal{X} \mid f(x) = y \right\} . -\end{equation} -%\lean{Probability.Preimage} \leanok -\end{definition} - -\subsection{Defs} - -This section contains definitions of probability spaces, probability operators, and the expectation operator. See \texttt{Probability/Defs.lean}. -\begin{definition} -A \emph{finite probability distribution} $\Delta(n)$ for $n\in \Nats$ is defined as -\[ -\Delta_n := \left\{ p \in \mathbb{L}_n \mid \sum_{i \in \Nats_n} p_i = 1, p_i \ge 0, \forall i \in \Nats_n \right\}. -\] -We call a probability distribution \emph{degenerate} if $p_i = 1$ for some $i$; otherwise the probability distribution is \emph{supported}. -%\lean{Probability.Finprob} \leanok -\end{definition} - -The formal model we use to represent random events and processes is the following. -\begin{definition}[Complete Probability Space] \label{def:probability-space} - A \emph{complete probability space} consists of: - \begin{enumerate} - \item A sample space $\Omega := \Nats_n$, which is the set of all possible outcomes. - \item An \emph{implicit} event space $\mathcal{F} := 2^{\Omega}$, which we do not define explicitly in Lean. - \item A probability function $\mathbb{P} \in \Delta_N$ that satisfies that - \begin{enumerate} - \item $\mathbb{P}(\Omega) = 1$ - \item $\mathbb{P}(A_1 \cup A_2) = \mathbb{P}(A_1) + \mathbb{P}(A_2)$ for all $A_1, A_2\in \mathcal{F}$ such that $A_1 \cap A_2 = \emptyset$. - \end{enumerate} - \end{enumerate} -%\lean{Probability.Finprob} \leanok -\end{definition} -The notation $2^{\Omega}$ in \cref{def:probability-space} represents the power set of $\Omega$. - -The set $\mathcal{F}$ is often, but not always, the power set of $\Omega$. One reason it may not be a power set is that it is necessary when $\Omega$ is infinite; see Borel sets. Another reason to define $\mathcal{F}$ that is not a power set is to denote the available information. We will see an example of this later. Unless specified otherwise, we assume that the event set for a finite sample space is its power set and the $\sigma$-algebra of $\Real$ is the Borel set. - -\begin{definition}[Measure] - The measure $\PP [\mathcal{S}]$ of the set $\mathcal{S} \subseteq \Nats$ given a probability space is defined as - \[ - \PP [\mathcal{S}] := \sum_{i \in \Omega} \mathbb{I}[ i\in \mathcal{S} ]. - \] - The definition requires that the set is decidable. - %\lean{Probability.Finprob.measure} -\end{definition} - - -\begin{example} \label{exm:probability-space} - Suppose I want to model the behavior of coin flips. I have a coin that is equally likely to come up heads and tails. I flip the coin twice. One possible probability space would be - \begin{enumerate} - \item $\Omega := \left\{ \mathrm{HH}, \mathrm{HT}, \mathrm{TH}, \mathrm{TT} \right\}$. - \item $\mathcal{F} := 2^{\Omega } = \left\{ \emptyset, \Omega, \left\{ \mathrm{HH} \right\}, \left\{ \mathrm{HH}, \mathrm{HT} \right\}, \dots \right\}$. - \item $P(\left\{ \mathrm{HH} \right\}) := \frac{1}{4}, P(\left\{ \mathrm{HH}, \mathrm{HT} \right\}) := \frac{1}{2}$. The probabilities of other sets are derived using the properties in \cref{def:probability-space}. - \end{enumerate} -\end{example} - - -For the sake of completeness, we include also the notion of measurability. However, since we are only concerned with complete probability spaces, all random variables we define will be measurable. -\begin{definition} \label{def:preimage-measurable} -A function $f\colon \mathcal{X} \to \mathcal{Y}$ with $\sigma$-algebras $\mathcal{D} \subseteq 2^{\mathcal{X}}$ and $\mathcal{E} \subseteq 2^{\mathcal{Y}}$ is \emph{measurable} when the \emph{pre-image} of each $E\in \mathcal{E}$ is in $\mathcal{D}$: - \[ - f^{-1}(E) \in \mathcal{D}. - \] -Here, the application to $f^{-1}$ to a set $E \subseteq \mathcal{Y}$ is defined as: -\begin{equation} -f^{-1}(E) := \bigcup_{e \in E} f^{-1}(e) . -\end{equation} -\end{definition} -The measurability in the definition of random variables is required for us to reason about their probability distributions. - -\begin{definition}[Random Variable] - A \emph{random variable} is a measurable function $\tilde{x}\colon \Nats \to \mathcal{Y}$ where $\mathcal{Y}$ is a measurable set. Note that the way we define random variables, they are independent of a particular probability space. The measurability requirement is vacuous in probability spaces. - %\lean{Probability.FinRV} \leanok -\end{definition} -When manipulating random variables we denote them with a tilde, such as $\tilde{x}, \tilde{s}$. This corresponds to the common practice to use uppercase letters to denote random variables. We drop the tilde when treating the random variable as an ordinary function. - -\todo{We could get a lot of mileage from treating random variables as vectors. Expectation and probability are then linear operators on this vector space of random variables. This is only possible when using a complete vector space. Question: Do we need finiteness for this? That is, do random variables need to be defined for a finite $\Omega$?} - - -\begin{remark} -We define some ad-hoc operations on random variables. However, it would be better to treat them as a vector space. Then probability and expectation can be seen as linear operators on this vector space. - -Other notable definition is that the equality with a scalar $\tilde{x} = x_0$ is interpreted as a boolean random variable $\tilde{b}: \mathbb{B} \to \left\{ 0,1 \right\}$ : -\[ - b(\omega) := (\tilde{x} = x_0)(\omega) := \mathbb{I}[ x(\omega) = x_0]. -\] - -\end{remark} - -We use the operator (function) $\mathbb{P}$ denote the probability of the set of sample space that corresponds to a given condition. We use a non-standard definition for convenience (and show that this definition equals to the standard one later). -\begin{definition} -For any complete probability space, $(\Omega, \mathcal{F}, P)$ and a random variable $\tilde{b}\colon \Omega \to \mathbb{B}$, the operator $\PP$ is defined as -\[ -\PP\left[ \tilde{b} \right] := \sum_{i\in \Omega} \mathbb{P}(i) b(i), -\] -where the boolean is interpreted as $0$ for false and $1$ for true. -%\lean{Probability.Probability.probability} -\end{definition} -The definition can be seen as an inner product between the probabilities and the random variable. - -The following theorem shows that our definition of a probability operator is equal to the standard definition. -\begin{theorem} - For any probability space, $(\Omega, \mathcal{F}, P)$ and a random variable $\tilde{b}\colon \Omega \to \mathbb{B}$: - \[ - \PP\left[\tilde{b} \right] := \mathbb{P}[b^{-1}(\true)]. - \] - %%\lean{Probability.prob\_iprod\_eq\_def} \leanok -\end{theorem} - -\emph{Probability distributions} are typically defined for random variables. The distribution of a random variable is characterized by the \emph{cumulative distribution function}~(CDF), defined as follows. -\begin{definition}[CDF] - The \emph{cumulative distribution function} $F_{\tilde{x}}\colon \Real \to [0,1]$ of a real-valued random variable $\tilde{x}\colon \Omega \to \Real$ is defined as - \[ - F_{\tilde{x}}(x) := \PP\left[\tilde{x} \le x\right], \qquad \forall x\in \Real. - \] -\end{definition} - - -Some random variables also have a \emph{probability density function}~(PDF), and discrete random variables have a \emph{probability mass function}~(PMF). In this class, we will be primarily dealing with discrete random variables. -\begin{definition}[PMF] -The \emph{probability mass function} $p_{\tilde{x}}\colon \mathcal{E} \to [0,1]$ for a discrete random variable $\tilde{x}\colon \Omega \to \mathcal{E}$ with a finite $\mathcal{E}$ is defined as - \[ - p_{\tilde{x}}(e) := \PP\left[\tilde{x} = e\right], \qquad \forall e\in \mathcal{E} . - \] -%\lean{Probability.Probability.PMF} -\end{definition} - - -\begin{definition} \label{def:joint-probability} -The \emph{joint probability} of two random variables $\tilde{x}\colon \Omega \to \mathcal{X}$ and $\tilde{y} \colon \Omega \to \mathcal{Y}$ is defined as -\[ - \PP\left[\tilde{x} = x, \tilde{y} = y\right] := \PP\left[\tilde{x} = x \wedge \tilde{y} = y\right] , \forall x\in \mathcal{X}, y\in \mathcal{Y}. -\] -One can also define a joint PMF for two discrete random variables $p_{\tilde{x}, \tilde{y}}(x,y)$ for $x\in \mathcal{X}$ and $y\in \mathcal{Y}$ as -\[ - p_{\tilde{x}, \tilde{y}}(x,y) = \PP\left[\tilde{x} = x, \tilde{y} = y\right]. -\] -\end{definition} - -\begin{definition} \label{def:probability_cnd} -The \emph{conditional probability} of $\tilde{b}\colon \Omega \to \mathbb{B}$ on $\tilde{c}\colon \Omega \to \mathbb{B}$ is defined as -\[ -\PP\left[ \tilde{b} \mid \tilde{c} \right] := \frac{ \PP[\tilde{b} \wedge \tilde{c} ] }{ \PP \left[ \tilde{c} \right]} -\] -%%\lean{Probability.Probability.probability\_cnd} -\end{definition} - -\begin{definition}[Independent RV] - Events $A,B \in \mathcal{F}$ are independent when - \[ - \PP[ A \cap B ] = \PP[ A ] \cdot \PP[ B ]. - \] - Random variables $\tilde{x}\colon \Omega \to \mathcal{X}$ and $\tilde{y}\colon \Omega \to \mathcal{Y}$ are \emph{independent} when - \[ - F_{\tilde{x}, \tilde{y}}(x,y) = F_{\tilde{x}}(x) \cdot F_{\tilde{y}}(y), - \forall x\in \mathcal{X}, y\in \mathcal{Y}. - \] -\end{definition} -Discrete random variables $\tilde{x}$ and $\tilde{y}$ are independent if and only if -\[ - \PP\left[\tilde{x} = x, \tilde{y} = y\right] = \PP\left[\tilde{x} = x\right] \cdot \PP\left[\tilde{y} = y\right], \forall x\in \mathcal{X}, y\in \mathcal{Y}. -\] - - -The expectation of a random variable is its mean. As with probabilities, we use a non-standard definition for the sake of computational convenience and then show that it is equivalent to the common definition of probabilities. -\begin{definition}[Expected Value] \label{def:expectation} -Suppose that $\tilde{x}\colon \Omega \to \Real $ is a random variable and that $x(\Omega) := \left\{ x(\omega) \mid \omega \in \Omega \right\}$ is finite. Then -\[ - \E\left[\tilde{x}\right] := \sum_{\omega \in \Omega } x(\omega) \cdot \PP(\omega). -\] -\end{definition} - -The following theorem shows that our definition of expected value is equal to the standard definition of expected values. -\begin{theorem}[Expected Value] \label{def:expectation} -Suppose that $\tilde{x}\colon \Omega \to \Real $ is a random variable and that $x(\Omega) := \left\{ x(\omega) \mid \omega \in \Omega \right\}$ is finite. Then -\[ - \E\left[\tilde{x}\right] = \sum_{x\in \tilde{x}(\Omega)} x\cdot \PP\left[\tilde{x} = x\right]. -\] -%%\lean{Probability.Probability.exp\_eq\_correct} -\end{theorem} -In general probability spaces, the expectation is defined using the Lebesgue integral: -\[ - \E\left[\tilde{x}\right] := \int_{\Omega} \tilde{x} \, dP, -\] -which is equivalent to the form in the theorem above in finite probability spaces. - -Some of the most important properties of expectations are as follows. -\begin{theorem} - Assume $\tilde{x},\tilde{y}\colon \Omega \to \Real$ and $c\in \Real $. Then: - \begin{align*} - \E\left[c\cdot \tilde{x}\right] &= c \cdot \E\left[\tilde{x}\right]\\ - \E\left[\tilde{x} + \tilde{y}\right] &= \E\left[\tilde{x}\right] + \E\left[\tilde{y}\right]\\ - \E\left[\tilde{x} \cdot \tilde{y}\right] &= \E\left[\tilde{x}\right] \cdot \E\left[\tilde{y}\right] \quad \text{for independent } \tilde{x}, \tilde{y}. - \end{align*} -\end{theorem} - -\begin{definition} -A \emph{random variable} defined on a finite probability space $P$ is a mapping $\tilde{x}\colon \Omega \to \Real$. -%\lean{FinRV} \leanok -\end{definition} - -\subsection{Induction} - -This section includes properties that aim to enable one to perform induction on probability spaces. - -\subsection{Basic} - -This section states the basic properties of probability spaces, and operators. - -\begin{definition} \label{def:probability-measure} -A \emph{finite probability measure} $p\colon \Omega \to \Real_+$ on a finite set $\Omega$ is any function that satisfies -\[ -\sum_{\omega \in \Omega } p(\omega) = 1. -\] -%\lean{Probability.Finprob} \leanok -\end{definition} - - -\begin{theorem}[Law of the unconscious statistician] \label{thm:unconscious} - For any discrete $\tilde{x}\colon \Omega \to \Real$ and $g\colon \Real \to \Real$: - \[ - \E\left[g(\tilde{x})\right] = \sum_{x\in \mathcal{X}} g(x) \cdot \PP\left[\tilde{x} = x\right]. - \] -\end{theorem} - - -For the remainder of \cref{sec:prob-spac-expect}, we assume that $P = (\Omega, p)$ is a \emph{finite probability space}. All random variables are defined on the space $P$ unless specified otherwise. - -\subsection{OLD CONTENT: TO BE MOVED} - -Here, we define probability and expectation operators. - -\begin{definition} -A \emph{boolean} set is $\mathbb{B} = \left\{ \false, \true \right\}$. -%%\lean{Bool} \leanok -\end{definition} - -\begin{definition} \label{def:expect} -The \emph{expectation} of a random variable $\tilde{x} \colon \Omega \to \Real$ is -\[ -\E \left[ \tilde{x} \right] := \sum_{\omega \in \Omega } p(\omega ) \cdot \tilde{x}(\omega). -\] -%%\lean{Finprob.expect} \leanok -\end{definition} - -\begin{definition} \label{def:indicator} -An \emph{indicator} function $\I \colon \mathbb{B} \to \left\{ 0, 1 \right\}$ is defined for $b\in \mathbb{B}$ as -\[ -\I(b) := -\begin{cases} -1 &\text{if } b = \operatorname{true}, \\ -0 &\text{if } b = \operatorname{false}. -\end{cases} -\] -%\lean{Probability.Probability.indicator} \leanok -\end{definition} - -\begin{definition} \label{def:expect-cnd} -The \emph{conditional expectation} of $\tilde{x} \colon \Omega \to \Real$ conditioned on $\tilde{b} \colon \Omega \to \mathbb{B}$ is defined as -\[ -\E \left[ \tilde{x} \mid \tilde{b} \right] := -\frac{1}{\PP[\tilde{b}]} \E \left[ \tilde{x} \cdot \I \circ \tilde{b} \right], -\] -where we define that $x / 0 = 0$ for each $x\in \Real$. -%%\lean{Finprob.expect_cnd} \leanok -\end{definition} - - -\begin{remark} -It is common to prohibit conditioning on a zero probability event both for expectation and probabilities. In this document, we follow the Lean convention, where the division by $0$ is $0$; see \texttt{div\_zero}. However, even some basic probability and expectation results may require that we assume that the conditioned event does not have probability zero for it to hold. -\end{remark} - -\begin{definition} \label{def:expect-cnd-rv} -The \emph{random conditional expectation} of a random variable $\tilde{x} \colon \Omega \to \Real$ conditioned on $\tilde{y} \colon \Omega \to \mathcal{Y}$ for a finite set $\mathcal{Y}$ is the random variable $\E \left[ \tilde{x} \mid \tilde{y} \right]\colon \Omega \to \Real$ is defined as -\[ -\E \left[ \tilde{x} \mid \tilde{y} \right](\omega) -:= -\E \left[ \tilde{x} \mid \tilde{y} = \tilde{y}(\omega) \right], \quad \forall \omega \in \Omega. -\] -%%\lean{Finprob.expect_cnd_rv} \leanok -\end{definition} - -\begin{remark} -The Lean file defines expectations more broadly for a data type $\rho$ which is more general than just $\Real$. The main reason to generalize to both $\Real$ and $\Real_+$. However, in principle, the definitions could be used to reason with expectations that go beyond real numbers and may include other algebras, such as vectors or matrices. -\end{remark} - - -\begin{lemma} \label{lem:ind-and-eq-prod-ind} -Suppose that $\tilde{b}, \tilde{c} \colon \Omega \to \mathbb{B}$. Then: -\[ -\I \left( \tilde{b} \wedge \tilde{c} \right) = \I ( \tilde{b} ) \cdot \I ( \tilde{c} ), -\] -where the equality applies for all $\omega \in \Omega$. -\end{lemma} - -\begin{theorem} \label{thm:exp-zero-cond} -Suppose that $\tilde{c} \colon \Omega \to \mathbb{B}$ such that $\PP\left[ \tilde{c} \right] = 0$. Then for any $\tilde{x} \colon \Omega \to \Real$: -\[ -\E \left[ \tilde{x} \mid \tilde{c} \right] = 0. -\] -\end{theorem} -\begin{proof} -Immediate from the definition and the fact that $0 \cdot x = 0$ for $x\in \Real$. -\end{proof} - -\begin{theorem} \label{thm:prob-zero-cond} -Suppose that $\tilde{c} \colon \Omega \to \mathbb{B}$ such that $\PP\left[ \tilde{c} \right] = 0$. Then for any $\tilde{b} \colon \Omega \to \Real$: -\[ -\PP\left[ \tilde{b} \mid \tilde{c} \right] = 0. -\] -\end{theorem} -\begin{proof} -Immediate from \cref{thm:exp-zero-cond}. -\end{proof} - -\begin{theorem} \label{thm:prob-eq-prob-cond-prob} -Suppose that $\tilde{b}, \tilde{c} \colon \Omega \to \mathbb{B}$, then -\[ -\PP \left[ \tilde{b} \wedge \tilde{c} \right] -= -\PP \left[ \tilde{b} \mid \tilde{c} \right] \cdot \PP \left[ \tilde{c} \right]. -\] -\end{theorem} -\begin{proof} The property holds immediately when $\PP \left[ \tilde{c} \right] = 0$. Assume that $\PP \left[ \tilde{c} \right] > 0$. Then: - \begin{align*} - \PP \left[ \tilde{b} \wedge \tilde{c} \right] - &= \E \left[ \I(\tilde{b} \wedge \tilde{c}) \right] && \why{\cref{def:probability}} \\ - &= \E \left[ \I(\tilde{b}) \cdot \I(\tilde{c}) \right] && \why{\cref{lem:ind-and-eq-prod-ind}}\\ - &= \frac{1}{\PP \left[ \tilde{c} \right]}\E \left[ \I(\tilde{b}) \cdot \I(\tilde{c}) \right] \cdot \PP \left[ \tilde{c} \right] && \cdot 1 \\ - &= \E \left[ \I(\tilde{b}) \mid \tilde{c} \right] \cdot \PP \left[ \tilde{c} \right] - && \why{\cref{def:expect-cnd}} \\ - &= \PP \left[ \tilde{b} \mid \tilde{c} \right] \cdot \PP \left[ \tilde{c} \right] && \why{\cref{def:probability_cnd}}. - \end{align*} -\end{proof} - -\begin{lemma} \label{lem:prob-ge-measure} - Let $\tilde{y}\colon \Omega \to \mathcal{Y}$ with a finite $\mathcal{Y}$. Then - \[ - \PP[\tilde{y} = y(\omega)] \ge p(\omega ), \quad \omega \in \Omega . - \] -\end{lemma} -\begin{proof} - \begin{align*} - \PP[\tilde{y} = y(\omega)] - &= \sum_{\omega' \in \Omega } p(\omega) \cdot \I(\tilde{y}(\omega') = \tilde{y}(\omega)) && \why{\cref{def:probability}} \\ - &\ge p(\omega ) && \omega \in \Omega \why{ and } p(\omega') \ge 0, \forall \omega '\in \Omega. - \end{align*} -\end{proof} - - -\begin{remark} -\cref{thm:exp-congr} shows the equivalence of expectations for surely equal random variables. -\end{remark} - -\begin{theorem} \label{thm:exp-add-rv} - Random variables $\tilde{x}, \tilde{y} \colon \Omega \to \Real$ satisfy that - \[ - \E \left[ \tilde{x} + \tilde{y} \right] - = - \E \left[ \tilde{x} \right] + \E \left[ \tilde{y} \right]. - \] -\end{theorem} -\begin{proof} - From the distributive property of sums. -\end{proof} - -\begin{theorem} \label{thm:exp-const} - A random variable $\tilde{x}\colon \Omega \to \Real$ and $c\in \Real $ satisfies that - \[ - \E \left[ c \right] = c. - \] -\end{theorem} - - -\begin{theorem} \label{thm:exp-add-const} -Suppose that $\tilde{x} \colon \Omega \to \Real$ and $c\in \Real$. Then -\[ -\E \left[ c + \tilde{x} \right] = c + \E \left[ \tilde{x} \right]. -\] -%%\lean{Finprob.exp_add_const} -\end{theorem} -\begin{proof} - From \cref{thm:exp-add-rv,thm:exp-const}. -\end{proof} - - -\begin{theorem} \label{thm:exp-cnd-rv-add-const} -Suppose that $\tilde{x}, \tilde{y} \colon \Omega \to \Real$ and $\tilde{z}\colon \Omega \to \mathcal{V}$ are random variables and $c\in \Real$, such that $\tilde{y}(\omega) = c + \tilde{x}(\omega)$. Then -\[ -\E \left[ \tilde{y} \mid \tilde{z} \right](\omega) = c + \E \left[ \tilde{x} \mid \tilde{z} \right](\omega), \quad \forall \omega \in \Omega. -\] -%%\lean{Finprob.exp_cnd_rv_add_const} -\end{theorem} -\begin{proof} -From \cref{thm:exp-add-const}. -\end{proof} - -\begin{theorem} \label{thm:exp-monotone} - Suppose that $\tilde{x}, \tilde{y} \colon \Omega \to \Real$ satisfy that - \[ - \forall \omega \in \Omega, p(\omega ) > 0 \Rightarrow \tilde{x}(\omega) \ge \tilde{y}(\omega). - \] - Then - \[ - \E\left[\tilde{x}\right] \ge \E\left[\tilde{y}\right] . - \] - %%\lean{Finprob.exp_monotone} -\end{theorem} - - -\begin{theorem}[Congruence of Expectation] \label{thm:exp-congr} - Suppose that $\tilde{x}, \tilde{z} \colon \Omega \to \Real$ satisfy that - \[ - \forall \omega \in \Omega, p(\omega) > 0 \Rightarrow \tilde{x}(\omega) = \tilde{z}(\omega). - \] - Then - \[ - \E \left[ \tilde{x} \right] = \E \left[ \tilde{z} \right]. - \] -%%\lean{Finprob.exp_congr} -\end{theorem} -\begin{proof} -Immediately from the congruence of sums. -\end{proof} -\subsection{The Laws of The Unconscious Statisticians} - -\begin{theorem} \label{thm:exp-sum-val} -Let $\tilde{x} \colon \Omega \to \Real $ be a random variable. Then: -\[ -\E \left[ \tilde{x} \right] -= -\sum_{x\in \tilde{x}(\Omega)} \PP \left[ \tilde{x} = x \right] \cdot x. -\] -%%\lean{Finprob.exp_sum_val} -\end{theorem} -\begin{proof} -Let $\mathcal{X} := \tilde{x}(\Omega)$, which is a finite set. Then: -\begin{align*} -\E \left[ \tilde{x} \right] -&= \sum_{\omega \in \Omega } p(\omega ) \cdot \tilde{x}(\omega) && \why{\cref{def:expect}} \\ -&= \sum_{\omega \in \Omega } \sum_{x\in \mathcal{X}} p(\omega ) \cdot \tilde{x}(\omega) \cdot \I(x = \tilde{x}(\omega)) && \why{??} \\ -&= \sum_{\omega \in \Omega } \sum_{x\in \mathcal{X}} p(\omega ) \cdot x \cdot \I(x = \tilde{x}(\omega)) && \why{??} \\ -&= \sum_{x\in \mathcal{X}} x \cdot \sum_{\omega \in \Omega } p(\omega ) \cdot \I(x = \tilde{x}(\omega)) && \why{??}\\ -&= \sum_{x\in \mathcal{X}} x \cdot \E \left[ \I(x = \tilde{x}(\omega)) \right] && \why{\cref{def:expect}} \\ -&= \sum_{x\in \mathcal{X}} x \cdot \PP \left[x = \tilde{x}(\omega) \right]. && \why{\cref{def:probability}} -\end{align*} -\end{proof} - -The following theorem generalizes the theorem above. -\begin{theorem}\label{thm:exp-sum-val-cnd} -Let $\tilde{x} \colon \Omega \to \Real$ and $\tilde{b} \colon \Omega \to \mathcal{Y}$ be random variables. Then: -\[ - \E \left[ \tilde{x} \mid \tilde{b} \right] - = - \sum_{x\in \tilde{x}(\Omega)} \PP \left[ \tilde{x} = x \mid \tilde{b} \right] \cdot x. -\] -%%\lean{Finprob.exp_sum_val_cnd} -\end{theorem} - -\begin{theorem} \label{thm:exp-sum-val-cnd-rv} -Let $\tilde{x} \colon \Omega \to \Real$ and $\tilde{y} \colon \Omega \to \mathcal{Y}$ be random variables with $\mathcal{Y}$ finite. Then: -\[ - \E \left[ \E \left[ \tilde{x} \mid \tilde{y} \right] \right] - = - \sum_{y\in \mathcal{Y}} \E \left[ \tilde{x} \mid \tilde{y} = y \right] \cdot - \PP\left[ \tilde{y} = y \right]. -\] -%%\lean{Finprob.exp_sum_val_cnd_rv} -\end{theorem} - - -\subsection{Total Expectation and Probability} - - -\begin{theorem}[Law of Total Probability] \label{thm:total-probability} -Let $\tilde{b} \colon \Omega \to \mathbb{B}$ and $\tilde{y} \colon \Omega \to \mathcal{Y}$ be random variables with a finite set $\mathcal{Y}$. Then: -\[ -\sum_{y\in \mathcal{Y}} \PP\left[\tilde{b} \wedge (\tilde{y} = y) \right] = \PP \left[ \tilde{b} \right]. -\] -%%\lean{Finprob.total_expectation} -\end{theorem} - - -\begin{theorem}[Law of Total Expectation] \label{thm:total-expectation} -Let $\tilde{x} \colon \Omega \to \mathcal{X}$ and $\tilde{y} \colon \Omega \to \mathcal{Y}$ be random variables with a finite set $\mathcal{Y}$. Then: -\[ -\E\left[\E\left[\tilde{x} \mid \tilde{y}\right]\right] = \E \left[ \tilde{x} \right]. -\] -%%\lean{Finprob.total_expectation} -\end{theorem} -\begin{proof} -Recall that we are allowing the division by 0 and assume that $x / 0 = 0$. -\begin{align*} - \E\left[\E\left[\tilde{x} \mid \tilde{y}\right]\right] - &= \sum_{\omega \in \Omega } p (\omega) \cdot \E \left[ \tilde{x} \mid \tilde{y} \right](\omega) && \why{\cref{def:expect}}\\ - &= \sum_{\omega \in \Omega } p (\omega) \cdot \E \left[ \tilde{x} \mid \tilde{y} = \tilde{y}(\omega) \right] && \why{\cref{def:expect-cnd-rv}}\\ - &= \sum_{\omega \in \Omega } \frac{p(\omega)}{\PP \left[ \tilde{y}= \tilde{y}(\omega)\right]} \sum_{\omega'\in \Omega } p(\omega') \cdot \tilde{x}(\omega') \cdot \I\left( \tilde{y}(\omega') = \tilde{y}(\omega) \right) && \why{\cref{def:expect-cnd}} \\ - &= \sum_{\omega'\in \Omega } p(\omega') \cdot \tilde{x}(\omega') \cdot \sum_{\omega \in \Omega } \frac{p(\omega)}{\PP \left[ \tilde{y}= \tilde{y}(\omega)\right]} \I\left( \tilde{y}(\omega') = \tilde{y}(\omega) \right) && \why{rearrange} \\ - &= \sum_{\omega'\in \Omega } p(\omega') \cdot \tilde{x}(\omega') \cdot \sum_{\omega \in \Omega } \frac{p(\omega)}{\PP \left[ \tilde{y} = \tilde{y}(\omega')\right]} \I\left( \tilde{y}(\omega') = \tilde{y}(\omega) \right) && \why{equals when } \tilde{y}(\omega') = \tilde{y}(\omega) \\ - &= \sum_{\omega'\in \Omega } p(\omega') \cdot \tilde{x}(\omega') && \why{see below} \\ - &= \E \left[ \tilde{x} \right]. -\end{align*} -Above, we used the fact that -\[ - p(\omega') \cdot \sum_{\omega \in \Omega } \frac{p(\omega)}{\PP \left[ \tilde{y} = \tilde{y}(\omega')\right]} \I\left( \tilde{y}(\omega') = \tilde{y}(\omega) \right) = p(\omega'), -\] -which follows by analyzing two cases. First, when $p(\omega') = 0$, then the equality holds immediately. If $p(\omega') > 0$ then by \cref{lem:prob-ge-measure}, $\PP \left[ \tilde{y} = \tilde{y}(\omega')\right] > 0$, we get from \cref{def:probability} that -\[ - \sum_{\omega \in \Omega } \frac{p(\omega)}{\PP \left[ \tilde{y} = \tilde{y}(\omega')\right]} \I\left( \tilde{y}(\omega') = \tilde{y}(\omega) \right) - = - \frac{\PP \left[ \tilde{y} = \tilde{y}(\omega')\right]}{\PP \left[ \tilde{y} = \tilde{y}(\omega')\right]} - = 1, - \] - which completes the step. -\end{proof} -The following proof is simpler but may require some more advanced properties. -\begin{proof}[Alternate proof] -\begin{align*} -\E\left[\E\left[\tilde{x} \mid \tilde{y}\right] \right] -&= \sum_{y\in \mathcal{Y}} \E\left[\tilde{x} \mid \tilde{y} = y \right] \cdot \PP\left[ \tilde{y} = y \right] \\ -&= \sum_{y\in \mathcal{Y}} \sum_{x\in \mathcal{X}} x \cdot \PP\left[\tilde{x} = x \mid \tilde{y} = y \right] \cdot \PP\left[ \tilde{y} = y \right] \\ -&= \sum_{y\in \mathcal{Y}} \sum_{x\in \mathcal{X}} x \cdot \PP\left[\tilde{x} = x, \tilde{y} = y \right] \\ -&= \sum_{x\in \mathcal{X}} x \cdot \sum_{y\in \mathcal{Y}} \PP\left[\tilde{x} = x, \tilde{y} = y \right] \\ -&= \sum_{x\in \mathcal{X}} x \cdot \PP\left[\tilde{x} = x \right] = \E \left[ \tilde{x} \right]. -\end{align*} -\end{proof} - - -%% -- Non-Degeneracy -\subsection{Non-Degeneracy} -\begin{theorem}[Non-degeneracy of $L_1$-norm] \label{thm:non-degeneracy} -Let $(\Omega,\mathcal{F},\PP)$ be a discrete probability space with $\Omega=\{\omega_1,\omega_2,\ldots\}$ countable, and let $p_i=P(\{\omega_i\})\geq0$ with $\sum_i p_i =1$. Let $X$ be a random variable with $X_i=X(\omega_i)$. Then -\[ -\E[|X|]=0 \Longleftrightarrow \PP(\{\omega\in\Omega:X(\omega)=0\})=1. -\] -\end{theorem} - -\begin{proof} -\textit{Proof of ($\Leftarrow$):} Assume $\PP(X=0)=1$. -If $\PP(X=0)=1$, then $\PP(X\neq0)=0$. -In discrete terms $p_i=0$ for all $\omega_i$ such that $X_i\neq0$. -Now compute $\E[|X|]$ such that -\[ -\E[|X|]=\sum_i |X_i| \cdot p_i. -\] - - Case 1: $X_i=0\Rightarrow |X_i|p_i=0\cdot p_i=0$.\\ - Case 2: $X_i\neq0\Rightarrow p_i=0\Rightarrow |X_i|p_i=|X_i|\cdot0=0$. -Every term is zero, so $\E[|X|]=0$. - -\textit{Proof of ($\Rightarrow$):} Assume $\E[|X|]=0$. -We have -\[ -\E[|X|]=\sum_i |X_i| \cdot p_i=0, -\] -where $|X_i|p_i\geq0$ for all $i$. For a sum of nonnegative terms to be zero, each term must be zero -\[ -|X_i|p_i=0 \quad \text{for all } i. -\] -Thus, for each $i$, either $p_i=0$ or $|X_i|=0$ (i.e., $X_i=0$). -Let $N=\{\omega_i:X_i\neq0\}$. -For $\omega_i\in N$, we have $X_i\neq0\Rightarrow |X_i|\neq0\Rightarrow$ from $|X_i|p_i=0$ we must have $p_i=0$. -Therefore: -\[ -\PP(X\neq0)=\sum_{\omega_i\in N} p_i =\sum_{\omega_i\in N} 0=0. -\] -Thus $\P(X=0)=1-\PP(X\neq0)=1$. Hence, we conclude that -\[ -\E[|X|]=0 \Longleftrightarrow \PP(X=0)=1. -\] -\end{proof} - - -\section{Formal Decision Framework} - -\subsection{Markov Decision Process} - -\newcommand{\Fin}{\operatorname{Fin}} - -\begin{definition} \label{def:MDP} -A \emph{Markov decision process} $M := (S, A, P, r)$ consists of: -\begin{itemize} - \item a positive integer $S \in \Nats_{>0}$ representing the number of states, with index set $\Fin(S) = \{0,1, \dots, S-1\}$ - \item a positive integer $A \in \Nats_{>0}$ representing the number of actions, with index set $\Fin(A) = \{0,1, \dots, A-1\}$ - \item a transition function $P \colon \Fin(S) \times \Fin(A) \to \Delta(\Fin(S))$, mapping each state–action pair to finite probability distribution over next states - \item a reward function $r \colon \Fin(S) \times \Fin(A) \times \Fin(S) \to \Real$, mapping each transition \((s,a,s')\) to a real number -\end{itemize} -%\lean{MDPs.MDP} \leanok -\end{definition} - -\subsection{Histories} - -We implicitly assume in the remainder of the section an MDP $M = (S, A, P, r)$. -\begin{definition} \label{def:Hist} -A \emph{history} $h$ in a set of histories $\mathcal{H}$ is a sequence of states and actions defined for $M$ recursively as -\[ - h := \langle s_0 \rangle, \quad - \why{or} \quad - h := \langle h', a, s \rangle, -\] -where $s_0, s \in \Fin(S)$, $a \in \Fin(A)$, and $h'\in \mathcal{H}$. -%\lean{MDPs.Hist} \leanok -\end{definition} - -\begin{definition} \label{def:hist-length} -The \emph{length} $|h|\in \Nats$ of a history $h\in \mathcal{H}$ is defined as -\begin{align*} -|\langle s \rangle| &:= 0, \\ -|\langle h', s, a \rangle| &:= 1 + |h'|, \qquad h' \in \mathcal{H}. -\end{align*} -\end{definition} - -\begin{definition} \label{def:hist-ne} -The set $\mathcal{H}_{\mathrm{NE}}$ of \emph{non-empty histories} is -\[ -\mathcal{H}_{\mathrm{NE}} := \left\{ h \in \mathcal{H} \mid |h| \ge 1 \right\}. -\] -%%\lean{MDPs.HistNE} \leanok -\end{definition} - -\begin{definition}\label{def:histories} - \emph{Following histories} $\mathcal{H}(h,t) \subseteq \mathcal{H}$ for $h\in \mathcal{H}$ of length $t\in \Nats$ are defined recursively as - \[ - \mathcal{H}(h,t) := - \begin{cases} - \left\{ h \right\} &\text{if } t = 0, \\ - \left\{ \langle h', a, s \rangle \mid h\in \mathcal{H}(h', t-1), a\in \mathcal{A}, s\in \mathcal{S} \right\} &\text{otherwise}. \\ - \end{cases} - \] -\end{definition} - -\begin{definition} \label{def:histories-horizon} - The set of \emph{histories} $\mathcal{H}_{t}$ of \emph{length} $t \in \Nats$ is defined recursively as - \[ - \mathcal{H}_t = - \begin{cases} - \left\{ \langle s \rangle \mid s\in \mathcal{S} \right\} &\text{if } t = 0, \\ - \left\{ \langle h, a, s\rangle \mid h \in \mathcal{H}_{t-1}, a\in \mathcal{A}, s\in \mathcal{S} \right\} &text{otherwise}. - \end{cases} - \] - %%\lean{MDPs.HistoriesHorizon} \leanok -\end{definition} - -\begin{theorem} \label{thm:hist-lenth-eq-horizon} - For $h \in \mathcal{H}$: - \[ - |h'| = |h| + t, \qquad \forall h'\in \mathcal{H}(h, t). - \] -%%\lean{MDPs.hist_lenth_eq_horizon} -\end{theorem} -\begin{proof} -The theorem follows by induction on $t$ from the definition. -\end{proof} - -\begin{definition} - We use $\tilde{s}_k\colon \mathcal{H} \to \mathcal{S}$ to denote the 0-based $k$-th state of each history. - %%\lean{MDPs.state}\leanok -\end{definition} - -\begin{definition} - We use $\tilde{a}_k\colon \mathcal{H} \to \mathcal{A}$ to denote the 0-based $k$-th action of each history. - %%\lean{MDPs.action}\leanok -\end{definition} - -\begin{definition} \label{def:reward} -The \emph{history-reward} random variable $\rew\colon \mathcal{H} \to \Real$ for $h = \langle h', a, s \rangle \in \mathcal{H}$ for $h'\in \mathcal{H}$, $a\in \mathcal{A}$, and $s\in \mathcal{S}$ is defined recursively as -\[ -\rew(h) := r(s_{|h|}(h'), a, s) + r_{\mathrm{h}}(h'). -\] -%%\lean{MDPs.reward} \leanok -\end{definition} - -\begin{definition}\label{reward_at} -The \emph{history-reward} random variable $\rew_k \colon \mathcal{H} \to \Real$ for $h = \langle h', a, s \rangle \in \mathcal{H}$ for $h'\in \mathcal{H}$, $a\in \mathcal{A}$, and $s\in \mathcal{S}$ is defined as the $k$-th reward (0-based) of a history. -%%\lean{MDPs.reward_at} \leanok -\end{definition} - -\begin{definition}\label{reward_to} -The \emph{history-reward} random variable $\rew_{\le k} \colon \mathcal{H} \to \Real$ for $h = \langle h', a, s \rangle \in \mathcal{H}$ for $h'\in \mathcal{H}$, $a\in \mathcal{A}$, and $s\in \mathcal{S}$ is defined as the sum of all $k$-th or earlier rewards (0-based) of a history. -%%\lean{MDPs.reward_to} \leanok -\end{definition} - -\begin{definition}\label{reward_from} -The \emph{history-reward} random variable $\rew_{\ge k}\colon \mathcal{H} \to \Real$ for $h = \langle h', a, s \rangle \in \mathcal{H}$ for $h'\in \mathcal{H}$, $a\in \mathcal{A}$, and $s\in \mathcal{S}$ is defined as the sum of $k$-th or later reward (0-based) of a history. -%%\lean{MDPs.reward_from} \leanok -\end{definition} - -\subsection{Policies} - -\begin{definition} \label{def:decision-rule} -The set of \emph{decision rules} $\mathcal{D}$ is defined as \(\mathcal{D} := \mathcal{A}^{\mathcal{S}}. \) A single action $a \in \mathcal{A}$ can also be interpreted as a decision rule $\mathcal{d} := s \mapsto a$. -%%\lean{MDPs.DecisionRule}\leanok -\end{definition} - -\begin{definition} \label{def:policy-hr} -The set of \emph{history-dependent policies} is \(\PiHR := \Delta(\mathcal{A})^{\mathcal{H}}. \) -%%\lean{MDPs.PolicyHR} \leanok -\end{definition} - - -\begin{definition} \label{def:policy-md} -The set of \emph{Markov deterministic policies} $\PiMD$ is \(\PiMD := \mathcal{D}^{\Nats}. \) -A Markov deterministic policy $\pi \in \PiMD$ can also be interpreted as $\bar{\pi} \in \PiHR$: -\[ - \bar{\pi}(h) := \delta \left[ \pi(|h|, s_{|h|}(h)) \right], -\] -where $\delta$ is the Dirac distribution, and $s_{|h|}$ is the history's last state. -%%\lean{MDPs.PolicyMD}\leanok -\end{definition} - -\begin{definition} \label{def:policy-sd} -The set of \emph{stationary deterministic policies} $\Pi_{\mathrm{SD}}$ is defined as \(\Pi_{\mathrm{SD}} := \mathcal{D}. \) -A stationary policy $\pi \in \Pi_{\mathrm{SD}}$ can be interpreted as $\bar{\pi} \in \PiHR$: -\[ - \bar{\pi}(h) := \delta \left[ \pi(s_{|h|}(h)) \right], -\] -where $\delta$ is the Dirac distribution and $s_{|h|}$ is the history's last state. -%%\lean{MDPs.PolicySD}\leanok -\end{definition} - -\subsection{Distribution} - - -\begin{definition}\label{def:hist-dist} -The \emph{history probability distribution} $p^{\mathrm{h}}_T \colon \PiHR \to \Delta(\mathcal{H}(h,t))$ and $\pi \in \PiHR$ is defined for each $T \in \Nats$ and $h\in \mathcal{H}(\hat{h},t)$ as -\[ -(p^{\mathrm{h}}_T(\pi))(h) := -\begin{cases} -\I(h = \hat{h}) &\text{if } T = 0, \\ -p_{T-1}^{\mathrm{h}}(h', \pi) \cdot \pi(h',a) \cdot p(s_{|h|}(h'), a , s) &\text{if } T > 1 \wedge h = \langle h', a, s \rangle. -\end{cases} -\] -Moreover, the function $p^{\mathrm{h}}$ maps policies to correct probability distribution. -%%\lean{MDPs.HistDist} \leanok -\end{definition} - -TODO: This definition needs to be updated. -A probability space $(\Omega_{h,t}, 2^{\Omega_{h,t} }, \hat{p}_{h,\pi})$ which is defined as -\begin{align} -\label{eq:omega-def} - \Omega_{h,t} &:= \left\{ h' \in \mathcal{H}_{|h| + t} \mid s_k(h) = s_k(h') \wedge a_k(h) = a_k(h'), \forall k \le |h| \right\}, \\ - \label{eq:phat-def} -\hat{p}_{h,\pi}\left(\left< h', a, s \right>\right) -&:= \begin{cases} -1 &\text{if } \left< h', a, s \right> = h, \\ -\hat{p}_{h,\pi}(h') \cdot \pi(h', a) \cdot p(s_{|h'|}(h'), a, s) -&\text{otherwise}, -\end{cases} -\end{align} -for each $\left< h', a, s \right> \in \Omega_{h,t}$. The random variables are defined as $\tilde{s}_k(h') := s_k(h')$, $\tilde{a}_k(h') := a_k(h')$, $\forall h'\in \Omega_{h,t}$. We interpret the subscripts analogously on all operators, including other risk measures, and $\E$, and $\mathbb{P}$. - - - -\begin{definition}\label{def:expect-h} -The \emph{history-dependent expectation} is defined for each $t\in \Nats$, $\pi\in \PiHR$, $\hat{h}\in \mathcal{H}$ and a $\tilde{x}\colon \mathcal{H} \to \Real$ as -\begin{equation*} -\E^{\hat{h}, \pi, t} [\tilde{x}] -:= \E \left[ \tilde{x} \right] -= \sum_{h \in \mathcal{H}(\hat{h}, t)} p^{\mathrm{h}}(h, \pi) \cdot \tilde{x}(h). -\end{equation*} -In the $\E$ operator above, the random variable $\tilde{x}$ lives in a probability space $(\Omega, p)$ where $\Omega = \mathcal{H}(\hat{h}, t)$ and $p(h) = p^{\mathrm{h}}(h, \pi), \forall h\in \Omega$. -Moreover, if $\hat{h}$ is a state, then it is interpreted as a history with the single initial state. -%%\lean{MDPs.expect_h} \leanok -\end{definition} - - -\begin{definition}\label{def:expect-h-cnd} -The \emph{history-dependent expectation} is defined for each $t\in \Nats$, $\pi\in \PiHR$, $\hat{h}\in \mathcal{H}$, $\tilde{x}\colon \mathcal{H} \to \Real$, $\tilde{b}\colon \mathcal{H} \to \mathcal{\mathbb{B}}$ as -\begin{equation*} -\E^{\hat{h}, \pi, t} [\tilde{x} \mid \tilde{b} ] -:= \E \left[ \tilde{x} \mid \tilde{b} \right]. -\end{equation*} -In the $\E$ operator above, the random variables $\tilde{x}$ and $\tilde{b}$ live in a probability space $(\Omega, p)$ where $\Omega = \mathcal{H}(\hat{h}, t)$ and $p(h) = p^{\mathrm{h}}(h, \pi), \forall h\in \Omega$. -Moreover, if $\hat{h}$ is a state, then it is interpreted as a history with the single initial state. -%%\lean{MDPs.expect_h_cnd} \leanok -\end{definition} - -\begin{definition}\label{def:expect-h-cnd-rv} -The \emph{history-dependent expectation} is defined for each $t\in \Nats$, $\pi\in \PiHR$, $\hat{h}\in \mathcal{H}$, $\tilde{x}\colon \mathcal{H} \to \Real$, $\tilde{y}\colon \mathcal{H} \to \mathcal{\mathcal{V}}$ as -\begin{equation*} -\E^{\hat{h}, \pi, t} [\tilde{x} \mid \tilde{y} ](h) -:= \E \left[ \tilde{x} \mid \tilde{y} = \tilde{y}(h) \right](h), \quad\forall h\in \mathcal{H}(\hat{h}, t). -\end{equation*} -In the $\E$ operator above, the random variables $\tilde{x}$ and $\tilde{h}$ live in a probability space $(\Omega, p)$ where $\Omega = \mathcal{H}(\hat{h}, t)$ and $p(h) = p^{\mathrm{h}}(h, \pi), \forall h\in \Omega$. -Moreover, if $\hat{h}$ is a state, then it is interpreted as a history with the single initial state. -%%\lean{MDPs.expect_h_cnd_rv} \leanok -\end{definition} - - -\subsection{Basic Properties} - - -\begin{theorem} \label{thm:exph-congr} -Assume $\tilde{x} \colon \mathcal{H} \to \Real$ and $c \in \Real$. Then $\forall h\in \mathcal{H}, \pi\in \PiHR, t \in \Nats$: -\[ - \E^{\hat{h}, \pi, t} \left[ c + \tilde{x} \right] - = - c + \E^{\hat{h}, \pi, t} \left[ \tilde{x} \right]. -\] -%%\lean{MDPs.exph_congr} \leanok -\end{theorem} -\begin{proof} -Directly from \cref{thm:exp-congr}. -\end{proof} - -\begin{theorem} \label{thm:exph-add-rv} -Suppose that $\tilde{x}, \tilde{y} \colon \mathcal{H} \to \Real $. Then $\forall h\in \mathcal{H}, \pi\in \PiHR, t \in \Nats$: -\[ - \E^{\hat{h}, \pi, t} \left[ \tilde{x} + \tilde{y} \right] - = - \E^{\hat{h}, \pi, t} \left[ \tilde{x} \right] + \E^{\hat{h}, \pi, t} \left[ \tilde{y} \right]. -\] -%%\lean{MDPs.exph_add_rv} -\end{theorem} -\begin{proof} - From \cref{thm:exp-add-rv}. -\end{proof} - -\begin{theorem} \label{thm:exph-const} -Suppose that $c\in \Real$. Then $\forall h\in \mathcal{H}, \pi\in \PiHR, t \in \Nats$: -\[ - \E^{\hat{h}, \pi, t} \left[ c \right] = c. -\] -%%\lean{MDPs.exph_const} -\end{theorem} -\begin{proof} - From \cref{thm:exp-const}. -\end{proof} - -\begin{theorem} \label{thm:exph-add-const} -Suppose that $\tilde{x}, \tilde{y} \colon \mathcal{H} \to \Real $ satisfy that $\tilde{x}(h) = \tilde{y}(h), \forall h \in \mathcal{H}$. Then $\forall h\in \mathcal{H}, \pi\in \PiHR, t \in \Nats$: -\[ - \E^{\hat{h}, \pi, t} \left[ \tilde{x} \right] - = - c + \E^{\hat{h}, \pi, t} \left[ \tilde{y} \right]. -\] -%%\lean{MDPs.exph_add_const} -\end{theorem} -\begin{proof} -From \cref{thm:exp-add-const}. -\end{proof} - -\begin{theorem}\label{thm:expret-eq-sum-rew} -For each $\hat{h}\in \mathcal{H}$, $\pi \in \PiHR$, and $t\in \Nats$: -\[ -\E^{\hat{h}, \pi, t} \left[ \rew \right] -= -\E^{\hat{h}, \pi, t} \left[ \sum_{k=0}^{|\tilde{\id}|-1} r(\tilde{s}_k, \tilde{a}_k, \tilde{s}_{k+1}) \right], -\] -where $\tilde{\id}(h)$ is the identity function, $|\cdot|$ is the length of a history (0-based), $\tilde{s}_k\colon \mathcal{H} \to \mathcal{S}$ and $\tilde{a}_k\colon \mathcal{H} \to \mathcal{A}$ are the 0-based $k$-th state and action, respectively of each history. -%%\lean{MDPs.expret_eq_sum_rew} \leanok -\end{theorem} -\begin{proof} -Follows from \cref{thm:exph-congr} and the equality of the reward function $\tilde{r}^{\mathrm{h}}$ and the sum in the expectation. -\end{proof} - -\begin{theorem}\label{thm:sum-rew-eq-sum-rew-rg} -For each $h\in \mathcal{H}$, $\pi \in \PiHR$, and $t\in \Nats$: -\[ -\E^{h, \pi, t} \left[ \rew \right] -= -\rew(h) + \E^{h, \pi, t} \left[ \rew_{\ge k_0} \right], -\] -where $k_0:=|h|$. - %%\lean{MDPs.sum_rew_eq_sum_rew_rg } -\end{theorem} -\begin{proof} -Follows from \cref{thm:exph-add-const}. -\end{proof} - -\begin{theorem}\label{thm:sum-rew-cnd} -For each $\hat{h}\in \mathcal{H}$, $\pi \in \PiHR$, $t\in \Nats$, $h\in \mathcal{H}$: -\begin{equation*} - \PP^{\hat{h},\pi.t}[\tilde{s}_{k_0}=\tilde{s}_{k_0}(\omega) \wedge \tilde{a}_{k_0} = \tilde{a}_{k_0}(\omega) ] > 0 -\quad\Rightarrow \quad - \E^{\hat{h}, \pi, t} \left[ \rew_{k_0} \mid \tilde{s}_{k_0}, \tilde{a}_{k_0}\right](h) = \rew_{k_0}(h), \forall h \in \mathcal{H}. -\end{equation*} -where $k_0:= |\hat{h}|$. -%%\lean{MDPs.sum_rew_eq_sum_rew_rg } -\end{theorem} -\begin{proof} - From \cref{thm:exp-const}. -\end{proof} - -\begin{theorem} \label{thm:exph-horizon-trim} -Assume $h \in \mathcal{H}$ and $f \colon \mathcal{H} \to \Real$ such that $s_0 := s_{|h|}(h)$ -\[ -f(\left< h, a, s \right>) = f(\left< s_0, a, a \right>) , \forall a\in \mathcal{A}, s\in \mathcal{S}. -\] -Then -\[ -\E^{h,\pi,1}\left[ \tilde{f} \right] = -\E^{s_0,\pi,1}\left[ \tilde{f} \right]. -\] -\end{theorem} -%%\lean{MDPs.exph_horizon_trim } -\begin{proof} - Directly from the definition of the expectation. -\end{proof} - -\subsection{Total Expectation} - -\begin{theorem}[Total Expectation] \label{thm:total-expectation-h} - For each $h\in \mathcal{H}$, $\pi\in \PiHR$, $t\in \Nats$, $\tilde{x}\colon \mathcal{H} \to \Real$ and $\tilde{y}\colon \mathcal{H} \to \mathcal{V}$: - \[ - \E^{h, \pi, t} \left[ \E^{h, \pi, t} \left[ \tilde{x} \mid \tilde{y} \right] \right] - = - \E^{h, \pi, t} \left[ \tilde{x} \right]. - \] -%%\lean{MDPs.total_expectation_h} -\end{theorem} -\begin{proof} -From \cref{thm:total-expectation}. -\end{proof} - -\begin{theorem}\label{thm:exp-horizon-cut} -Suppose that the random variable $\tilde{x}\colon \mathcal{H} \to \Real$ satisfies for some $k, t \in \Nats$, with $k \le t$, that - \[ - \tilde{x}(h) = \tilde{x}(h_{\le k}) , \forall h\in \mathcal{H}, - \] - where $h_{\le k}$ is the prefix of $h$ of length $k$. Then for each $h\in \mathcal{H}, \pi\in \PiHR$: - \[ - \E^{h,\pi,t} \left[ \tilde{x} \right] = \E^{h,\pi,k} \left[ \tilde{x} \right]. - \] - %%%\lean{MDPs.exp_horizon_cut} -\end{theorem} - - -\subsection{Conditional Properties} - -\begin{theorem} \label{thm:exph-cond-eq-hist} -For each $\beta >0$, $h\in \mathcal{H}$, $\pi\in \PiHR$, $t\in \Nats$, $\tilde{x}\colon \mathcal{H} \to \Real$, $s\in \mathcal{S}$, $a\in \mathcal{A}$: -\[ - \E^{h,\pi,t+1}[\tilde{x} \mid \tilde{a}_{|h|} = a, \tilde{s}_{|h|+1} = s] - = - \E^{\langle h, a, s\rangle,\pi,t}[\tilde{x}], -\] -%%\lean{MDPs.exph_cond_eq_hist} -\end{theorem} -\begin{proof} - Let - \[ - \begin{aligned} - b &:= \mathbb{P}^{h,\pi,t+1} \left[ \tilde{a}_{|h|} = a, \tilde{s}_{|h|+1} = s \right] \\ - &= \hat{p}_{h,\pi}(h') \cdot \pi(h', a) \cdot p(s_{|h'|}(h'), a, s) \\ - &> 0 - \end{aligned} - \] - where the inequality holds from the hypothesis. Also, let - \[ - \mathbb{B} := \left\{ h'\in \Omega_{h,t+1} \mid a_{|h|}(h') = a \wedge s_{|h|+1}(h') = s\right\}. - \] - Note that - \begin{equation} \label{eq:set-equivalence} - \mathbb{B} = \Omega_{\left< h', a, s \right>, t}, -\end{equation} - which can be seen by algebraic manipulation from \eqref{eq:omega-def}. - - Using the notation above, we can show the result as - \begin{align*} - \E^{h,\pi,t+1}[\tilde{x} \mid \tilde{a}_{|h|} = a, \tilde{s}_{|h|+1} = s] - &= \frac{1}{b} \sum_{h'\in \mathbb{B}} \hat{p}_{h,\pi}(h') \cdot x(h') - && \why{definition}\\ - &= \frac{1}{b} \sum_{h'\in \Omega_{\left< h', a, s \right>, t}} \hat{p}_{h,\pi}(h') \cdot x(h') - && \why{\cref{eq:set-equivalence}} \\ - &= \sum_{h'\in \Omega_{\left< h', a, s \right>, t}} \hat{p}_{\left,\pi}(h') \cdot x(h') - && \why{\cref{eq:phat-def}} \\ - &= \E^{\langle h, a, s\rangle,\pi,t}[\tilde{x}]. && \why{definition} - \end{align*} -\end{proof} - -\section{Dynamic Program: History-Dependent Finite Horizon} - -In this section, we derive dynamic programming equations for histories. We assume an MDP $M = (\mathcal{S}, \mathcal{A}, p, r)$ throughout this section. - -The main idea of the proof is to: -\begin{enumerate} -\item Derive (exponential-size) dynamic programming equations for the history-dependent value function of history-dependent policies - \begin{enumerate} - \item Define the value function - \item Define an optimal value function - \end{enumerate} -\item Show that value functions decompose to equivalence classes -\item Show that the value function for the equivalence classes can be computed efficiently -\end{enumerate} - -\subsection{Definitions} - -\begin{definition} \label{def:ObjectiveFH} - A finite horizon objective definition is given by $O := (s_0, T)$ where $s_0\in \mathcal{S}$ is the initial state and $T\in \Nats$ is the horizon. - %%\lean{MDPs.ObjectiveFH} \leanok -\end{definition} - -In the reminder of the section, we assume an objective $O = (s_0, T)$. -\begin{definition} \label{sec:objective-fh} - The \emph{finite horizon objective function} for and objective $O$ is $\pi\in \PiHR$ is defined as - \[ - \rho(\pi, O) - := - \E^{s_0, \pi, T} \left[\rew \right]. - \] - %%\lean{MDPs.objective_fh} \leanok -\end{definition} - -\begin{definition}\label{def:optimal-fh} - A policy $\pi\opt\in \PiHR$ is \emph{return optimal} for an objective $O$ if - \[ - \rho(\pi\opt, O) \ge \rho(\pi, O), \quad \forall \pi \in \PiHR. - \] - %%\lean{MDPs.OptimalVF_fh} \leanok -\end{definition} - -\begin{definition} \label{def:values-h} - The \emph{set of history-dependent value functions} $\mathcal{U}$ is defined as - \[ - \mathcal{U} := \Real^{\mathcal{H}}. - \] - %%\lean{MDPs.ValuesH} \leanok -\end{definition} - -\begin{definition} \label{def:u-pi} -A \emph{history-dependent policy value function} $\hat{u}^{\pi}_t\colon \mathcal{H} \to \Real$ for each $h\in \mathcal{H}$, $\pi \in \PiHR$, and $t\in \Nats$ is defined as - \[ - \hat{u}^{\pi}_t(h) := \E^{h, \pi, t} \left[ \rew_{\ge |h|} \right], - \] - %%\lean{MDPs.hvalue_π} \leanok -\end{definition} - - -\begin{definition} -The optimal history-dependent value function $\hat{u}_t\opt\colon \mathcal{H} \to \Real$ is defined for a horizon $t\in \Nats$ as - \[ - \hat{u}\opt_t(h) := \sup_{\pi\in \PiHR} \hat{u}_t^{\pi}(h). - \] -\end{definition} - -The following definition is another way of defining an optimal policy. -\begin{definition} \label{def:optimal-fh} -For each $t\in \Nats$, a policy $\pi\opt \in \PiHR$ is optimal if -\[ - \hat{u}_t^{\pi\opt}(h) \ge \hat{u}_t^{\pi}(h), \quad \forall \pi \in \PiHR, h\in \mathcal{H}. - \] - %%\lean{MDPs.Optimal_fh}\leanok -\end{definition} - -\begin{theorem} \label{def:optimalvf-imp-optimal} -A policy $\pi\opt \in \PiHR$ optimal in \cref{def:optimal-fh} is also optimal in \cref{def:optimal-fh} for any initial state $s_0$ and horizon $T$. -%%\lean{MDPs.optimalvf_imp_optimal} -\end{theorem} - -\subsection{History-dependent Dynamic Program} - -The following definitions of history-dependent value functions use a dynamic program formulation. - -\begin{definition} \label{def:DPhpi} -The \emph{history-dependent policy Bellman operator} $L_{\mathrm{h}}^{\pi} \colon \mathcal{U} \to \mathcal{U}$ is defined for each $\pi\in \PiHR$ as - \[ - (L_{\mathrm{h}}^{\pi} \tilde{u}) (h) - := - \E^{h, \pi, 1} \left[ \rew_{|h|} + \tilde{u} \right], \quad \forall h\in \mathcal{H}, \tilde{u} \in \mathcal{U}, - \] - where the value function $\tilde{u}$ is interpreted as a random variable on defined on the sample space $\Omega = \mathcal{H}$. - %%\lean{MDPs.DPhπ} \leanok -\end{definition} - -\begin{definition}\label{def:DPhopt} -The \emph{history-dependent optimal Bellman operator} $L_{\mathrm{h}}\opt \colon \mathcal{U} \to \mathcal{U}$ is defined as - \[ - (L\opt_{\mathrm{h}} \tilde{u}) (h) - := - \max_{a\in \mathcal{A}} \E^{h, a, 1} \left[ \rew_{|h|} + \tilde{u} \right], \quad \forall h\in \mathcal{H}, \tilde{u} \in \mathcal{U}, - \] - where the value function $\tilde{u}$ is interpreted as a random variable on defined on the sample space $\Omega = \mathcal{H}$. - %%\lean{MDPs.DPhopt}\leanok -\end{definition} - -\begin{definition} \label{def:u-dp-pi} - The history-dependent \emph{DP value function} $u_t^{\pi} \in \mathcal{U}$ for a policy $\pi\in \PiHR$ and $t\in \Nats$ is defined as - \[ - u_t^{\pi} := - \begin{cases} - 0 &\text{if } t = 0, \\ - L_{\mathrm{h}}^{\pi} u_{t-1}^{\pi} &\text{otherwise}. - \end{cases} - \] - %%\lean{MDPs.u_dp_π}\leanok -\end{definition} - -\begin{definition} \label{def:u-dp-opt} - The history-dependent \emph{DP value function} $u_t\opt \in \mathcal{U}$ for $t\in \Nats$ is defined as - \[ - u_t\opt := - \begin{cases} - 0 &\text{if } t = 0, \\ - L_{\mathrm{h}}\opt u_{t-1}\opt &\text{otherwise}. - \end{cases} - \] - %%\lean{MDPs.u_dp_opt}\leanok -\end{definition} - -\begin{lemma}\label{thm:dp-opt-ge-dp-pi} - Suppose that $u^1, u^2 \in \mathcal{U}$ satisfy that $u^1(h) \ge u^2(h), \forall h\in \mathcal{H}$. Then - \[ - (L_{\mathrm{h}}\opt u^1)(h) \ge - (L_{\mathrm{h}}^{\pi} u^2)(h), \quad \forall \pi\in \PiHR, h\in \mathcal{H}. - \] -\end{lemma} -%%\lean{MDPs.dp_opt_ge_dp_pi} -\begin{proof} - From \cref{thm:exp-monotone}. -\end{proof} - -The following theorem shows the history-dependent value function can be computed by the dynamic program. The following theorem is akin to \citep[theorem~4.2.1]{Puterman2005}. -\begin{theorem} \label{thm:dph-correct-vf} -For each $\pi\in \PiHR$ and $t\in \Nats$: - \[ - \hat{u}^{\pi}_t(h) - = - u^{\pi}_t(h), \quad \forall h\in \mathcal{H}. - \] - %%\lean{MDPs.dph_correct_vf } -\end{theorem} -\begin{proof} -By induction on $t$. The base case for $t=0$ follows from the definition. The inductive case for $t + 1$ follows for each $h\in \mathcal{H}$ when $|h| = k_0$ as -{\allowdisplaybreaks -\begin{align*} -\hat{u}_{t+1}^{\pi}(h) -&= \E^{h, \pi, t+1} \left[ \rew_{\ge k_0} \right] && \why{\cref{def:u-pi}}\\ -&= \E^{h, \pi, t+1} \left[ \E^{h, \pi, t+1} \left[ \rew_{\ge k_0} \mid \tilde{a}_{k_0}, \tilde{s}_{k_0+1} \right] \right] && \why{\cref{thm:total-expectation-h}} \\ -&= \E^{h, \pi, t+1} \left[\rew_{k_0} + \E^{h, \pi, t+1} \left[ \rew_{\ge k_0+1} \mid \tilde{a}_{k_0}, \tilde{s}_{k_0+1} \right] \right] && \why{\cref{thm:sum-rew-cnd}} \\ -&= \E^{h, \pi, t+1} \left[\rew_{k_0} + \E^{\langle h,\tilde{a}_{k_0},\tilde{s}_{k_0+1}\rangle, \pi, t} \left[ \rew_{\ge k_0+1} \right]\right] && \why{\cref{thm:exph-cond-eq-hist}} \\ -&= \E^{h, \pi, t+1} \left[\rew_{k_0} + \hat{u}_t(\langle h,\tilde{a}_{k_0},\tilde{s}_{k_0+1}\rangle; \pi) \right] && \why{\cref{def:u-pi}} \\ -&= \E^{h, \pi, t+1} \left[\rew_{k_0} + u^{\pi}_t(\langle h,\tilde{a}_{k_0},\tilde{s}_{k_0+1}\rangle) \right] && \why{inductive assm} \\ -&= \E^{h, \pi, 1} \left[\rew + \tilde{u}^{\pi}_t \right] && \why{\cref{thm:exp-horizon-cut}} \\ -&= L_{\mathrm{h}}^{\pi} u_t^{\pi} && \why{\cref{def:DPhpi}} \\ -&= u^{\pi}_t(h). && \why{\cref{def:u-dp-pi}} -\end{align*} -} -Also, we use $\tilde{u}_t^{\pi}$ to emphasize when we treat $u^{\pi}_t$ as a random variable. -\end{proof} - -The following theorem is akin to \citep[theorem~4.3.2]{Puterman2005}. -\begin{theorem}\label{thm:dph-opt-vf-opt} -For each $t\in \Nats$: - \[ - u\opt_t(h) - \ge - \hat{u}_t(h; \pi), \quad \forall h\in \mathcal{H}, \pi\in \PiHR. - \] - %%\lean{MDPs.dph_opt_vf_opt} -\end{theorem} -\begin{proof} - By induction on $t$. The base case is immediate. The inductive case follows for $t+1$ as follows. For each $\pi\in \PiHR$: - \begin{align*} - u_{t+1}\opt(h) - &= (L_{\mathrm{h}}\opt u_t\opt)(h) && \why{\cref{def:DPhopt}} \\ - &\ge (L_{\mathrm{h}}^{\pi} \hat{u}_t(\cdot; \pi))(h) && \why{ind asm, \cref{thm:dp-opt-ge-dp-pi}} \\ - &= (L_{\mathrm{h}}^{\pi} u_t^{\pi})(h) && \why{\cref{thm:dph-correct-vf}} \\ - &= u_t^{\pi}(h) && \why{\cref{def:u-pi}} \\ - &= \hat{u}_t(h; \pi). && \why{\cref{thm:dph-correct-vf}} \\ - \end{align*} -\end{proof} - -\section{Expected Dynamic Program: Markov Policy} - -\subsection{Optimality} -We discuss results needed to prove the optimality of Markov policies. - -\begin{definition} \label{def:Values} - The set of \emph{independent value functions} is defined as $\mathcal{V} := \Real^{\mathcal{S}}$. - %%\lean{MDPs.Values}\leanok -\end{definition} - -\begin{definition}\label{def:DPMopt} -A \emph{Markov Bellman operator} $L\opt\colon \mathcal{V} \to \mathcal{V}$ is defined as -\[ -(L\opt v)(h) := -\max_{a\in \mathcal{A}} \E^{h, a, 1} \left[ \rew + v(\tilde{s}_{|h|}) \right], \quad \forall \tilde{u} \in \mathcal{U}, -\] -%%\lean{MDPs.DPMopt}\leanok -\end{definition} - -\begin{definition} \label{def:v-dp-opt} -The \emph{optimal value function} $v\opt_t\in \mathcal{V}, t\in \Nats$ is defined as -\[ -v\opt_t := -\begin{cases} - 0 &\text{if } t = 0 \\ - (L\opt v\opt _{t-1}) &\text{otherwise}. -\end{cases} -\] -%%\lean{MDPs.v_dp_opt}\leanok -\end{definition} - -\begin{theorem} -Suppose that $t\in \Nats $. Then: -\[ -v\opt_t(s_{|h|}(h)) = u\opt_t(h), \quad \forall h\in \mathcal{H}. -\] -%%\lean{MDPs.v_dp_opt_eq_u_opt} -\end{theorem} -\begin{proof} -By induction on $t$. The base case follows immediately from the definition. The inductive step for $t+1$ follows as: -\begin{align*} - u\opt_{t+1}(h) - &= \max_{a\in \mathcal{A}} \E^{h, a, 1} \left[ \rew_{|h|} + \tilde{u}\opt_t \right] - && \why{\cref{def:u-dp-opt}} \\ - &= \max_{a\in \mathcal{A}} \E^{h, a, 1} \left[ \rew_{|h|} + v\opt_t(\tilde{s}_l) \right] - && \why{inductive asm.} \\ - &= \max_{a\in \mathcal{A}} \E^{s_0, a, 1} \left[ \rew_{|h|} + v\opt_t(\tilde{s}_l) \right] - && \why{\cref{thm:exph-horizon-trim}} \\ - &= \max_{a\in \mathcal{A}} \E^{s_0, a, 1} \left[ \rew + v\opt_t(\tilde{s}_l) \right] - && \why{\cref{thm:exph-congr}} \\ - &= v\opt_{t+1}(s_{|h|}(h)) && \why{\cref{def:v-dp-opt}}. -\end{align*} -\end{proof} - - -\begin{definition} \label{def:pi-opt} - The \emph{optimal finite-horizon policy} $\pi\opt_t, t\in \Nats$ is defined as - \[ - \pi\opt_t(k,s) := - \begin{cases} - \arg\max_{a\in \mathcal{A}} \E^{s, a, 1} \left[ \rew + v_{t-k}\opt (\tilde{s}_{|h|}) \right] - &\text{if } k \le t, \\ - a_0 &\text{otherwise}, - \end{cases} - \] - where $a_0$ is an arbitrary action. - %%\lean{MDPs.πopt}\leanok -\end{definition} - -\begin{theorem} -Assume a horizon $T\in \Nats$. Then: -\[ - v\opt_{T-|h|}(s_{|h|}(h)) = u^{\pi\opt_{T-h}}_{T-|h|}(h), \quad - \forall h\in \left\{ h\in \mathcal{H} \mid |h| \le T \right\}. -\] -%%\lean{MDPs.v_dp_opt_eq_u_opt} -\end{theorem} -\begin{proof} -Fix some $T \in \Nats$. By induction on $k$ from $k = T$ to $k=0$. The base case is immediate from the definition. We prove the inductive case for $k-1$ from $k$ as -\begin{align*} -u^{\pi\opt_T}_{T-k + 1}(h) - &= \E^{h, \pi\opt_T, 1} \left[ \rew_k + \tilde{u}^{\pi\opt_T}_{T-k} \right] - && \why{\cref{def:DPhpi}} \\ - &= \E^{h, a\opt, 1} \left[ \rew_k + \tilde{u}^{\pi\opt_T}_{T-k} \right] - && \why{???} \\ - &= \E^{h, a\opt, 1} \left[ \rew_{k} + v_{T-k}\opt(\tilde{s}_1) \right] - && \why{ind asm} \\ - &= \E^{s_0, a\opt, 1} \left[ \rew + v_{T-k}\opt(\tilde{s}_1) \right] - && \why{\cref{thm:exph-horizon-trim}} \\ - &= \max_{a\in \mathcal{A}} \E^{s_0, a, 1} \left[ \rew + v_{T-k}\opt(\tilde{s}_1) \right] - && \why{???} \\ - &= v_{T-k+1}\opt(s_0). && \why{\cref{def:v-dp-opt}} -\end{align*} -Here, $k := |h|$, $a\opt := \pi\opt_T(k, s_0)$, and $s_0 := s_{|h|}(h)$ -\end{proof} - -\subsection{Evaluation} -We discuss results pertinent to the evaluation of Markov policies. - -Markov value functions depend on the length of the history. -\begin{definition} \label{def:ValuesM} - The set of \emph{independent value functions} is defined as $\mathcal{V}_{\mathrm{M}} := \Real^{\Nats \times \mathcal{S}}$. - %%\lean{MDPs.ValuesM}\leanok -\end{definition} - - -\begin{definition}\label{def:DPMopt} -A \emph{Markov policy Bellman operator} $L_k^{\pi}\colon \mathcal{V} \to \mathcal{V}$ for $\pi\in \Pi$ is defined as -\[ -(L^{\pi} v)(k, s) := -\max_{a\in \mathcal{A}} \E^{s, a, 1} \left[ \rew + v(k + 1, \tilde{s}_{|h|}) \right], \quad \forall v\in \mathcal{V}_{\mathrm{M}}, k\in \Nats, s\in \mathcal{S}. -\] -%%\lean{MDPs.DPMopt}\leanok -\end{definition} - -\begin{definition} \label{def:v-dp-pi} - The \emph{Markov policy value function} $v^{\pi}_t\in \mathcal{V}_{\mathrm{M}}, t \in \Nats$ for $\pi\in \PiMD$ is defined as - \[ - v^{\pi}_t := - \begin{cases} - 0 &\text{if } t = 0, \\ - (L^{\pi} v^{\pi}_{t-1}) &\text{otherwise}. - \end{cases} - \] - %%\lean{MDPs.v_dp_π}\leanok -\end{definition} - -\section{Probability Maticies} - - -\begin{definition} -A probability matrix $\bold{P}$ is defined as an $n \times n$ matrix with the properties: -\begin{align*} - &\bold{P}_{ij} \geq 0 \quad \forall i,j \in \{1,2,\dots,n\} \\ - &\bold{P}\bold{1} = \bold{1} -\end{align*} -\end{definition} -\begin{theorem} -Let $\bold{d} \in \Delta^n$. Then -\[ -\bold{d}^{T}\bold{P} \in \Delta^n -\] -\end{theorem} -\begin{proof} -Let $i \in \{1,2,\dots,n\}$ and $\bold{p} = \bold{d}^{T}\bold{P} $. Then $p_i = \bold{d}^{T}\bold{P}_i$ where $\bold{P}_i$ is the $i^{th}$ column of $\bold{P}$. Since all the elements of both $\bold{P}_i$ and $\bold{d}$ are non-negative, their inner product must also be non-negative. Since -\begin{align*} - \bold{d}^{T}\bold{P}\bold{1} &= \bold{d}^{T}\bold{1} \\ - &= 1~, -\end{align*} -$\bold{p}$ satisfies that its elements are non-negative and sum to 1 making it a member of $\Delta^n$ -\end{proof} - -%%% Local Variables: -%%% coding: utf-8 -%%% mode: LaTeX -%%% TeX-master: "print" -%%% TeX-engine: luatex -%%% End: diff --git a/blueprint/src/extra_styles.css b/blueprint/src/extra_styles.css deleted file mode 100644 index b96c776..0000000 --- a/blueprint/src/extra_styles.css +++ /dev/null @@ -1,25 +0,0 @@ -/* This file contains CSS tweaks for this blueprint. - * As an example, we included CSS rules that put - * a vertical line on the left of theorem statements - * and proofs. - * */ - -div.theorem_thmcontent { - border-left: .15rem solid black; -} - -div.proposition_thmcontent { - border-left: .15rem solid black; -} - -div.lemma_thmcontent { - border-left: .1rem solid black; -} - -div.corollary_thmcontent { - border-left: .1rem solid black; -} - -div.proof_content { - border-left: .08rem solid grey; -} diff --git a/blueprint/src/latexmkrc b/blueprint/src/latexmkrc deleted file mode 100644 index 38d5963..0000000 --- a/blueprint/src/latexmkrc +++ /dev/null @@ -1,5 +0,0 @@ -# This file configures the latexmk command you can use to compile -# the pdf version of the blueprint -$pdf_mode = 1; -$pdflatex = 'xelatex -synctex=1'; -@default_files = ('print.tex'); \ No newline at end of file diff --git a/blueprint/src/macros/common.tex b/blueprint/src/macros/common.tex deleted file mode 100644 index 4e23bd7..0000000 --- a/blueprint/src/macros/common.tex +++ /dev/null @@ -1,41 +0,0 @@ -% In this file you should put all LaTeX macros and settings to be used both by -% the pdf version and the web version. -% This should be most of your macros. - -\usepackage{color} -\usepackage{natbib} -\bibliographystyle{abbrvnat} - -\theoremstyle{plain} -\newtheorem{theorem}{Theorem}[subsection] -\newtheorem{proposition}[theorem]{Proposition} -\newtheorem{lemma}[theorem]{Lemma} -\newtheorem{corollary}[theorem]{Corollary} - -\theoremstyle{definition} -\newtheorem{definition}[theorem]{Definition} - -\theoremstyle{remark} -\newtheorem{remark}[theorem]{Remark} -\newtheorem{example}[theorem]{Example} - -\newcommand{\Nats}{\mathbb{N}} -\newcommand{\Real}{\mathbb{R}} -\newcommand{\E}{\mathbb{E}} -\newcommand{\PP}{\mathbb{P}} - -\newcommand{\opt}{^{\star}} - -\newcommand{\PiHR}{\Pi_{\mathrm{HR}}} -\newcommand{\PiMD}{\Pi_{\mathrm{MD}}} - - -\newcommand{\I}{\mathbb{I}} - -\newcommand{\rew}{\tilde{r}^{\mathrm{h}}} -\newcommand{\why}[1]{\text{[#1]}} -\newcommand{\todo}[1]{\textcolor{red}{[TODO: #1]}} - -\setlength{\parindent}{0pt} -\setlength{\parskip}{3mm} - diff --git a/blueprint/src/macros/print.tex b/blueprint/src/macros/print.tex deleted file mode 100644 index 1b98bd8..0000000 --- a/blueprint/src/macros/print.tex +++ /dev/null @@ -1,53 +0,0 @@ -% In this file you should put macros to be used only by -% the printed version. Of course they should have a corresponding -% version in macros/web.tex. -% Typically the printed version could have more fancy decorations. -% This should be a very short file. -% -% This file starts with dummy macros that ensure the pdf -% compiler will ignore macros provided by plasTeX that make -% sense only for the web version, such as dependency graph -% macros. - -\usepackage[framemethod=tikz]{mdframed} - -\usepackage{thmtools} - -\newcommand{\lean}[1]{\textcolor{blue}{(\emph{LEAN}: \texttt{#1})}} -\newcommand{\discussion}[1]{} -\newcommand{\leanok}{} -\newcommand{\mathlibok}{} -\newcommand{\notready}{} - -\DeclareMathOperator{\true}{true} -\DeclareMathOperator{\false}{false} -% \DeclareMathOperator{\id}{id} - -% Make sure that arguments of \uses and \proves are real labels, by using invisible refs: -% latex prints a warning if the label is not defined, but nothing is shown in the pdf file. -% It uses LaTeX3 programming, this is why we use the expl3 package. -\ExplSyntaxOn -\NewDocumentCommand{\uses}{m} - {\clist_map_inline:nn{#1}{\vphantom{\ref{##1}}}% - \ignorespaces} -\NewDocumentCommand{\proves}{m} - {\clist_map_inline:nn{#1}{\vphantom{\ref{##1}}}% - \ignorespaces} -\ExplSyntaxOff - -% Theorems and such -\mdfdefinestyle{theoremstyle}{% - roundcorner=5pt, % round corners, makes it friendlier - leftmargin=1pt, rightmargin=1pt, % this helps with box warnings - hidealllines=false, % even friendlier - align = center, % -} - - -\surroundwithmdframed[style=theoremstyle,backgroundcolor=gray!15]{definition} -\surroundwithmdframed[style=theoremstyle,backgroundcolor=gray!15]{theorem} -\surroundwithmdframed[style=theoremstyle,backgroundcolor=gray!15]{lemma} -\surroundwithmdframed[style=theoremstyle,backgroundcolor=gray!15]{proposition} - - - diff --git a/blueprint/src/macros/web.tex b/blueprint/src/macros/web.tex deleted file mode 100644 index 403ce9e..0000000 --- a/blueprint/src/macros/web.tex +++ /dev/null @@ -1,9 +0,0 @@ -% In this file you should put macros to be used only by -% the web version. Of course they should have a corresponding -% version in macros/print.tex. -% Typically the printed version could have more fancy decorations. -% This will probably be a very short file. - -\newcommand{\true}{\operatorname{true}} -\newcommand{\false}{\operatorname{false}} -% \newcommand{\id}{\operatornam{id}} diff --git a/blueprint/src/plastex.cfg b/blueprint/src/plastex.cfg deleted file mode 100644 index de3dbae..0000000 --- a/blueprint/src/plastex.cfg +++ /dev/null @@ -1,17 +0,0 @@ -[general] -renderer=HTML5 -copy-theme-extras=yes -plugins=plastexdepgraph plastexshowmore leanblueprint - -[document] -toc-depth=3 -toc-non-files=True - -[files] -directory=../web/ -split-level= 0 - -[html5] -localtoc-level=0 -extra-css=extra_styles.css -mathjax-dollars=False \ No newline at end of file diff --git a/blueprint/src/print.tex b/blueprint/src/print.tex deleted file mode 100644 index 7c6ffa9..0000000 --- a/blueprint/src/print.tex +++ /dev/null @@ -1,40 +0,0 @@ -% This file makes a printable version of the blueprint -% It should include all the \usepackage needed for the pdf version. -% The template version assume you want to use a modern TeX compiler -% such as xeLaTeX or luaLaTeX including support for unicode -% and Latin Modern Math font with standard bugfixes applied. -% It also uses expl3 in order to support macros related to the dependency graph. -% It also includes standard AMS packages (and their improved version -% mathtools) as well as support for links with a sober decoration -% (no ugly rectangles around links). -% It is otherwise a very minimal preamble (you should probably at least -% add cleveref and tikz-cd). - -\documentclass[letterpaper]{article} - -\usepackage{geometry} -\usepackage{expl3} - -\usepackage{amssymb, amsthm, mathtools} -\usepackage[unicode,colorlinks=true,linkcolor=blue,urlcolor=magenta, citecolor=blue]{hyperref} -\usepackage{bm} -\usepackage{fullpage} -\usepackage[capitalize]{cleveref} - -\usepackage[warnings-off={mathtools-colon,mathtools-overbracket}]{unicode-math} - -\input{macros/common} -\input{macros/print} - -\title{Formalization of Basic Dynamic Programming} -\author{UNH Team} - -\begin{document} -\maketitle -\input{content} -\end{document} - -%%% Local Variables: -%%% mode: LaTeX -%%% TeX-master: t -%%% End: diff --git a/blueprint/src/web.tex b/blueprint/src/web.tex deleted file mode 100644 index a27f9a8..0000000 --- a/blueprint/src/web.tex +++ /dev/null @@ -1,28 +0,0 @@ -% This file makes a web version of the blueprint -% It should include all the \usepackage needed for this version. -% The template includes standard AMS packages. -% It is otherwise a very minimal preamble (you should probably at least -% add cleveref and tikz-cd). - -\documentclass{report} - -\usepackage{amssymb, amsthm, amsmath} -\usepackage{hyperref} -\usepackage{cleveref} -\usepackage[showmore, dep_graph]{blueprint} - - -\input{macros/common} -\input{macros/web} - -\home{https://formalproofs.github.io/probability} -\github{https://github.com/formalproofs/probability} -\dochome{https://formalproofs.github.io/probability/docs} - -\title{Probability} -\author{UNH Elite Team} - -\begin{document} -\maketitle -\input{content} -\end{document} diff --git a/home_page/404.html b/home_page/404.html deleted file mode 100644 index fcebf3f..0000000 --- a/home_page/404.html +++ /dev/null @@ -1,25 +0,0 @@ ---- -permalink: /404.html -layout: default ---- - - - -
-

404

- -

Page not found :(

-

The requested page could not be found.

-
\ No newline at end of file diff --git a/home_page/Gemfile b/home_page/Gemfile deleted file mode 100644 index 7876373..0000000 --- a/home_page/Gemfile +++ /dev/null @@ -1,25 +0,0 @@ -source "https://rubygems.org" - -# To upgrade, run `bundle update github-pages`. -gem "github-pages", group: :jekyll_plugins -# If you have any plugins, put them here! -group :jekyll_plugins do - #gem "jekyll-feed", "~> 0.12" -end - -# Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem -# and associated library. -platforms :mingw, :x64_mingw, :mswin, :jruby do - gem "tzinfo", "~> 1.2" - gem "tzinfo-data" -end - -# Performance-booster for watching directories on Windows. -gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin] - -# Lock `http_parser.rb` gem to `v0.6.x` on JRuby builds since newer versions of the gem -# do not have a Java counterpart. -gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby] - -# Used for locally serving the website. -gem "webrick", "~> 1.7" \ No newline at end of file diff --git a/home_page/Gemfile.lock b/home_page/Gemfile.lock deleted file mode 100644 index 5436ad6..0000000 --- a/home_page/Gemfile.lock +++ /dev/null @@ -1,287 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - activesupport (6.0.5) - concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - zeitwerk (~> 2.2, >= 2.2.2) - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) - coffee-script (2.4.1) - coffee-script-source - execjs - coffee-script-source (1.11.1) - colorator (1.1.0) - commonmarker (0.23.4) - concurrent-ruby (1.1.10) - dnsruby (1.61.9) - simpleidn (~> 0.1) - em-websocket (0.5.3) - eventmachine (>= 0.12.9) - http_parser.rb (~> 0) - ethon (0.15.0) - ffi (>= 1.15.0) - eventmachine (1.2.7) - execjs (2.8.1) - faraday (1.10.0) - faraday-em_http (~> 1.0) - faraday-em_synchrony (~> 1.0) - faraday-excon (~> 1.1) - faraday-httpclient (~> 1.0) - faraday-multipart (~> 1.0) - faraday-net_http (~> 1.0) - faraday-net_http_persistent (~> 1.0) - faraday-patron (~> 1.0) - faraday-rack (~> 1.0) - faraday-retry (~> 1.0) - ruby2_keywords (>= 0.0.4) - faraday-em_http (1.0.0) - faraday-em_synchrony (1.0.0) - faraday-excon (1.1.0) - faraday-httpclient (1.0.1) - faraday-multipart (1.0.3) - multipart-post (>= 1.2, < 3) - faraday-net_http (1.0.1) - faraday-net_http_persistent (1.2.0) - faraday-patron (1.0.0) - faraday-rack (1.0.0) - faraday-retry (1.0.3) - ffi (1.15.5) - forwardable-extended (2.6.0) - gemoji (3.0.1) - github-pages (226) - github-pages-health-check (= 1.17.9) - jekyll (= 3.9.2) - jekyll-avatar (= 0.7.0) - jekyll-coffeescript (= 1.1.1) - jekyll-commonmark-ghpages (= 0.2.0) - jekyll-default-layout (= 0.1.4) - jekyll-feed (= 0.15.1) - jekyll-gist (= 1.5.0) - jekyll-github-metadata (= 2.13.0) - jekyll-include-cache (= 0.2.1) - jekyll-mentions (= 1.6.0) - jekyll-optional-front-matter (= 0.3.2) - jekyll-paginate (= 1.1.0) - jekyll-readme-index (= 0.3.0) - jekyll-redirect-from (= 0.16.0) - jekyll-relative-links (= 0.6.1) - jekyll-remote-theme (= 0.4.3) - jekyll-sass-converter (= 1.5.2) - jekyll-seo-tag (= 2.8.0) - jekyll-sitemap (= 1.4.0) - jekyll-swiss (= 1.0.0) - jekyll-theme-architect (= 0.2.0) - jekyll-theme-cayman (= 0.2.0) - jekyll-theme-dinky (= 0.2.0) - jekyll-theme-hacker (= 0.2.0) - jekyll-theme-leap-day (= 0.2.0) - jekyll-theme-merlot (= 0.2.0) - jekyll-theme-midnight (= 0.2.0) - jekyll-theme-minimal (= 0.2.0) - jekyll-theme-modernist (= 0.2.0) - jekyll-theme-primer (= 0.6.0) - jekyll-theme-slate (= 0.2.0) - jekyll-theme-tactile (= 0.2.0) - jekyll-theme-time-machine (= 0.2.0) - jekyll-titles-from-headings (= 0.5.3) - jemoji (= 0.12.0) - kramdown (= 2.3.2) - kramdown-parser-gfm (= 1.1.0) - liquid (= 4.0.3) - mercenary (~> 0.3) - minima (= 2.5.1) - nokogiri (>= 1.13.4, < 2.0) - rouge (= 3.26.0) - terminal-table (~> 1.4) - github-pages-health-check (1.17.9) - addressable (~> 2.3) - dnsruby (~> 1.60) - octokit (~> 4.0) - public_suffix (>= 3.0, < 5.0) - typhoeus (~> 1.3) - html-pipeline (2.14.1) - activesupport (>= 2) - nokogiri (>= 1.4) - http_parser.rb (0.8.0) - i18n (0.9.5) - concurrent-ruby (~> 1.0) - jekyll (3.9.2) - addressable (~> 2.4) - colorator (~> 1.0) - em-websocket (~> 0.5) - i18n (~> 0.7) - jekyll-sass-converter (~> 1.0) - jekyll-watch (~> 2.0) - kramdown (>= 1.17, < 3) - liquid (~> 4.0) - mercenary (~> 0.3.3) - pathutil (~> 0.9) - rouge (>= 1.7, < 4) - safe_yaml (~> 1.0) - jekyll-avatar (0.7.0) - jekyll (>= 3.0, < 5.0) - jekyll-coffeescript (1.1.1) - coffee-script (~> 2.2) - coffee-script-source (~> 1.11.1) - jekyll-commonmark (1.4.0) - commonmarker (~> 0.22) - jekyll-commonmark-ghpages (0.2.0) - commonmarker (~> 0.23.4) - jekyll (~> 3.9.0) - jekyll-commonmark (~> 1.4.0) - rouge (>= 2.0, < 4.0) - jekyll-default-layout (0.1.4) - jekyll (~> 3.0) - jekyll-feed (0.15.1) - jekyll (>= 3.7, < 5.0) - jekyll-gist (1.5.0) - octokit (~> 4.2) - jekyll-github-metadata (2.13.0) - jekyll (>= 3.4, < 5.0) - octokit (~> 4.0, != 4.4.0) - jekyll-include-cache (0.2.1) - jekyll (>= 3.7, < 5.0) - jekyll-mentions (1.6.0) - html-pipeline (~> 2.3) - jekyll (>= 3.7, < 5.0) - jekyll-optional-front-matter (0.3.2) - jekyll (>= 3.0, < 5.0) - jekyll-paginate (1.1.0) - jekyll-readme-index (0.3.0) - jekyll (>= 3.0, < 5.0) - jekyll-redirect-from (0.16.0) - jekyll (>= 3.3, < 5.0) - jekyll-relative-links (0.6.1) - jekyll (>= 3.3, < 5.0) - jekyll-remote-theme (0.4.3) - addressable (~> 2.0) - jekyll (>= 3.5, < 5.0) - jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0) - rubyzip (>= 1.3.0, < 3.0) - jekyll-sass-converter (1.5.2) - sass (~> 3.4) - jekyll-seo-tag (2.8.0) - jekyll (>= 3.8, < 5.0) - jekyll-sitemap (1.4.0) - jekyll (>= 3.7, < 5.0) - jekyll-swiss (1.0.0) - jekyll-theme-architect (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-cayman (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-dinky (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-hacker (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-leap-day (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-merlot (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-midnight (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-minimal (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-modernist (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-primer (0.6.0) - jekyll (> 3.5, < 5.0) - jekyll-github-metadata (~> 2.9) - jekyll-seo-tag (~> 2.0) - jekyll-theme-slate (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-tactile (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-theme-time-machine (0.2.0) - jekyll (> 3.5, < 5.0) - jekyll-seo-tag (~> 2.0) - jekyll-titles-from-headings (0.5.3) - jekyll (>= 3.3, < 5.0) - jekyll-watch (2.2.1) - listen (~> 3.0) - jemoji (0.12.0) - gemoji (~> 3.0) - html-pipeline (~> 2.2) - jekyll (>= 3.0, < 5.0) - kramdown (2.3.2) - rexml - kramdown-parser-gfm (1.1.0) - kramdown (~> 2.0) - liquid (4.0.3) - listen (3.7.1) - rb-fsevent (~> 0.10, >= 0.10.3) - rb-inotify (~> 0.9, >= 0.9.10) - mercenary (0.3.6) - minima (2.5.1) - jekyll (>= 3.5, < 5.0) - jekyll-feed (~> 0.9) - jekyll-seo-tag (~> 2.1) - minitest (5.15.0) - multipart-post (2.1.1) - nokogiri (1.13.6-x86_64-linux) - racc (~> 1.4) - octokit (4.22.0) - faraday (>= 0.9) - sawyer (~> 0.8.0, >= 0.5.3) - pathutil (0.16.2) - forwardable-extended (~> 2.6) - public_suffix (4.0.7) - racc (1.6.0) - rb-fsevent (0.11.1) - rb-inotify (0.10.1) - ffi (~> 1.0) - rexml (3.2.5) - rouge (3.26.0) - ruby2_keywords (0.0.5) - rubyzip (2.3.2) - safe_yaml (1.0.5) - sass (3.7.4) - sass-listen (~> 4.0.0) - sass-listen (4.0.0) - rb-fsevent (~> 0.9, >= 0.9.4) - rb-inotify (~> 0.9, >= 0.9.7) - sawyer (0.8.2) - addressable (>= 2.3.5) - faraday (> 0.8, < 2.0) - simpleidn (0.2.1) - unf (~> 0.1.4) - terminal-table (1.8.0) - unicode-display_width (~> 1.1, >= 1.1.1) - thread_safe (0.3.6) - typhoeus (1.4.0) - ethon (>= 0.9.0) - tzinfo (1.2.9) - thread_safe (~> 0.1) - unf (0.1.4) - unf_ext - unf_ext (0.0.8.1) - unicode-display_width (1.8.0) - webrick (1.7.0) - zeitwerk (2.5.4) - -PLATFORMS - x86_64-linux - -DEPENDENCIES - github-pages - http_parser.rb (~> 0.6.0) - tzinfo (~> 1.2) - tzinfo-data - wdm (~> 0.1.1) - webrick (~> 1.7) - -BUNDLED WITH - 2.3.14 \ No newline at end of file diff --git a/home_page/_config.yml b/home_page/_config.yml deleted file mode 100644 index f9e07b7..0000000 --- a/home_page/_config.yml +++ /dev/null @@ -1,34 +0,0 @@ -# Welcome to Jekyll! -# -# This config file is meant for settings that affect your whole blog, values -# which you are expected to set up once and rarely edit after that. If you find -# yourself editing this file very often, consider using Jekyll's data files -# feature for the data you need to update frequently. -# -# For technical reasons, this file is *NOT* reloaded automatically when you use -# 'bundle exec jekyll serve'. If you change this file, please restart the server process. -# -# If you need help with YAML syntax, here are some quick references for you: -# https://learn-the-web.algonquindesign.ca/topics/markdown-yaml-cheat-sheet/#yaml -# https://learnxinyminutes.com/docs/yaml/ -# -# Site settings -# These are used to personalize your new site. If you look in the HTML files, -# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on. -# You can create any custom variable you would like, and they will be accessible -# in the templates via {{ site.myvariable }}. - -title: MDP Formalization -#email: your-email@example.com -description: by UNH -baseurl: "" # the subpath of your site, e.g. /blog -url: "https://formalproofs.github.io/probability" # the base hostname & protocol for your site, e.g. http://example.com -twitter_username: -github_username: formalproofs -repository: formalproofs/probability - -# Build settings -remote_theme: pages-themes/cayman@v0.2.0 -plugins: - - jekyll-remote-theme - - jekyll-github-metadata \ No newline at end of file diff --git a/home_page/_include/mathjax.html b/home_page/_include/mathjax.html deleted file mode 100644 index ab9c007..0000000 --- a/home_page/_include/mathjax.html +++ /dev/null @@ -1,17 +0,0 @@ - -{% if page.usemathjax %} - - - - -{% endif %} \ No newline at end of file diff --git a/home_page/_layouts/default.html b/home_page/_layouts/default.html deleted file mode 100644 index 95d8377..0000000 --- a/home_page/_layouts/default.html +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - - {% seo %} - - - - - - {% if jekyll.environment == "production" %} - - {% else %} - - {% endif %} - {% include head-custom.html %} - - - - - - - -
- {{ content }} - -
- {% if site.github.is_project_page %} - {{ site.github.repository_name }} - is maintained by UNH. Visit the GitHub repository for more information. - {% endif %} -
-
- - - \ No newline at end of file diff --git a/home_page/assets/css/style.scss b/home_page/assets/css/style.scss deleted file mode 100644 index 312da21..0000000 --- a/home_page/assets/css/style.scss +++ /dev/null @@ -1,9 +0,0 @@ ---- ---- - -/* In this file you can add css rules overriding rules defined by your Jekyll theme. */ - -/* Import the CSS from the specified theme */ -@import "{{ site.theme }}"; - -/* Put your rules below this line */ \ No newline at end of file diff --git a/home_page/index.md b/home_page/index.md deleted file mode 100644 index 2ab3ac6..0000000 --- a/home_page/index.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -# Feel free to add content and custom Front Matter to this file. -# To modify the layout, see https://jekyllrb.com/docs/themes/#overriding-theme-defaults - -# layout: home -usemathjax: true ---- - -Useful links: - -* [Zulip chat for Lean](https://leanprover.zulipchat.com/) for coordination -* [Blueprint]({{ site.url }}/blueprint/) -* [Blueprint as pdf]({{ site.url }}/blueprint.pdf) -* [Dependency graph]({{ site.url }}/blueprint/dep_graph_document.html) -* [Doc pages for this repository]({{ site.url }}/docs/) \ No newline at end of file diff --git a/latex/main.tex b/latex/main.tex new file mode 100644 index 0000000..f4e78b2 --- /dev/null +++ b/latex/main.tex @@ -0,0 +1,1605 @@ +\documentclass[letterpaper]{article} + +\usepackage{geometry} +\usepackage{expl3} % TODO: ?? + +\usepackage{amssymb, amsthm, mathtools} +\usepackage[unicode,colorlinks=true,linkcolor=blue,urlcolor=magenta, citecolor=blue]{hyperref} +\usepackage{bm} +\usepackage{fullpage} + +\usepackage{thmtools} +\usepackage[capitalize]{cleveref} + +% the following package only works with LuaTeX or XeLaTeX +%\usepackage[warnings-off={mathtools-colon,mathtools-overbracket}]{unicode-math} +\usepackage{graphicx} % Required for inserting images + +\usepackage{color} +\usepackage{natbib} +\bibliographystyle{abbrvnat} + +\theoremstyle{plain} +\newtheorem{theorem}{Theorem}[subsection] +\newtheorem{proposition}[theorem]{Proposition} +\newtheorem{lemma}[theorem]{Lemma} +\newtheorem{corollary}[theorem]{Corollary} + +\theoremstyle{definition} +\newtheorem{definition}[theorem]{Definition} + +\theoremstyle{remark} +\newtheorem{remark}[theorem]{Remark} +\newtheorem{example}[theorem]{Example} + +\newcommand{\Nats}{\mathbb{N}} +\newcommand{\Real}{\mathbb{R}} +\newcommand{\E}{\mathbb{E}} +\newcommand{\PP}{\mathbb{P}} + +\newcommand{\opt}{^{\star}} +\newcommand{\tr}{^{\mathsf{T}}} + +\newcommand{\PiHR}{\Pi_{\mathrm{HR}}} +\newcommand{\PiMD}{\Pi_{\mathrm{MD}}} + +\DeclareMathOperator{\bool}{Bool} +\DeclareMathOperator{\true}{true} +\DeclareMathOperator{\false}{false} +\DeclareMathOperator{\finvar}{VaR} +\DeclareMathOperator{\isvar}{IsVaR} + + + +\newcommand{\I}{\mathbb{I}} + +\newcommand{\rew}{\tilde{r}^{\mathrm{h}}} +\newcommand{\why}[1]{\text{[#1]}} +\newcommand{\todo}[1]{\textcolor{red}{[TODO: #1]}} +\definecolor{ccolor}{rgb}{0.80,0.33,0.0} +\newcommand{\cc}[1]{\textcolor{ccolor}{[#1]}} + + +\newcommand{\id}{\operatorname{id}} + +\setlength{\parindent}{0pt} +\setlength{\parskip}{3mm} + +\usepackage[framemethod=tikz]{mdframed} + + +\newcommand{\lean}[1]{\textcolor{blue}{(\emph{LEAN}: \texttt{#1})}} +\newcommand{\discussion}[1]{} +\newcommand{\leanok}{} +\newcommand{\mathlibok}{} +\newcommand{\notready}{} + +% \DeclareMathOperator{\true}{true} +% \DeclareMathOperator{\false}{false} +% \DeclareMathOperator{\id}{id} + +% Make sure that arguments of \uses and \proves are real labels, by using invisible refs: +% latex prints a warning if the label is not defined, but nothing is shown in the pdf file. +% It uses LaTeX3 programming, this is why we use the expl3 package. +\ExplSyntaxOn +\NewDocumentCommand{\uses}{m} + {\clist_map_inline:nn{#1}{\vphantom{\ref{##1}}}% + \ignorespaces} +\NewDocumentCommand{\proves}{m} + {\clist_map_inline:nn{#1}{\vphantom{\ref{##1}}}% + \ignorespaces} +\ExplSyntaxOff + +% Theorems and such +\mdfdefinestyle{theoremstyle}{% + roundcorner=5pt, % round corners, makes it friendlier + leftmargin=1pt, rightmargin=1pt, % this helps with box warnings + hidealllines=false, % even friendlier + align = center, % +} + +\surroundwithmdframed[style=theoremstyle,backgroundcolor=gray!15]{definition} +\surroundwithmdframed[style=theoremstyle,backgroundcolor=gray!15]{theorem} +\surroundwithmdframed[style=theoremstyle,backgroundcolor=gray!15]{lemma} +\surroundwithmdframed[style=theoremstyle,backgroundcolor=gray!15]{proposition} + + +\title{MDPLib} +\author{MDP Collective} +% \date{March 2026} + +\begin{document} + +\maketitle + +\begin{remark} +This document omits basic definitions, such as the comparison or arithmetic operations on random variables. Arithmetic operations on random variables are performed element-wise for each element of the sample set. Please see the Lean file for complete details. +\end{remark} + + +\section{Probability} + +Our treatment of the probability mostly follows +\begin{quote} + Deisenroth, M.P., Faisal, A.A. and Ong, C.S. (2021) Mathematics for machine learning. +\end{quote} + +Although this document states that the values are in $\Real$, the lean file is defined entirely in terms of rational numbers $\mathbb{Q}$ for computability. + +\subsection{Prelude} + +This section sets up some basic tools for working with probabilities. See \texttt{Probability/Prelude.lean}. + +\begin{definition} \label{def:probablity-value} +A $p\in \Real$ is a \emph{probability value} when $p \ge 0 \wedge p \le 1$. +\lean{Prob} +\end{definition} + +\begin{definition} +A finite set of natural numbers is defined as $\Nats_n := \left\{ 0, \dots , n-1 \right\}$. +\lean{Fin} +\end{definition} + +\begin{definition} \label{def:list} +A finite function (or a list) $\mathbb{L}_n^{\tau} := \Nats_n \to \tau$ of type $\tau$ and length $n$ is a function defined for $\Nats_{n} := \left\{ 0, \dots , n-1 \right\}$ of type $\tau$. +\lean{$\text{Fin}\, n \to \tau$} +\end{definition} + +The finite functions in \cref{def:list} are also used to represent vectors and obey standard vector operations, such as addition or Hadamard product. + +The following definition generalizes the inverse of a function to non-injective function which do not have an inverse. +\begin{definition}[Preimage] +The \emph{preimage} $f^{-1}\colon \mathcal{Y} \to 2^{\mathcal{X}}$ of a function $f\colon \mathcal{X} \to \mathcal{Y}$ is defined as +\begin{equation} \label{eq:preimage-set} +f^{-1}(y) := \left\{ x \in \mathcal{X} \mid f(x) = y \right\} . +\end{equation} +\lean{FinRV.preimage} \leanok +\end{definition} + +\begin{definition} \label{def:indicator} +An \emph{indicator} function $\I \colon \bool \to \left\{ 0, 1 \right\} \subseteq \mathbb{Q}$ is defined for $b\in \bool$ as +\[ +\I(b) := +\begin{cases} +1 &\text{if } b = \operatorname{true}, \\ +0 &\text{if } b = \operatorname{false}. +\end{cases} +\] +\lean{indicator} \leanok +\end{definition} + +\subsection{Defs} + +This section contains definitions of probability spaces, probability operators, and the expectation operator. See \texttt{Probability/Defs.lean}. +\begin{definition} +A \emph{finite probability distribution} $\Delta(n)$ for $n\in \Nats$ is defined as +\[ +\Delta_n := \left\{ p \in \mathbb{L}_n \mid \sum_{i \in \Nats_n} p_i = 1, p_i \ge 0, \forall i \in \Nats_n \right\}. +\] +Any $P \in \Delta_n$ can be also seen as a linear set function such that +\[ + P(\mathcal{A}) := \sum_{i\in \mathcal{A}} P_i, \qquad \mathcal{A} \subseteq \Nats_n. +\] +% We call a probability distribution \emph{degenerate} if $p_i = 1$ for some $i$; otherwise the probability distribution is \emph{supported}. +\lean{Findist} +\end{definition} + +The formal model we use to represent random events and processes is the following. +\begin{definition}[Complete Probability Space] \label{def:probability-space} + A \emph{complete probability space} of dimension $n \in \Nats$ consists of: + \begin{enumerate} + \item An \emph{implicit} sample space $\Omega := \Nats_n$, which is the set of all possible outcomes. + \item An \emph{implicit} event space $\mathcal{F} := 2^{\Omega}$, which we do not define explicitly in Lean. + \item A probability distribution $P \in \Delta_n$. + \end{enumerate} +\lean{Findist} \leanok +\end{definition} +The notation $2^{\Omega}$ in \cref{def:probability-space} represents the power set of $\Omega$. We assume that the event space $\mathcal{F}$ is the power set of $\Omega$. The main reason to define $\mathcal{F}$ that is not a power set in a finite probability space is to represent that the available information is limited. Instead, when defining a random process in which information is revealed gradually, we construct new probability spaces. + +Because the complete probability space \cref{def:probability-space} is identified with a $P \in \Delta_n$, we use it interchangeably with a distribution $P$. + + +% \begin{definition}[Measure] +% The measure $\PP [\mathcal{S}]$ of the set $\mathcal{S} \subseteq \Nats$ given a probability space is defined as +% \[ +% \PP [\mathcal{S}] := \sum_{i \in \Omega} \mathbb{I}[ i\in \mathcal{S} ]. +% \] +% The definition requires that the set is decidable. +% %\lean{Probability.Finprob.measure} +% \end{definition} + + +\begin{example} \label{exm:probability-space} + Suppose I want to model the behavior of coin flips. I have a coin that that is twice as likely to come up heads than tails. I flip the coin twice. One possible probability space would have a dimension 4 and be defined as follows. + \begin{enumerate} + \item The sample space is $\Omega := \Nats_4 = \left\{ 0, 1, 2, 3 \right\}$, where the sample space elements correspond to $ \left( \mathrm{HH}, \mathrm{HT}, \mathrm{TH}, \mathrm{TT} \right)$ and H is heads and T is tails. + \item The probability distribution is $P_0 = (\frac{2}{3})^2$ corresponding to the probability of seeing two heads, $P_1 = \frac{2}{3} \cdot \frac{1}{3}$ corresponding to the probability of heads followed by tails, \ldots. + \end{enumerate} +\end{example} + +% For the sake of completeness, we include also the notion of measurability. However, since we are only concerned with complete probability spaces, all random variables we define will be measurable. +% \begin{definition} \label{def:preimage-measurable} +% A function $f\colon \mathcal{X} \to \mathcal{Y}$ with $\sigma$-algebras $\mathcal{D} \subseteq 2^{\mathcal{X}}$ and $\mathcal{E} \subseteq 2^{\mathcal{Y}}$ is \emph{measurable} when the \emph{pre-image} of each $E\in \mathcal{E}$ is in $\mathcal{D}$: +% \[ +% f^{-1}(E) \in \mathcal{D}. +% \] +% Here, the application to $f^{-1}$ to a set $E \subseteq \mathcal{Y}$ is defined as: +% \begin{equation} +% f^{-1}(E) := \bigcup_{e \in E} f^{-1}(e) . +% \end{equation} +% \end{definition} +% The measurability in the definition of random variables is required for us to reason about their probability distributions. + +\begin{definition} +A \emph{random variable} $\tilde{x} \in \mathbb{L}_n^{\tau}$ for $n\in \Nats$ is a function that returns a value of type $\tau$. +\lean{FinRV} \leanok +\end{definition} +Note that we do not include the standard measurability requirement in the definition of a random variable because it is vacuous in complete probability spaces defined above. + +When manipulating random variables we denote them with a tilde, such as $\tilde{x}, \tilde{s}$. This corresponds to the common practice to use uppercase letters to denote random variables. We drop the tilde when treating the random variable as an ordinary function. We drop the tilde when the random variable is evaluated for some input, such as $x_i$. + +\begin{definition} + The set of Boolean random variables of dimension $n\in \Nats$ is defined as $\mathbb{B}_n := \mathbb{L}_n^{\bool}$. Here, $\bool := \left\{ \true, \false \right\}$. Implicitly, $b\in \mathbb{B}_n$ represents a finite set +\[ + \left\{ i\in \Nats_n \mid b_i = \true \right\} = \tilde{b}^{-1}(\true). +\] + \lean{FinRV n Bool} +\end{definition} + +We use the operator (function) $\mathbb{P}$ denote the probability of the set of sample space that corresponds to a given condition. We use a non-standard definition for convenience (and show that this definition equals to the standard one later). +\begin{definition} \label{def:probability} +Given $n\in \Nats$, $P \in \Delta_n$, and $\tilde{b} \in \mathbb{B}_n$, the probability operator $\PP$ is defined as +\[ +\PP^P\left[ \tilde{b} \right] := P(\tilde{b}^{-1}(\true)) = P \left( \left\{ i \in \Nats_n \mid b_i = \true \right\} \right) = \sum_{i\in \Omega} P_i b_i, +\] +where $b_i$ coerced to $\mathbb{Q}$ is interpreted as $0$ if $b_i = \false$ and $1$ if $b_i = \true$. +\lean{probability} \leanok +\end{definition} + +Note that \cref{def:probability} is the standard definition using the notion of the pre-image which is defined implicitly by the boolean random variable. + +\emph{Probability distributions} are typically defined for random variables. The distribution of a random variable is characterized by the \emph{cumulative distribution function}~(CDF), defined as follows. +\begin{definition}[CDF] + The \emph{cumulative distribution function} $F_{\tilde{x}}\colon \mathbb{Q} \to [0,1]$ of a real-valued random variable $\tilde{x} \in \mathbb{L}_n^{\mathbb{Q}}$ for $P\in \Delta_n$ is defined as + \[ + F_{\tilde{x}}(x) := \PP\left[\tilde{x} \le x \right], \qquad \forall x\in \mathbb{Q}. + \] +\lean{cdf} \leanok +\end{definition} + +Some random variables also have a \emph{probability density function}~(PDF), and discrete random variables have a \emph{probability mass function}~(PMF). In this class, we will be primarily dealing with discrete random variables. +\begin{definition}[PMF] +The \emph{probability mass function} $p_{\tilde{x}}\colon \mathcal{E} \to [0,1]$ for a discrete random variable $\tilde{x}\colon \Omega \to \mathcal{E}$ with a finite $\mathcal{E}$ is defined as + \[ + p_{\tilde{x}}(e) := \PP\left[\tilde{x} = e\right], \qquad \forall e\in \mathcal{E} . + \] +In Lean, \texttt{PMF} is the proposition that a given function is the PMF of a random variable. +\lean{PMF} \leanok +\end{definition} + + +\begin{definition} \label{def:joint-probability} +The \emph{joint probability} of two random variables $\tilde{x}\colon \Omega \to \mathcal{X}$ and $\tilde{y} \colon \Omega \to \mathcal{Y}$ is defined as +\[ + \PP\left[\tilde{x} = x, \tilde{y} = y\right] := \PP\left[\tilde{x} = x \wedge \tilde{y} = y\right] , \forall x\in \mathcal{X}, y\in \mathcal{Y}. +\] +One can also define a joint PMF for two discrete random variables $p_{\tilde{x}, \tilde{y}}(x,y)$ for $x\in \mathcal{X}$ and $y\in \mathcal{Y}$ as +\[ + p_{\tilde{x}, \tilde{y}}(x,y) = \PP\left[\tilde{x} = x, \tilde{y} = y\right]. +\] +\end{definition} + +\begin{definition} \label{def:probability_cnd} +The \emph{conditional probability} of $\tilde{b}\colon \Omega \to \mathbb{B}$ on $\tilde{c}\colon \Omega \to \mathbb{B}$ is defined as +\[ +\PP\left[ \tilde{b} \mid \tilde{c} \right] := \frac{ \PP[\tilde{b} \wedge \tilde{c} ] }{ \PP \left[ \tilde{c} \right]} +\] +\lean{probability\_cnd} \leanok +\end{definition} + +\begin{definition}[Independent RV] + Events $A,B \in \mathcal{F}$ are independent when + \[ + \PP[ A \cap B ] = \PP[ A ] \cdot \PP[ B ]. + \] + Random variables $\tilde{x}\colon \Omega \to \mathcal{X}$ and $\tilde{y}\colon \Omega \to \mathcal{Y}$ are \emph{independent} when + \[ + F_{\tilde{x}, \tilde{y}}(x,y) = F_{\tilde{x}}(x) \cdot F_{\tilde{y}}(y), + \forall x\in \mathcal{X}, y\in \mathcal{Y}. + \] +\end{definition} +Discrete random variables $\tilde{x}$ and $\tilde{y}$ are independent if and only if +\[ + \PP\left[\tilde{x} = x, \tilde{y} = y\right] = \PP\left[\tilde{x} = x\right] \cdot \PP\left[\tilde{y} = y\right], \forall x\in \mathcal{X}, y\in \mathcal{Y}. +\] +\cc{Gap: the joint CDF $F_{\tilde{x},\tilde{y}}$ appearing in the independence definition is never defined, and a CDF is only meaningful for an ordered codomain, so the stated definition does not typecheck for general $\mathcal{X},\mathcal{Y}$. Moreover, this ``if and only if'' characterization for discrete random variables is asserted with no proof.} + + +The expectation of a random variable is its mean. As with probabilities, we use a non-standard definition for the sake of computational convenience and then show that it is equivalent to the common definition of probabilities. +\begin{definition}[Expected Value] \label{def:expectation} +For $P \in \Delta_n$, and $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$ the \emph{expected value} is defined as +\[ + \E^P\left[\tilde{x}\right] := \sum_{i \in \Nats_n } x_i \cdot P_i. +\] +\lean{expect} \leanok +\end{definition} + +\Cref{thm:expect-correct} shows that this definition of expected value is equal to the standard definition of expected values. +Our definition in \cref{def:expectation} is non-standard. The usual definition uses the Lebesgue integral as follows: +\[ + \E^P\left[\tilde{x}\right] := \int_{\Omega} \tilde{x} \, dP. +\] + + + +The expectation operator is linear. +\begin{theorem} \label{thm:exp-linear} + Assume $\tilde{x},\tilde{y} \in \mathbb{L}_n^{\mathbb{Q}}$ and $c\in \mathbb{Q} $ and $P\in \Delta_n$. Then: + \begin{align*} + \E\left[c\cdot \tilde{x}\right] &= c \cdot \E\left[\tilde{x}\right]\\ + \E\left[\tilde{x} + \tilde{y}\right] &= \E\left[\tilde{x}\right] + \E\left[\tilde{y}\right]\\ + \E\left[\tilde{x} \cdot \tilde{y}\right] &= \E\left[\tilde{x}\right] \cdot \E\left[\tilde{y}\right] \quad \text{for independent } \tilde{x}, \tilde{y}. +\end{align*} +\lean{exp\_homogenous, exp\_additive\_two, exp\_additive} +(The product rule for independent random variables is not formalized in Lean.) +\cc{The product rule is not proved here either; its proof needs the (unproved) discrete characterization of independence stated above.} +\end{theorem} + +\begin{theorem}[Expectation of a constant] \label{thm:exp-const} + For $c \in \mathbb{Q}$ and $P\in \Delta_n$: + \[ + \E\left[ c \right] = c, + \] + where the constant $c$ is interpreted as a constant random variable. + \lean{exp\_const, exp\_one} \leanok +\end{theorem} + +\begin{theorem}[Monotonicity of expectation] \label{thm:exp-monotone} + For $\tilde{x},\tilde{y} \in \mathbb{L}_n^{\mathbb{Q}}$ with $\tilde{x} \le \tilde{y}$ element-wise, and $P \in \Delta_n$: + \[ + \E\left[ \tilde{x} \right] \le \E\left[ \tilde{y} \right]. + \] + \lean{exp\_monotone} \leanok +\end{theorem} + + +\begin{definition} \label{def:expect-cnd} +The \emph{conditional expectation} of $\tilde{x} \in \mathbb{L}_n^{\mathbb{Q}}$ conditioned on $\tilde{b} \in \mathbb{B}_n$ is defined as +\[ +\E \left[ \tilde{x} \mid \tilde{b} \right] := +\frac{1}{\PP[\tilde{b}]} \E \left[ \tilde{x} \cdot \I \circ \tilde{b} \right], +\] +where we define that $x / 0 = 0$ for each $x\in \Real$. +\lean{expect\_cnd} \leanok +\end{definition} + +\begin{remark} +It is common to prohibit conditioning on a zero probability event both for expectation and probabilities. In this document, we follow the Lean convention, where the division by $0$ is $0$; see \texttt{div\_zero}. However, even some basic probability and expectation results may require that we assume that the conditioned event does not have probability zero for it to hold. +\end{remark} + +\begin{definition} \label{def:expect-cnd-rv} +The \emph{random conditional expectation} of a random variable $\tilde{x} \in \mathbb{L}_n^{\mathbb{Q}}$ conditioned on $\tilde{l} \in \mathbb{L}_n^{\Nats_m}$ for a finite set $\mathcal{Y}$ is the random variable $\E \left[ \tilde{x} \mid \tilde{l} \right] \in \mathbb{L}_n^{\mathbb{Q}}$ is defined as +\[ +\E \left[ \tilde{x} \mid \tilde{l} \right](\omega) +:= +\E \left[ \tilde{x} \mid \tilde{l} = l_{\omega} \right], \quad \forall \omega \in \Nats_n. +\] +\lean{expect\_cnd\_rv} \leanok +\end{definition} + + + + +\subsection{Basic} + +This section states basic properties of probabilities and expectations. See \texttt{Probability/Basic.lean}. + +\begin{theorem} \label{thm:prob-in-range} + For each $P\in \Delta_n$ and $\tilde{b}\in \mathbb{B}_n$, the probability is a probability value (\cref{def:probablity-value}): + \[ + 0 \le \PP\left[ \tilde{b} \right] \le 1. + \] + \lean{Findist.ge\_zero, Findist.le\_one, Findist.in\_prob} \leanok +\end{theorem} + +\begin{theorem}[Complement] \label{thm:prob-complement} + For each $P\in \Delta_n$, $\tilde{b}\in \mathbb{B}_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, and $t\in \mathbb{Q}$: + \begin{align*} + \PP\left[ \tilde{b} \right] + \PP\left[ \neg \tilde{b} \right] &= 1, \\ + \PP\left[ \tilde{x} \le t \right] + \PP\left[ \tilde{x} > t \right] &= 1, \\ + \PP\left[ \tilde{x} < t \right] + \PP\left[ \tilde{x} \ge t \right] &= 1. + \end{align*} + \lean{prob\_compl\_sums\_to\_one, prob\_le\_compl\_gt, prob\_lt\_compl\_ge} \leanok +\end{theorem} + +\begin{theorem}[Monotonicity] \label{thm:prob-monotone} + Suppose that $P\in \Delta_n$, $\tilde{a}, \tilde{b} \in \mathbb{B}_n$, $\tilde{x}, \tilde{y} \in \mathbb{L}_n^{\mathbb{Q}}$, and $t_1 \le t_2 \in \mathbb{Q}$. Then: + \begin{align*} + (\forall \omega\in \Omega,\, a_{\omega} \Rightarrow b_{\omega}) + &\implies \PP\left[ \tilde{a} \right] \le \PP\left[ \tilde{b} \right], \\ + \tilde{x} \le \tilde{y} + &\implies \PP\left[ \tilde{y} \le t_1 \right] \le \PP\left[ \tilde{x} \le t_2 \right], \\ + \tilde{x} \le \tilde{y} + &\implies \PP\left[ \tilde{y} < t_1 \right] \le \PP\left[ \tilde{x} < t_2 \right], \\ + t < q &\implies \PP\left[ \tilde{x} \le t \right] \le \PP\left[ \tilde{x} < q \right]. + \end{align*} + \lean{prob\_bool\_monotone, prob\_le\_monotone, prob\_lt\_monotone, prob\_lt\_le\_monotone} \leanok + \cc{The last item uses the symbols $t$ and $q$, which are not among the quantified variables ($t_1,t_2$ are).} +\end{theorem} + +The CDF inherits the monotonicity properties of the probability operator. +\begin{theorem}[CDF monotonicity] \label{thm:cdf-monotone} + For each $P\in \Delta_n$ and $\tilde{x}, \tilde{y} \in \mathbb{L}_n^{\mathbb{Q}}$: + \begin{align*} + t_1 \le t_2 &\implies F_{\tilde{x}}(t_1) \le F_{\tilde{x}}(t_2), \\ + \tilde{x} \le \tilde{y} &\implies F_{\tilde{x}}(t) \ge F_{\tilde{y}}(t), \quad \forall t\in \mathbb{Q}. + \end{align*} + \lean{cdf\_nondecreasing, cdf\_monotone\_xy} \leanok +\end{theorem} + +\begin{theorem} \label{lem:prob-lumpy} + For each $t\in \Real$: + \[ + \PP[\tilde{x} = t] > 0 + \implies + \exists \omega \in \Omega, x(\omega) = t. + \] + \lean{prob\_atomic\_omega} \leanok +\end{theorem} +\begin{proof} + The result follows by induction on the size of the probability space $|\Omega|$ after generalizing the statement to drop the constraint $\bm{1}\tr \bm{p} = 1$. + \cc{The statement quantifies $t\in \Real$ although the whole development is over $\mathbb{Q}$, and $P$ and $\tilde{x}$ are never quantified. The proof is only a sketch: the generalized statement that the induction is on is not written down, and $\bm{1}$ and $\bm{p}$ are undefined notation. A direct contrapositive argument ($x_{\omega}\ne t$ for all $\omega$ gives $\PP[\tilde{x}=t]=\sum_{\omega} P_{\omega}\cdot 0 = 0$) would avoid the induction entirely.} +\end{proof} + +\begin{theorem} \label{lem:prob-increase} + For each $t\in \Real$, the exists $q > t$ such that + \[ + \PP[\tilde{x} \le t] = + \PP[\tilde{x} < q]. + \] + Moreover, if $t < \max_{\omega \in \Omega } x_{\omega }$, then $q$ can be chosen such that $q= x_{\omega}$ for some $\omega\in \Omega$. + \lean{rv\_le\_step\_lt, prob\_le\_step\_lt, prob\_le\_step\_lt\_max} \leanok +\end{theorem} +\begin{proof} + First, assume that $t < \max_{\omega \in \Omega } x_{\omega }$ and let $\mathcal{Y} := \left\{ x_{\omega} \mid x_{\omega} > t, \omega \in \Omega \right\}$ which is nonempty by the assumption. Then, $q := \min \mathcal{Y}$ exists because $\mathcal{Y}$ is finite and nonempty. The properties $q > t$ and $\exists \,\omega \in \Omega, \, q = x_{\omega}$ hold from $q\in \mathcal{Y}$. Next, for each $\omega \in \Omega$, $x_{\omega}\le t \Rightarrow x_{\omega} < q$ follows from transitivity of $\le$ and $<$. Finally, we can prove $x_{\omega} < q \Rightarrow x_{\omega} < t$ by contradiction from the fact the $q$ is the minimum of $\mathcal{Y}$. + \cc{Error: the implication to prove is $x_{\omega} < q \Rightarrow x_{\omega} \le t$ (not $x_{\omega} < t$); with $ t$ holds trivially, and since $x_{\omega} \le \max_{\omega \in \Omega} x_{\omega} \le t < q$ for every $\omega \in \Omega$, both $\tilde{x} \le t$ and $\tilde{x} < q$ hold identically (are the constant function $\true$), giving the required equality. +\end{proof} + +\begin{theorem} \label{thm:exp_cond_eq_def} + For $\tilde{x} \in \mathbb{L}_n^{\mathbb{Q}}$ and $\tilde{b} \in \mathbb{L}_n^{\bool}$: + \[ + \E\left[\tilde{x} \mid \tilde{b} \right] \cdot \PP\left[\tilde{b}\right] = \E\left[\tilde{x} \cdot \bm{1}_{\tilde{b}}\right]. + \] + \lean{exp\_cond\_eq\_def} \leanok +\end{theorem} +\begin{proof} +By conditioning on $\PP\left[\tilde{b}\right] = 0$ and then by algebraic manipulation. +\end{proof} + + +\begin{theorem}[Law of the unconscious statistician] \label{thm:lotus} + For any $\tilde{l} \in \mathbb{L}_n^{\Nats_m}$ and $g\colon \Nats_m \to \mathbb{Q}$: + \[ + \E\left[g(\tilde{l})\right] = \sum_{i \in \Nats_m} g(i) \cdot \PP\left[\tilde{l} = i\right]. + \] + \lean{LOTUS} \leanok +\end{theorem} +\begin{proof} +\begin{align*} + \E\left[g(\tilde{l})\right] + &= \sum_{i \in \mathbb{N}_m} \E\left[g(\tilde{l}) \mid \tilde{l} = i\right] \cdot \PP\left[\tilde{l} = i\right] && \text{(def.\ expectation)} \\ + &= \sum_{i \in \mathbb{N}_m} g(i) \cdot \PP\left[\tilde{l} = i\right] && \text{(def.\ conditional expectation)} +\end{align*} +where the second step uses $g(\tilde{l}) = g(i)$ on the event $\{\tilde{l} = i\}$, and the convention $0 \cdot (\,\cdot\,) = 0$ handles the case $\PP[\tilde{l} = i] = 0$. +\cc{Circularity: the first step is justified as ``def.\ expectation'', but it is exactly the law of total expectation (\cref{thm:total-expectation}), whose proof in turn invokes this theorem. One of the two must be proved directly from \cref{def:expectation} by partitioning $\Omega$ into the preimages $\tilde{l}^{-1}(i)$, $i\in \Nats_m$. The second step also needs \cref{thm:exp_cond_eq_def} rather than the definition alone, since $\E[\,\cdot \mid \tilde{l}=i]$ divides by $\PP[\tilde{l}=i]$.} +\end{proof} + +\begin{theorem}[Law of total expectation] \label{thm:total-expectation} +Let $\tilde{x} \in \mathbb{L}_n^{\mathbb{Q}}$ and $\tilde{y} \in \mathbb{L}_{n}^{\mathbb{N}_m}$ be random variables. Then: +\[ +\E\left[\E\left[\tilde{x} \mid \tilde{y}\right]\right] = \E \left[ \tilde{x} \right]. +\] +\lean{law\_total\_exp} \leanok +\end{theorem} +\begin{proof} +\begin{align*} + \E\left[\E\left[\tilde{x} \mid \tilde{y}\right]\right] &= \sum_{i \in \mathbb{N}_m} \PP \left[\tilde{y} = i\right] \cdot \E\left[\tilde{x} \mid \tilde{y} = i\right] && \why{\cref{thm:lotus}} \\ + &= \sum_{i \in \mathbb{N}_m} \E\left[\tilde{x} \cdot \mathbf{1}_{\{\tilde{y} = i\}}\right] && \why{\cref{thm:exp_cond_eq_def}} \\ + &= \E\left[\tilde{x}\right] && \why{partition of $\Omega$} +\end{align*} +\cc{The first step applies \cref{thm:lotus} with $g(i) := \E[\tilde{x}\mid\tilde{y}=i]$, which should be said explicitly; note the resulting circularity with the proof of \cref{thm:lotus}, which in turn invokes this identity. The last step also needs linearity of expectation (\cref{thm:exp-linear}) to exchange the finite sum with $\E$, in addition to the partition property $\sum_{i} \mathbf{1}_{\{\tilde{y}=i\}} = 1$.} +\end{proof} + +The following theorem shows that our definition of the expectation operator coincides with the standard definition. +\begin{theorem} \label{thm:expect-correct} + For $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, we have that + \[ + \E[\tilde{x}] = \sum_{x \in \tilde{x}(\Nats_n)} x \cdot \PP[\tilde{x} = x]. + \] + \lean{expect\_def\_correct} \notready + (The Lean proof of this theorem is currently incomplete.) +\end{theorem} +\begin{proof} +Let $\mathcal{X} := \tilde{x}(\Nats_n)$ and choose some \emph{bijection} $o \colon \Nats_m \to \mathcal{X} $ where $m := |\mathcal{X}|$. Then by enumerating $\mathcal{X}$: + \begin{align*} + \sum_{x \in \tilde{X}} x \cdot \PP[\tilde{x} = x] + &= \sum_{k\in \Nats_m} o(k) \cdot \PP[\tilde{x} = o(k)] \\ + &= \sum_{k\in \Nats_m} o(k) \cdot \PP[o^{-1}(\tilde{x}) = k] + \end{align*} +The result then follows from \cref{thm:lotus} by setting $l_i = o^{-1}(x_i), i\in \Nats_n$ and $g(k) = o(k), k\in \Nats_m$. +\cc{Gaps: (i) the first display writes $\sum_{x\in \tilde{X}}$ where $\mathcal{X}$ is meant; (ii) the step $\PP[\tilde{x}=o(k)] = \PP[o^{-1}(\tilde{x})=k]$ needs the injectivity of $o$ and that $o^{-1}$ is only defined on $\mathcal{X} = \tilde{x}(\Nats_n)$, which is where $\tilde{x}$ takes its values; (iii) the final appeal to \cref{thm:lotus} additionally requires $o(o^{-1}(x_i)) = x_i$ for all $i$, i.e.\ that $g\circ \tilde{l} = \tilde{x}$, which is not stated.} +\end{proof} + + +\begin{theorem}[Law of Total Probability] \label{thm:total-probability} +Let $\tilde{b} \colon \Omega \to \mathbb{B}$ and $\tilde{y} \colon \Omega \to \mathcal{Y}$ be random variables with a finite set $\mathcal{Y}$. Then: +\[ +\sum_{y\in \mathcal{Y}} \PP\left[\tilde{b} \wedge (\tilde{y} = y) \right] = \PP \left[ \tilde{b} \right]. +\] +\lean{law\_of\_total\_probs} \leanok +\cc{No proof is given. Also, $\tilde{y}$ is typed into an arbitrary finite $\mathcal{Y}$ while the probability operator is only defined for $\mathbb{L}_n^{\Nats_m}$-valued variables, and $\mathbb{B}$ is used where $\bool$ was defined.} +\end{theorem} + +\subsection{Quantile} + +This section defines quantiles of random variables and states their basic properties. See \texttt{Probability/Quantile.lean}. + +\begin{definition}[Quantile] \label{def:quantile} +For $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, and $\alpha \in \mathbb{Q}$, a value $q\in \mathbb{Q}$ is an \emph{$\alpha$-quantile} of $\tilde{x}$ when +\[ + \PP\left[ \tilde{x} \le q \right] \ge \alpha + \quad \wedge \quad + \PP\left[ \tilde{x} \ge q \right] \ge 1 - \alpha. +\] +The set of all $\alpha$-quantiles is denoted by $\mathcal{Q}(P, \tilde{x}, \alpha) := \left\{ q\in \mathbb{Q} \mid q \text{ is an } \alpha\text{-quantile} \right\}$. +\lean{Statistic.IsQuantile, Statistic.Quantile} \leanok +\cc{$\alpha$ is only required to be in $\mathbb{Q}$; for $\alpha \notin [0,1]$ the definition degenerates (for $\alpha<0$ every $q$ larger than $\max_{\omega} x_{\omega}$ qualifies, for $\alpha>1$ the set is empty). A hypothesis $\alpha\in[0,1]$ appears to be intended.} +\end{definition} + +\begin{definition}[Lower quantile bound] \label{def:quantile-lower} +For $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, and $\alpha \in \mathbb{Q}$, a value $q\in \mathbb{Q}$ is a \emph{lower bound on the $\alpha$-quantile} of $\tilde{x}$ when +\[ + \PP\left[ \tilde{x} \ge q \right] \ge 1 - \alpha. +\] +The set of all such values is denoted by $\mathcal{Q}_{\mathrm{L}}(P, \tilde{x}, \alpha)$. +\lean{Statistic.IsQuantileLower, Statistic.QuantileLower} \leanok +\end{definition} + +\begin{theorem} \label{thm:qsetlower-lt} +For each $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in \mathbb{Q}$, and $q\in \mathbb{Q}$: +\[ + q \in \mathcal{Q}_{\mathrm{L}}(P, \tilde{x}, \alpha) + \iff + \PP\left[ \tilde{x} < q \right] \le \alpha. +\] +\lean{Statistic.qsetlower\_def\_lt} \leanok +\end{theorem} +\begin{proof} +From \cref{thm:prob-complement} by algebraic manipulation. +\end{proof} + +\begin{theorem} \label{thm:quantile-sub-lower} +Every quantile is a lower quantile bound: +\[ + \mathcal{Q}(P, \tilde{x}, \alpha) \subseteq \mathcal{Q}_{\mathrm{L}}(P, \tilde{x}, \alpha). +\] +\lean{Statistic.quantile\_implies\_quantilelower, Statistic.quantile\_subset\_quantilelower} \leanok +\end{theorem} + +\begin{theorem}[Negation] \label{thm:quantile-neg} +For each $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in \mathbb{Q}$, and $q\in \mathbb{Q}$: +\[ + q \in \mathcal{Q}(P, \tilde{x}, \alpha) + \iff + -q \in \mathcal{Q}(P, -\tilde{x}, 1-\alpha). +\] +\lean{Statistic.isquant\_neg, Statistic.quantile\_neg} \leanok +\end{theorem} + +\begin{theorem}[Monotone transformation] \label{thm:quantile-strictmono} +Suppose that $f\colon \mathbb{Q} \to \mathbb{Q}$ is strictly increasing. Then: +\[ + q \in \mathcal{Q}(P, \tilde{x}, \alpha) + \iff + f(q) \in \mathcal{Q}(P, f \circ \tilde{x}, \alpha), +\] +and analogously for $\mathcal{Q}_{\mathrm{L}}$. If $f$ is merely non-decreasing, then only the forward implications hold. +\lean{Statistic.quantile\_f\_strictmono, Statistic.quantilelower\_f\_strictmono, Statistic.quantile\_f\_monotone, Statistic.quantilelower\_f\_monotone} \leanok +\end{theorem} + +\begin{theorem}[Translation] \label{thm:quantilelower-cash} +Adding a constant $c\in \mathbb{Q}$ to a random variable shifts its lower quantile set: +\[ + \mathcal{Q}_{\mathrm{L}}(P, \tilde{x} + c \bm{1}, \alpha) + = + \left\{ q + c \mid q \in \mathcal{Q}_{\mathrm{L}}(P, \tilde{x}, \alpha) \right\}. +\] +\lean{Statistic.quantilelower\_cashinv, Statistic.quantilelower\_cash\_image} \leanok +\end{theorem} + +\section{Risk Measures} + +\subsection{VaR: Value at Risk} + +This section defines the Value at Risk (VaR) risk measure and states its basic properties. See \texttt{Risk/VaR.lean}. + +\begin{definition}[Risk level] \label{def:risk-level} +An $\alpha \in \mathbb{Q}$ is a \emph{risk level} when $0 \le \alpha < 1$. +\lean{Risk.IsRiskLevel, Risk.RiskLevel} \leanok +\end{definition} + +\begin{definition} \label{def:var} +A condition $\isvar(P,\tilde{x},\alpha,v)$ for $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in [0,1)$, and $v\in \mathbb{Q}$ states that $v$ is the greatest element of the lower quantile set (\cref{def:quantile-lower}): + \[ + \isvar(P,\tilde{x},\alpha,v) := \left(v \in \max \mathcal{Q}_{\mathrm{L}}(P, \tilde{x}, \alpha)\right) + = \left(v \in \max \left\{ t \in \mathbb{Q} \mid \PP^P[\tilde{x} < t] \le \alpha \right\}\right), + \] + where the second equality follows from \cref{thm:qsetlower-lt}. + \lean{Risk.IsVaR} \leanok + \cc{Notation error: $\max \mathcal{Q}_{\mathrm{L}}$ denotes an element, so ``$v \in \max \mathcal{Q}_{\mathrm{L}}$'' is not well-formed; the intended meaning is that $v$ is a \emph{greatest element} of $\mathcal{Q}_{\mathrm{L}}$, i.e.\ $v\in \mathcal{Q}_{\mathrm{L}} \wedge \forall t\in \mathcal{Q}_{\mathrm{L}}, t\le v$. The same abuse occurs in \cref{def:var-q}. Nothing here (or later) proves that such a greatest element \emph{exists} for every $P,\tilde{x},\alpha$; that is what \cref{thm:finvar-correct} supplies, so the two should be ordered accordingly.} +\end{definition} + +An alternative definition uses the quantile set (\cref{def:quantile}) instead of the lower quantile set; \cref{thm:varq-eq-var} shows that the two definitions are equivalent. +\begin{definition} \label{def:var-q} +A condition $\isvar_{\mathrm{Q}}(P,\tilde{x},\alpha,v)$ for $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in [0,1)$, and $v\in \mathbb{Q}$ states that $v$ is the greatest element of the quantile set: + \[ + \isvar_{\mathrm{Q}}(P,\tilde{x},\alpha,v) := \left(v \in \max \mathcal{Q}(P, \tilde{x}, \alpha)\right). + \] + \lean{Risk.IsVaR\_Q} \leanok +\end{definition} + +\begin{definition} \label{def:finvar} +The discrete VaR operator for $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in [0,1)$ is defined as +\[ + \finvar_{\alpha } \left[ \tilde{x} \right] + := + \max \left\{ x_i \mid i\in \Nats_n,\, \PP \left[ \tilde{x} < x_i \right] \le \alpha \right\}. +\] +The maximum is well-defined because the set is finite and nonempty; see \cref{thm:finvarset-nonempty}. +\lean{Risk.FinVaR} \leanok +\end{definition} + +\begin{remark} +\Cref{def:finvar} differs from the standard VaR definition in \cref{def:var} in that it only searches for the discrete values that satisfy the VaR condition. +\end{remark} + +\begin{theorem} \label{thm:finvarset-nonempty} +For $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in [0,1)$, the set +$\left\{ x_i \mid i\in \Nats_n,\, \PP \left[ \tilde{x} < x_i \right] \le \alpha \right\}$ is nonempty. +\lean{Risk.FinVarSet\_nonempty} \leanok +\end{theorem} +\begin{proof} +The minimum $x_{\min} := \min_{\omega\in \Omega} x_{\omega}$ is a member of the set because $\PP[\tilde{x} < x_{\min}] = 0 \le \alpha$. +\end{proof} + +\begin{theorem} \label{thm:finvar-prob-cond} +For $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in [0,1)$: + \[ + \PP\left[\tilde{x} < \finvar_{\alpha }[\tilde{x}]\right] + \le \alpha + < \PP\left[\tilde{x} \le \finvar_{\alpha }[\tilde{x}] \right]. + \] + \lean{Risk.finvar\_prob\_cond} \leanok +\end{theorem} +\begin{proof} +The inequality $\PP\left[\tilde{x} < \finvar_{\alpha }[\tilde{x}]\right] \le \alpha $ follows directly from the definition of $\finvar$ because the maximizer is a member of the set. Then, to prove $\alpha < \PP\left[\tilde{x} \le \finvar_{\alpha }[\tilde{x}] \right]$ by contradiction, we show that $\alpha \ge \PP\left[\tilde{x} \le \finvar_{\alpha }[\tilde{x}] \right]$ contradicts with the maximality of $x_i$ in \cref{def:finvar}. In particular, for any $x_i$ there exists $x_j > x_i$ such that $\alpha \ge \PP\left[\tilde{x} \le x_i \right] = \PP\left[\tilde{x} \le x_j \right]$ using \cref{lem:prob-increase} since $\PP\left[\tilde{x} \le x_i \right] \le \alpha < 1$. +\cc{Error: \cref{lem:prob-increase} gives $\PP[\tilde{x}\le x_i] = \PP[\tilde{x} < x_j]$, with a \emph{strict} inequality on the right, not $\PP[\tilde{x}\le x_j]$. The strict form is also what is needed: it shows $x_j$ belongs to the set of \cref{def:finvar}, and $x_j > x_i$ then contradicts maximality. In addition, the ``moreover'' clause of \cref{lem:prob-increase} (which is what yields $x_j = x_{\omega}$ for some $\omega$) requires $x_i < \max_{\omega} x_{\omega}$; this follows from $\PP[\tilde{x}\le x_i]\le \alpha<1$ but the argument is not spelled out.} +\end{proof} + +\begin{theorem} \label{thm:var-prob-cond} + For $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in [0,1)$: + \[ + \isvar(P,\tilde{x},\alpha, v) + \Longleftrightarrow + \PP^P[\tilde{x} < v] \le \alpha < \PP^P[\tilde{x} \le v]. + \] + \lean{Risk.var\_prob\_cond} \leanok +\end{theorem} +\begin{proof} + We decompose the proof into four parts: the first two parts show the forward implication and second two parts show the backward implication. + + \emph{First}, the implication $\isvar(P,\tilde{x},\alpha, v) \implies \PP^P[\tilde{x} < v] \le \alpha$ follows directly from the definition of $\isvar$ algebraic manipulation. + + \emph{Second}, we prove $\isvar(P,\tilde{x},\alpha, v) \implies \alpha < \PP^P[\tilde{x} \le v]$ by contradiction supposing that $\alpha \ge \PP^P[\tilde{x} \le v]$. Then, using \cref{lem:prob-increase}, there exists $q > v$ such that $\alpha \ge \PP^P[\tilde{x} \le v] = \PP^P[\tilde{x} < q]$ which contradicts that $v$ is maximal in the definition of $\isvar$. + + \emph{Third}, the implication $\PP^P[\tilde{x} < v] \le \alpha \implies v \in \left\{ t \in \mathbb{Q} \mid \PP^P[\tilde{x} < t] \le \alpha \right\}$ follows immediately from the definition. + + \emph{Fourth}, to prove $\alpha < \PP^P[\tilde{x} \le v] \implies \forall t \in \left\{ t \in \mathbb{Q} \mid \PP^P[\tilde{x} < t] \le \alpha \right\}, v \ge t$ by contradiction, suppose that $\exists t \in \left\{ t \in \mathbb{Q} \mid \PP^P[\tilde{x} < t] \le \alpha \right\}, v < t$. Then, by the monotonicity of the probability operator we get a contradiction as + \[ + \alpha + < \PP^P \left[ \tilde{x} \le v \right] + \le \PP^P \left[ \tilde{x} < t \right] + \le \alpha. + \] +\cc{The forward direction additionally needs $v$ to be a \emph{member} of the set (not merely an upper bound), which the ``First'' paragraph glosses as ``follows directly \ldots{} algebraic manipulation''; a word appears to be missing there too. Nothing in this proof establishes that a $v$ satisfying $\isvar$ exists.} +\end{proof} + +The following theorem is the main correctness result: the discrete VaR operator computes a value that satisfies the VaR condition. +\begin{theorem}[Correctness of discrete VaR] \label{thm:finvar-correct} +For $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in [0,1)$: + \[ + \isvar\left(P, \tilde{x}, \alpha, \finvar_{\alpha}[\tilde{x}]\right). + \] + \lean{Risk.finvar\_correct} \leanok +\end{theorem} +\begin{proof} +Combine \cref{thm:finvar-prob-cond} with \cref{thm:var-prob-cond}. +\end{proof} + +\begin{theorem}[Equivalence of VaR definitions] \label{thm:varq-eq-var} +For $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in [0,1)$, and $v\in \mathbb{Q}$: + \[ + \isvar_{\mathrm{Q}}(P,\tilde{x},\alpha, v) + \Longleftrightarrow + \isvar(P,\tilde{x},\alpha, v). + \] + Consequently, $\isvar_{\mathrm{Q}}(P,\tilde{x},\alpha, v) \Leftrightarrow \PP^P[\tilde{x} < v] \le \alpha < \PP^P[\tilde{x} \le v]$. + \lean{Risk.varq\_eq\_var, Risk.varq\_prob\_cond} \leanok +\end{theorem} +\begin{proof} +The sets $\mathcal{Q}(P,\tilde{x},\alpha)$ and $\mathcal{Q}_{\mathrm{L}}(P,\tilde{x},\alpha)$ are mutually cofinal (each element of one set is dominated by an element of the other), and therefore they share upper bounds and greatest elements. +\cc{Gap: only one half of the cofinality claim is established anywhere ($\mathcal{Q}\subseteq \mathcal{Q}_{\mathrm{L}}$, \cref{thm:quantile-sub-lower}); the other half --- that every $q\in \mathcal{Q}_{\mathrm{L}}$ is dominated by some element of $\mathcal{Q}$ --- is asserted without proof. The only apparent route to it is \cref{thm:var-is-quantile}, which is stated \emph{after} this theorem and is itself given without proof, so as written the development is circular. Note also that cofinality alone does not imply equal greatest elements; the inclusion $\mathcal{Q}\subseteq\mathcal{Q}_{\mathrm{L}}$ is needed as well and should be cited.} +\end{proof} + +Every VaR value is a quantile. +\begin{theorem} \label{thm:var-is-quantile} +For $P\in \Delta_n$, $\tilde{x}\in \mathbb{L}_n^{\mathbb{Q}}$, $\alpha\in [0,1)$, and $v\in \mathbb{Q}$: + \[ + \isvar(P,\tilde{x},\alpha, v) + \implies + v \in \mathcal{Q}(P, \tilde{x}, \alpha). + \] + In particular, the quantile set $\mathcal{Q}(P, \tilde{x}, \alpha)$ is nonempty. + \lean{Risk.var\_is\_quantile, Risk.quantile\_nonempty} \leanok + \cc{No proof is given. The natural proof goes through \cref{thm:var-prob-cond} together with \cref{thm:prob-complement}; it should be written out, especially since \cref{thm:varq-eq-var} appears to depend on it.} +\end{theorem} + +\subsubsection{VaR Properties} + +\begin{theorem}[Monotonicity] \label{thm:var-monotone} +Suppose that $\tilde{x} \le \tilde{y}$ element-wise. Then, for $\alpha\in [0,1)$: + \[ + \isvar(P,\tilde{x},\alpha, v_1) \wedge \isvar(P,\tilde{y},\alpha, v_2) + \implies + v_1 \le v_2. + \] + \lean{Risk.var\_monotone} \leanok + \cc{No proof is given (it follows from \cref{thm:var-prob-cond} and \cref{thm:prob-monotone}). Also, $P$, $\tilde{x}$, $\tilde{y}$, $v_1$, $v_2$ are not quantified in the statement.} +\end{theorem} + +\begin{theorem}[Translation invariance] \label{thm:var-translation} +For each $c\in \mathbb{Q}$ and $\alpha\in [0,1)$: + \[ + \finvar_{\alpha}\left[ \tilde{x} + c \bm{1} \right] + = + \finvar_{\alpha}\left[ \tilde{x} \right] + c. + \] + \lean{Risk.isvar\_translation\_invariant, Risk.var\_translation\_invariant} \leanok +\end{theorem} +\begin{proof} +From \cref{thm:quantilelower-cash} and \cref{thm:finvar-correct}. +\cc{Gap: \cref{thm:finvar-correct} only says that $\finvar_{\alpha}$ \emph{satisfies} the $\isvar$ predicate. To conclude an \emph{equality} of $\finvar$ values one needs the (missing) lemma that the value satisfying $\isvar(P,\tilde{x},\alpha,\cdot)$ is unique --- immediate from the uniqueness of a greatest element, but never stated. The same missing lemma is needed in \cref{thm:var-strictmono}. Also, $P$, $\tilde{x}$ are not quantified.} +\end{proof} + +\begin{theorem}[Monotone transformation] \label{thm:var-strictmono} +Suppose that $f\colon \mathbb{Q} \to \mathbb{Q}$ is strictly increasing. Then, for $\alpha\in [0,1)$: + \[ + \finvar_{\alpha}\left[ f \circ \tilde{x} \right] + = + f\left( \finvar_{\alpha}\left[ \tilde{x} \right] \right). + \] + \lean{Risk.isvar\_f\_strictmono, Risk.var\_f\_strictmono} \leanok + \cc{No proof is given; it should follow from \cref{thm:quantile-strictmono} plus the uniqueness lemma noted above. $P$ and $\tilde{x}$ are again unquantified.} +\end{theorem} + +\begin{corollary}[Positive homogeneity] \label{thm:var-homog} +For each $c > 0$ and $\alpha\in [0,1)$: + \[ + \finvar_{\alpha}\left[ c \cdot \tilde{x} \right] + = + c \cdot \finvar_{\alpha}\left[ \tilde{x} \right]. + \] + \lean{Risk.var\_positive\_homog} \leanok +\end{corollary} +\begin{proof} +Apply \cref{thm:var-strictmono} with $f(x) := c\cdot x$. +\end{proof} + + +\section{Formal Decision Framework} + +\subsection{Markov Decision Process} + +\newcommand{\Fin}{\operatorname{Fin}} + +\begin{definition} \label{def:MDP} +A \emph{Markov decision process} $M := (S, A, P, r)$ consists of: +\begin{itemize} + \item positive integer $S \in \Nats_{>0}$ representing state set $\Nats_S$ + \item positive integer $A \in \Nats_{>0}$ representing action set $\Nats_A$ + \item transition function $P \colon \Nats_S \times \Nats_A \to \Delta_S$, mapping each state–action to the next state distribution + \item reward function $r \colon \Nats_S \times \Nats_A \times \Nats_S \to \Real$, mapping each transition to a reward +\end{itemize} +\lean{MDPs.MDP} \leanok +\cc{The rest of the document writes $\mathcal{S}$, $\mathcal{A}$ and $p$ for the components of $M$, none of which are introduced here ($\Nats_S$, $\Nats_A$ and $P$ are). Also, $r$ maps into $\Real$ while everything else is restricted to $\mathbb{Q}$ for computability.} +\end{definition} + +\subsection{Histories} + +We implicitly assume in the remainder of the section an MDP $M = (S, A, P, r)$. +\begin{definition} \label{def:Hist} +A \emph{history} $h$ in a set of histories $\mathcal{H}$ is a sequence of states and actions defined for $M$ recursively as +\[ + h := \langle s_0 \rangle, \quad + \why{or} \quad + h := \langle h', a, s \rangle, +\] +where $s_0, s \in \Fin(S)$, $a \in \Fin(A)$, and $h'\in \mathcal{H}$. +\lean{MDPs.Hist} \leanok +\end{definition} + +\begin{definition} \label{def:hist-length} +The \emph{length} $|h|\in \Nats$ of a history $h\in \mathcal{H}$ is the number of actions taken, defined as +\begin{align*} +|\langle s \rangle| &:= 0, \\ +|\langle h', a, s \rangle| &:= 1 + |h'|, \qquad h' \in \mathcal{H}. +\end{align*} +\lean{MDPs.Hist.length} \leanok +\end{definition} + +\begin{theorem} \label{thm:hist-foll-len} +For each $h\in \mathcal{H}$, $a\in \mathcal{A}$, and $s\in \mathcal{S}$: +\[ + |\langle h, a, s \rangle| = |h| + 1. +\] +\lean{MDPs.hist\_foll\_len} \leanok +\end{theorem} + +\begin{definition} \label{def:hist-last} +The \emph{last state} $s_{\mathrm{last}}\colon \mathcal{H} \to \mathcal{S}$ of a history is defined as +\begin{align*} + s_{\mathrm{last}}(\langle s \rangle) &:= s, \\ + s_{\mathrm{last}}(\langle h', a, s \rangle) &:= s. +\end{align*} +\lean{MDPs.Hist.last} \leanok +\end{definition} + +\begin{definition} \label{def:hist-ne} +The set $\mathcal{H}_{\mathrm{NE}}$ of \emph{non-empty histories} is +\[ +\mathcal{H}_{\mathrm{NE}} := \left\{ h \in \mathcal{H} \mid |h| \ge 1 \right\}. +\] +\lean{MDPs.HistNE} \leanok +\end{definition} + +\begin{definition}\label{def:histories} + \emph{Following histories} $\mathcal{H}(h,t) \subseteq \mathcal{H}$ for $h\in \mathcal{H}$ and $t\in \Nats$ are defined recursively as + \[ + \mathcal{H}(h,t) := + \begin{cases} + \left\{ h \right\} &\text{if } t = 0, \\ + \left\{ \langle h', a, s \rangle \mid h'\in \mathcal{H}(h, t-1), a\in \mathcal{A}, s\in \mathcal{S} \right\} &\text{otherwise}. \\ + \end{cases} + \] + \lean{MDPs.Histories} \leanok +\end{definition} + +\begin{definition} \label{def:histories-horizon} + The set of \emph{histories} $\mathcal{H}_{t}$ of \emph{length} $t \in \Nats$ is defined recursively as + \[ + \mathcal{H}_t = + \begin{cases} + \left\{ \langle s \rangle \mid s\in \mathcal{S} \right\} &\text{if } t = 0, \\ + \left\{ \langle h, a, s\rangle \mid h \in \mathcal{H}_{t-1}, a\in \mathcal{A}, s\in \mathcal{S} \right\} &\text{otherwise}. + \end{cases} + \] + \lean{MDPs.MDP.HistoriesHorizon} \leanok +\end{definition} + +The following theorem shows that $\mathcal{H}_t$ contains exactly the histories of length $t$; as a consequence, the histories of length $t$ form a finite type. +\begin{theorem} \label{thm:hist-horizon-exact} + For each $t\in \Nats$ and $h\in \mathcal{H}$: + \[ + h \in \mathcal{H}_t \iff |h| = t. + \] + \lean{MDPs.hist\_horiz\_complete, MDPs.hist\_horiz\_exact, MDPs.MDP.HistoriesHorizonT} \leanok +\end{theorem} +\begin{proof} +Both directions follow by induction on $t$. +\end{proof} + +\begin{theorem} \label{thm:hist-lenth-eq-horizon} + For $h \in \mathcal{H}$: + \[ + |h'| = |h| + t, \qquad \forall h'\in \mathcal{H}(h, t). + \] +\lean{MDPs.hist\_lenth\_eq\_horizon} \notready +(The Lean proof of this theorem is currently incomplete.) +\end{theorem} +\begin{proof} +The theorem follows by induction on $t$ from the definition. +\end{proof} + +\begin{definition} \label{def:hist-prefix} +The \emph{prefix} $h_{\le k} \in \mathcal{H}$ of a history $h \in \mathcal{H}$ is the longest prefix of $h$ with length at most $k\in \Nats$. +\lean{MDPs.Hist.prefix} \leanok +\end{definition} + +\begin{remark} +The definitions and theorems in the remainder of this document (the $k$-th states and actions of histories, history rewards, policies, history distributions, and the dynamic programming results) have \emph{not yet been formalized} in the Lean code. The \texttt{MDPs.*} names in comments are placeholders for the intended formalization. +\end{remark} + +\begin{definition} + We use $\tilde{s}_k\colon \mathcal{H} \to \mathcal{S}$ to denote the 0-based $k$-th state of each history. + %%\lean{MDPs.state}\leanok +\end{definition} + +\begin{definition} + We use $\tilde{a}_k\colon \mathcal{H} \to \mathcal{A}$ to denote the 0-based $k$-th action of each history. + %%\lean{MDPs.action}\leanok +\end{definition} + +\begin{definition} \label{def:reward} +The \emph{history-reward} random variable $\rew\colon \mathcal{H} \to \Real$ for $h = \langle h', a, s \rangle \in \mathcal{H}$ for $h'\in \mathcal{H}$, $a\in \mathcal{A}$, and $s\in \mathcal{S}$ is defined recursively as +\[ +\rew(h) := r(s_{|h|}(h'), a, s) + r_{\mathrm{h}}(h'). +\] +%%\lean{MDPs.reward} \leanok +\cc{Three errors: (i) the base case $\rew(\langle s \rangle) := 0$ is missing, so the recursion is not well-founded; (ii) the right-hand side writes $r_{\mathrm{h}}(h')$, an undefined symbol --- it should be $\rew(h')$; (iii) $s_{|h|}(h')$ is out of range for $h'$, which has states indexed $0,\dots,|h'| = |h|-1$; the intended term is the last state of $h'$, i.e.\ $s_{|h'|}(h')$.} +\end{definition} + +\begin{definition}\label{reward_at} +The \emph{history-reward} random variable $\rew_k \colon \mathcal{H} \to \Real$ for $h = \langle h', a, s \rangle \in \mathcal{H}$ for $h'\in \mathcal{H}$, $a\in \mathcal{A}$, and $s\in \mathcal{S}$ is defined as the $k$-th reward (0-based) of a history. +%%\lean{MDPs.reward_at} \leanok +\end{definition} + +\begin{definition}\label{reward_to} +The \emph{history-reward} random variable $\rew_{\le k} \colon \mathcal{H} \to \Real$ for $h = \langle h', a, s \rangle \in \mathcal{H}$ for $h'\in \mathcal{H}$, $a\in \mathcal{A}$, and $s\in \mathcal{S}$ is defined as the sum of all $k$-th or earlier rewards (0-based) of a history. +%%\lean{MDPs.reward_to} \leanok +\end{definition} + +\begin{definition}\label{reward_from} +The \emph{history-reward} random variable $\rew_{\ge k}\colon \mathcal{H} \to \Real$ for $h = \langle h', a, s \rangle \in \mathcal{H}$ for $h'\in \mathcal{H}$, $a\in \mathcal{A}$, and $s\in \mathcal{S}$ is defined as the sum of $k$-th or later reward (0-based) of a history. +%%\lean{MDPs.reward_from} \leanok +\cc{\Cref{reward_at,reward_to,reward_from} give no defining equations at all --- only prose --- and the preamble ``for $h=\langle h',a,s\rangle$'' is vacuous since it constrains nothing. In particular it is left unspecified whether $\rew_{\ge k}$ includes the reward at index $k$, and what these evaluate to when $|h| \le k$. The dynamic-programming proofs later depend on exactly this convention (e.g.\ the split $\rew = \rew_{\le k_0-1} + \rew_{\ge k_0}$ used in \cref{thm:sum-rew-eq-sum-rew-rg} and \cref{thm:dph-correct-vf}), so it must be pinned down.} +\end{definition} + +\subsection{Policies} + +\begin{definition} \label{def:decision-rule} +The set of \emph{decision rules} $\mathcal{D}$ is defined as \(\mathcal{D} := \mathcal{A}^{\mathcal{S}}. \) A single action $a \in \mathcal{A}$ can also be interpreted as a decision rule $\mathcal{d} := s \mapsto a$. +%%\lean{MDPs.DecisionRule}\leanok +\cc{Typo: $\mathcal{d}$ should be $d$ (or $\mathcal{D}$-typeset consistently).} +\end{definition} + +\begin{definition} \label{def:policy-hr} +The set of \emph{history-dependent policies} is \(\PiHR := \Delta(\mathcal{A})^{\mathcal{H}}. \) +%%\lean{MDPs.PolicyHR} \leanok +\end{definition} + + +\begin{definition} \label{def:policy-md} +The set of \emph{Markov deterministic policies} $\PiMD$ is \(\PiMD := \mathcal{D}^{\Nats}. \) +A Markov deterministic policy $\pi \in \PiMD$ can also be interpreted as $\bar{\pi} \in \PiHR$: +\[ + \bar{\pi}(h) := \delta \left[ \pi(|h|, s_{|h|}(h)) \right], +\] +where $\delta$ is the Dirac distribution, and $s_{|h|}$ is the history's last state. +%%\lean{MDPs.PolicyMD}\leanok +\end{definition} + +\begin{definition} \label{def:policy-sd} +The set of \emph{stationary deterministic policies} $\Pi_{\mathrm{SD}}$ is defined as \(\Pi_{\mathrm{SD}} := \mathcal{D}. \) +A stationary policy $\pi \in \Pi_{\mathrm{SD}}$ can be interpreted as $\bar{\pi} \in \PiHR$: +\[ + \bar{\pi}(h) := \delta \left[ \pi(s_{|h|}(h)) \right], +\] +where $\delta$ is the Dirac distribution and $s_{|h|}$ is the history's last state. +%%\lean{MDPs.PolicySD}\leanok +\end{definition} + +\subsection{Distribution} + + +\begin{definition}\label{def:hist-dist} +The \emph{history probability distribution} $p^{\mathrm{h}}_T \colon \PiHR \to \Delta(\mathcal{H}(h,t))$ and $\pi \in \PiHR$ is defined for each $T \in \Nats$ and $h\in \mathcal{H}(\hat{h},t)$ as +\[ +(p^{\mathrm{h}}_T(\pi))(h) := +\begin{cases} +\I(h = \hat{h}) &\text{if } T = 0, \\ +p_{T-1}^{\mathrm{h}}(h', \pi) \cdot \pi(h',a) \cdot p(s_{|h|}(h'), a , s) &\text{if } T > 1 \wedge h = \langle h', a, s \rangle. +\end{cases} +\] +Moreover, the function $p^{\mathrm{h}}$ maps policies to correct probability distribution. +%%\lean{MDPs.HistDist} \leanok +\cc{Several problems: (i) $\hat{h}$ and $t$ occur free --- they are never bound by the definition, although the codomain $\Delta(\mathcal{H}(h,t))$ and the base case $\I(h=\hat{h})$ both use them (and the codomain uses $h$ where $\hat{h}$ is presumably meant); (ii) the second case is guarded by $T>1$, leaving $T=1$ undefined --- it should be $T\ge 1$; (iii) the recursive call is written $p^{\mathrm{h}}_{T-1}(h',\pi)$, with arguments in the opposite order to the left-hand side $(p^{\mathrm{h}}_T(\pi))(h)$; (iv) $s_{|h|}(h')$ is out of range for $h'$ and should be $s_{|h'|}(h')$, and $p$ is the transition function called $P$ in \cref{def:MDP}; (v) the closing sentence asserts that $p^{\mathrm{h}}$ returns a genuine probability distribution (i.e.\ that it is nonnegative and sums to one over $\mathcal{H}(\hat{h},t)$) without any proof.} +\end{definition} + +TODO: This definition needs to be updated. +A probability space $(\Omega_{h,t}, 2^{\Omega_{h,t} }, \hat{p}_{h,\pi})$ which is defined as +\begin{align} +\label{eq:omega-def} + \Omega_{h,t} &:= \left\{ h' \in \mathcal{H}_{|h| + t} \mid s_k(h) = s_k(h') \wedge a_k(h) = a_k(h'), \forall k \le |h| \right\}, \\ + \label{eq:phat-def} +\hat{p}_{h,\pi}\left(\left< h', a, s \right>\right) +&:= \begin{cases} +1 &\text{if } \left< h', a, s \right> = h, \\ +\hat{p}_{h,\pi}(h') \cdot \pi(h', a) \cdot p(s_{|h'|}(h'), a, s) +&\text{otherwise}, +\end{cases} +\end{align} +for each $\left< h', a, s \right> \in \Omega_{h,t}$. The random variables are defined as $\tilde{s}_k(h') := s_k(h')$, $\tilde{a}_k(h') := a_k(h')$, $\forall h'\in \Omega_{h,t}$. We interpret the subscripts analogously on all operators, including other risk measures, and $\E$, and $\mathbb{P}$. +\cc{This block is explicitly flagged as needing an update, and it currently conflicts with \cref{def:hist-dist}: two different history distributions ($p^{\mathrm{h}}$ and $\hat{p}_{h,\pi}$) are defined, and the later proofs use $\hat{p}$ while \cref{def:expect-h} uses $p^{\mathrm{h}}$. Concretely: (i) \eqref{eq:omega-def} imposes $a_k(h)=a_k(h')$ for all $k\le |h|$, but actions of a history of length $|h|$ are indexed only $0,\dots,|h|-1$, so the case $k=|h|$ is undefined; (ii) $\hat{p}_{h,\pi}$ is declared on $\Omega_{h,t}$, whose elements all have length $|h|+t$, yet the recursion evaluates $\hat{p}_{h,\pi}(h')$ at strictly shorter histories that are not in $\Omega_{h,t}$ --- the domain of the recursion must be enlarged (e.g.\ to all $h'$ extending $h$); (iii) the first case ``$=1$ if $\langle h',a,s\rangle = h$'' can never fire when $t\ge 1$, and cannot be written in the form $\langle h',a,s\rangle$ at all when $|h|=0$, so the base case is unreachable/ill-formed as stated.} + + + +\begin{definition}\label{def:expect-h} +The \emph{history-dependent expectation} is defined for each $t\in \Nats$, $\pi\in \PiHR$, $\hat{h}\in \mathcal{H}$ and a $\tilde{x}\colon \mathcal{H} \to \Real$ as +\begin{equation*} +\E^{\hat{h}, \pi, t} [\tilde{x}] +:= \E \left[ \tilde{x} \right] += \sum_{h \in \mathcal{H}(\hat{h}, t)} p^{\mathrm{h}}(h, \pi) \cdot \tilde{x}(h). +\end{equation*} +In the $\E$ operator above, the random variable $\tilde{x}$ lives in a probability space $(\Omega, p)$ where $\Omega = \mathcal{H}(\hat{h}, t)$ and $p(h) = p^{\mathrm{h}}(h, \pi), \forall h\in \Omega$. +Moreover, if $\hat{h}$ is a state, then it is interpreted as a history with the single initial state. +%%\lean{MDPs.expect_h} \leanok +\cc{$p^{\mathrm{h}}(h,\pi)$ is written without the horizon subscript and with the arguments in the opposite order to \cref{def:hist-dist}. It is also left unproved that $p$ restricted to $\Omega=\mathcal{H}(\hat{h},t)$ is a probability distribution, which is what makes the appeal to \cref{def:expectation} legitimate; and $\tilde{x}$ maps into $\Real$ while \cref{def:expectation} requires $\mathbb{Q}$.} +\end{definition} + + +\begin{definition}\label{def:expect-h-cnd} +The \emph{history-dependent expectation} is defined for each $t\in \Nats$, $\pi\in \PiHR$, $\hat{h}\in \mathcal{H}$, $\tilde{x}\colon \mathcal{H} \to \Real$, $\tilde{b}\colon \mathcal{H} \to \mathcal{\mathbb{B}}$ as +\begin{equation*} +\E^{\hat{h}, \pi, t} [\tilde{x} \mid \tilde{b} ] +:= \E \left[ \tilde{x} \mid \tilde{b} \right]. +\end{equation*} +In the $\E$ operator above, the random variables $\tilde{x}$ and $\tilde{b}$ live in a probability space $(\Omega, p)$ where $\Omega = \mathcal{H}(\hat{h}, t)$ and $p(h) = p^{\mathrm{h}}(h, \pi), \forall h\in \Omega$. +Moreover, if $\hat{h}$ is a state, then it is interpreted as a history with the single initial state. +%%\lean{MDPs.expect_h_cnd} \leanok +\end{definition} + +\begin{definition}\label{def:expect-h-cnd-rv} +The \emph{history-dependent expectation} is defined for each $t\in \Nats$, $\pi\in \PiHR$, $\hat{h}\in \mathcal{H}$, $\tilde{x}\colon \mathcal{H} \to \Real$, $\tilde{y}\colon \mathcal{H} \to \mathcal{\mathcal{V}}$ as +\begin{equation*} +\E^{\hat{h}, \pi, t} [\tilde{x} \mid \tilde{y} ](h) +:= \E \left[ \tilde{x} \mid \tilde{y} = \tilde{y}(h) \right](h), \quad\forall h\in \mathcal{H}(\hat{h}, t). +\end{equation*} +In the $\E$ operator above, the random variables $\tilde{x}$ and $\tilde{h}$ live in a probability space $(\Omega, p)$ where $\Omega = \mathcal{H}(\hat{h}, t)$ and $p(h) = p^{\mathrm{h}}(h, \pi), \forall h\in \Omega$. +Moreover, if $\hat{h}$ is a state, then it is interpreted as a history with the single initial state. +%%\lean{MDPs.expect_h_cnd_rv} \leanok +\cc{The right-hand side $\E[\tilde{x}\mid \tilde{y}=\tilde{y}(h)](h)$ applies a scalar to $(h)$; per \cref{def:expect-cnd-rv} it should read $\E[\tilde{x}\mid\tilde{y}](h)$. Also, $\tilde{y}$ is allowed an arbitrary codomain $\mathcal{V}$ whereas \cref{def:expect-cnd-rv} requires $\Nats_m$-valued conditioning variables, and the last sentence refers to ``$\tilde{x}$ and $\tilde{h}$'' where $\tilde{y}$ is meant.} +\end{definition} + + +\subsection{Basic Properties} + + +\begin{theorem} \label{thm:exph-congr} +Assume $\tilde{x} \colon \mathcal{H} \to \Real$ and $c \in \Real$. Then $\forall h\in \mathcal{H}, \pi\in \PiHR, t \in \Nats$: +\[ + \E^{\hat{h}, \pi, t} \left[ c + \tilde{x} \right] + = + c + \E^{\hat{h}, \pi, t} \left[ \tilde{x} \right]. +\] +%%\lean{MDPs.exph_congr} \leanok +\cc{The quantifier ranges over $h\in \mathcal{H}$ but the statement uses $\hat{h}$, which is unbound. The same mismatch recurs in \cref{thm:exph-add-rv,thm:exph-const,thm:exph-add-const}.} +\end{theorem} +\begin{proof} +Directly from \cref{thm:exp-linear,thm:exp-const}. +\end{proof} + +\begin{theorem} \label{thm:exph-add-rv} +Suppose that $\tilde{x}, \tilde{y} \colon \mathcal{H} \to \Real $. Then $\forall h\in \mathcal{H}, \pi\in \PiHR, t \in \Nats$: +\[ + \E^{\hat{h}, \pi, t} \left[ \tilde{x} + \tilde{y} \right] + = + \E^{\hat{h}, \pi, t} \left[ \tilde{x} \right] + \E^{\hat{h}, \pi, t} \left[ \tilde{y} \right]. +\] +%%\lean{MDPs.exph_add_rv} +\end{theorem} +\begin{proof} + From \cref{thm:exp-linear}. +\end{proof} + +\begin{theorem} \label{thm:exph-const} +Suppose that $c\in \Real$. Then $\forall h\in \mathcal{H}, \pi\in \PiHR, t \in \Nats$: +\[ + \E^{\hat{h}, \pi, t} \left[ c \right] = c. +\] +%%\lean{MDPs.exph_const} +\end{theorem} +\begin{proof} + From \cref{thm:exp-const}. +\end{proof} + +\begin{theorem} \label{thm:exph-add-const} +Suppose that $\tilde{x}, \tilde{y} \colon \mathcal{H} \to \Real $ satisfy that $\tilde{x}(h) = \tilde{y}(h), \forall h \in \mathcal{H}$. Then $\forall h\in \mathcal{H}, \pi\in \PiHR, t \in \Nats$: +\[ + \E^{\hat{h}, \pi, t} \left[ \tilde{x} \right] + = + c + \E^{\hat{h}, \pi, t} \left[ \tilde{y} \right]. +\] +%%\lean{MDPs.exph_add_const} +\cc{The statement is malformed: the hypothesis says $\tilde{x}$ and $\tilde{y}$ are \emph{equal} pointwise, yet the conclusion introduces an undeclared constant $c$. Presumably the intended hypothesis is $\tilde{x}(h) = c + \tilde{y}(h)$ for all $h$ --- in which case the result is exactly \cref{thm:exph-congr} and the two should be merged. As stated, the theorem is false (take $\tilde x = \tilde y$ nonzero and $c\ne 0$). This matters because \cref{thm:sum-rew-eq-sum-rew-rg} cites it.} +\end{theorem} +\begin{proof} +From \cref{thm:exp-linear}. +\end{proof} + +\begin{theorem}\label{thm:expret-eq-sum-rew} +For each $\hat{h}\in \mathcal{H}$, $\pi \in \PiHR$, and $t\in \Nats$: +\[ +\E^{\hat{h}, \pi, t} \left[ \rew \right] += +\E^{\hat{h}, \pi, t} \left[ \sum_{k=0}^{|\tilde{\id}|-1} r(\tilde{s}_k, \tilde{a}_k, \tilde{s}_{k+1}) \right], +\] +where $\tilde{\id}(h)$ is the identity function, $|\cdot|$ is the length of a history (0-based), $\tilde{s}_k\colon \mathcal{H} \to \mathcal{S}$ and $\tilde{a}_k\colon \mathcal{H} \to \mathcal{A}$ are the 0-based $k$-th state and action, respectively of each history. +%%\lean{MDPs.expret_eq_sum_rew} \leanok +\end{theorem} +\begin{proof} +Follows from \cref{thm:exph-congr} and the equality of the reward function $\tilde{r}^{\mathrm{h}}$ and the sum in the expectation. +\cc{The citation is not to the point: \cref{thm:exph-congr} is about adding a constant and plays no role. The content of the theorem is the pointwise identity $\rew(h) = \sum_{k=0}^{|h|-1} r(s_k(h),a_k(h),s_{k+1}(h))$, which needs a proof by induction on $|h|$ from \cref{def:reward}; that induction is not given. The statement's notation is also unclear: $|\tilde{\id}|$ is used as a random upper summation limit and $\tilde{\id}$ is described only as ``the identity function''.} +\end{proof} + +\begin{theorem}\label{thm:sum-rew-eq-sum-rew-rg} +For each $h\in \mathcal{H}$, $\pi \in \PiHR$, and $t\in \Nats$: +\[ +\E^{h, \pi, t} \left[ \rew \right] += +\rew(h) + \E^{h, \pi, t} \left[ \rew_{\ge k_0} \right], +\] +where $k_0:=|h|$. + %%\lean{MDPs.sum_rew_eq_sum_rew_rg } +\end{theorem} +\begin{proof} +Follows from \cref{thm:exph-add-const}. +\cc{\Cref{thm:exph-add-const} is malformed (see the comment there), so this proof rests on nothing. What is actually needed is (i) the decomposition $\rew = \rew_{\le k_0-1} + \rew_{\ge k_0}$, and (ii) that $\rew_{\le k_0-1}$ is \emph{constant} equal to $\rew(h)$ on $\mathcal{H}(h,t)$ --- both unproved, and both sensitive to the unspecified indexing convention of \cref{reward_to,reward_from}. Note the potential off-by-one: $\rew_{\ge k_0}$ with $k_0=|h|$ must not double-count the reward already included in $\rew(h)$.} +\end{proof} + +\begin{theorem}\label{thm:sum-rew-cnd} +For each $\hat{h}\in \mathcal{H}$, $\pi \in \PiHR$, $t\in \Nats$, $h\in \mathcal{H}$: +\begin{equation*} + \PP^{\hat{h},\pi,t}[\tilde{s}_{k_0}=\tilde{s}_{k_0}(\omega) \wedge \tilde{a}_{k_0} = \tilde{a}_{k_0}(\omega) ] > 0 +\quad\Rightarrow \quad + \E^{\hat{h}, \pi, t} \left[ \rew_{k_0} \mid \tilde{s}_{k_0}, \tilde{a}_{k_0}\right](h) = \rew_{k_0}(h), \forall h \in \mathcal{H}. +\end{equation*} +where $k_0:= |\hat{h}|$. +%%\lean{MDPs.sum_rew_eq_sum_rew_rg } +\end{theorem} +\begin{proof} + From \cref{thm:exp-const}. +\cc{The statement is malformed and, on the natural reading, false. (i) $\omega$ is free --- it appears only inside the hypothesis and is never quantified; presumably the hypothesis should be attached to each $h$, i.e.\ $\PP^{\hat h,\pi,t}[\tilde{s}_{k_0}=s_{k_0}(h)\wedge \tilde{a}_{k_0}=a_{k_0}(h)]>0$. (ii) $h$ is bound twice (declared in the preamble and again by ``$\forall h\in\mathcal{H}$''), and should be restricted to $\mathcal{H}(\hat{h},t)$. (iii) More seriously, $\rew_{k_0}$ is the reward $r(s_{k_0},a_{k_0},s_{k_0+1})$, which also depends on $\tilde{s}_{k_0+1}$; conditioning on $(\tilde{s}_{k_0},\tilde{a}_{k_0})$ alone therefore does \emph{not} determine it, so it is not a constant on the conditioning event and \cref{thm:exp-const} does not apply. (iv) Correspondingly, \cref{thm:dph-correct-vf} invokes this theorem with conditioning on $(\tilde{a}_{k_0},\tilde{s}_{k_0+1})$, not $(\tilde{s}_{k_0},\tilde{a}_{k_0})$ --- the statement should be changed to that pair (for which it is true, since $\tilde{s}_{k_0}$ is fixed at $s_{\mathrm{last}}(\hat{h})$ when $k_0=|\hat{h}|$).} +\end{proof} + +\begin{theorem} \label{thm:exph-horizon-trim} +Assume $h \in \mathcal{H}$ and $f \colon \mathcal{H} \to \Real$ such that $s_0 := s_{|h|}(h)$ +\[ +f(\left< h, a, s \right>) = f(\left< s_0, a, a \right>) , \forall a\in \mathcal{A}, s\in \mathcal{S}. +\] +Then +\[ +\E^{h,\pi,1}\left[ \tilde{f} \right] = +\E^{s_0,\pi,1}\left[ \tilde{f} \right]. +\] +\end{theorem} +%%\lean{MDPs.exph_horizon_trim } +\begin{proof} + Directly from the definition of the expectation. +\cc{The statement is false as written for $\pi\in \PiHR$: $\E^{h,\pi,1}$ uses $\pi(h)$ whereas $\E^{s_0,\pi,1}$ uses $\pi(\langle s_0\rangle)$, and a history-dependent policy may choose differently at these two histories. The hypothesis constrains only $f$, not $\pi$ (which is in fact never quantified in the statement). Every use of this theorem later is with a fixed action in place of $\pi$, so it should be restated for $\E^{h,a,1}$. Separately, the hypothesis contains a typo: $f(\langle s_0,a,a\rangle)$ should be $f(\langle \langle s_0\rangle, a, s\rangle)$.} +\end{proof} + +\subsection{Total Expectation} + +\begin{theorem}[Total Expectation] \label{thm:total-expectation-h} + For each $h\in \mathcal{H}$, $\pi\in \PiHR$, $t\in \Nats$, $\tilde{x}\colon \mathcal{H} \to \Real$ and $\tilde{y}\colon \mathcal{H} \to \mathcal{V}$: + \[ + \E^{h, \pi, t} \left[ \E^{h, \pi, t} \left[ \tilde{x} \mid \tilde{y} \right] \right] + = + \E^{h, \pi, t} \left[ \tilde{x} \right]. + \] +%%\lean{MDPs.total_expectation_h} +\end{theorem} +\begin{proof} +From \cref{thm:total-expectation}. +\end{proof} + +\begin{theorem}\label{thm:exp-horizon-cut} +Suppose that the random variable $\tilde{x}\colon \mathcal{H} \to \Real$ satisfies for some $k, t \in \Nats$, with $k \le t$, that + \[ + \tilde{x}(h) = \tilde{x}(h_{\le k}) , \forall h\in \mathcal{H}, + \] + where $h_{\le k}$ is the prefix of $h$ of length $k$. Then for each $h\in \mathcal{H}, \pi\in \PiHR$: + \[ + \E^{h,\pi,t} \left[ \tilde{x} \right] = \E^{h,\pi,k} \left[ \tilde{x} \right]. + \] + %%%\lean{MDPs.exp_horizon_cut} + \cc{No proof is given, and this theorem carries real weight (it is the step that collapses horizon $t+1$ to horizon $1$ in \cref{thm:dph-correct-vf}). Note also that the hypothesis is phrased with \emph{absolute} prefix length $k$, whereas the application in \cref{thm:dph-correct-vf} needs it relative to $|h|$; as stated the hypothesis is not satisfied there.} +\end{theorem} + + +\subsection{Conditional Properties} + +\begin{theorem} \label{thm:exph-cond-eq-hist} +For each $h\in \mathcal{H}$, $\pi\in \PiHR$, $t\in \Nats$, $\tilde{x}\colon \mathcal{H} \to \Real$, $s\in \mathcal{S}$, $a\in \mathcal{A}$: +\[ + \E^{h,\pi,t+1}[\tilde{x} \mid \tilde{a}_{|h|} = a, \tilde{s}_{|h|+1} = s] + = + \E^{\langle h, a, s\rangle,\pi,t}[\tilde{x}], +\] +when $\mathbb{P} [\tilde{a}_{|h|} = a, \tilde{s}_{|h|+1} = s] > 0$. +%%\lean{MDPs.exph_cond_eq_hist} +\end{theorem} +\begin{proof} + Let + \[ + \begin{aligned} + b &:= \mathbb{P}^{h,\pi,t+1} \left[ \tilde{a}_{|h|} = a, \tilde{s}_{|h|+1} = s \right] \\ + &= \hat{p}_{h,\pi}(h') \cdot \pi(h', a) \cdot p(s_{|h'|}(h'), a, s) \\ + &> 0 + \end{aligned} + \] + where the inequality holds from the hypothesis. Also, let + \[ + \mathbb{B} := \left\{ h'\in \Omega_{h,t+1} \mid a_{|h|}(h') = a \wedge s_{|h|+1}(h') = s\right\}. + \] + Note that + \begin{equation} \label{eq:set-equivalence} + \mathbb{B} = \Omega_{\left< h', a, s \right>, t}, +\end{equation} + which can be seen by algebraic manipulation from \eqref{eq:omega-def}. + \cc{Notation collision and errors: $h'$ is used simultaneously as a free symbol in the displayed formula for $b$, as the bound summation index, and inside $\Omega_{\langle h',a,s\rangle,t}$ --- the latter should be $\Omega_{\langle h,a,s\rangle,t}$ in all three occurrences, and the formula for $b$ should read $b = \pi(h,a)\cdot p(s_{|h|}(h),a,s)$ (using $\hat{p}_{h,\pi}(h)=1$). Also, \eqref{eq:set-equivalence} is asserted ``by algebraic manipulation'' with no argument, and the third step of the display --- replacing $\frac{1}{b}\hat{p}_{h,\pi}$ by $\hat{p}_{\langle h,a,s\rangle,\pi}$ --- requires the factorization $\hat{p}_{h,\pi}(h') = b\cdot \hat{p}_{\langle h,a,s\rangle,\pi}(h')$ for all $h'$ in the set, which needs an induction on $t$ rather than a direct appeal to \eqref{eq:phat-def}. Finally, the conditioning $\E[\cdot\mid \tilde{a}_{|h|}=a,\tilde{s}_{|h|+1}=s]$ on a \emph{pair} of variables is not covered by \cref{def:expect-h-cnd}, which conditions on a single Boolean variable.} + + Using the notation above, we can show the result as + \begin{align*} + \E^{h,\pi,t+1}[\tilde{x} \mid \tilde{a}_{|h|} = a, \tilde{s}_{|h|+1} = s] + &= \frac{1}{b} \sum_{h'\in \mathbb{B}} \hat{p}_{h,\pi}(h') \cdot x(h') + && \why{definition}\\ + &= \frac{1}{b} \sum_{h'\in \Omega_{\left< h', a, s \right>, t}} \hat{p}_{h,\pi}(h') \cdot x(h') + && \why{\cref{eq:set-equivalence}} \\ + &= \sum_{h'\in \Omega_{\left< h', a, s \right>, t}} \hat{p}_{\left,\pi}(h') \cdot x(h') + && \why{\cref{eq:phat-def}} \\ + &= \E^{\langle h, a, s\rangle,\pi,t}[\tilde{x}]. && \why{definition} + \end{align*} +\end{proof} + +\section{Dynamic Program: History-Dependent Finite Horizon} + +In this section, we derive dynamic programming equations for histories. We assume an MDP $M = (\mathcal{S}, \mathcal{A}, p, r)$ throughout this section. + +The main idea of the proof is to: +\begin{enumerate} +\item Derive (exponential-size) dynamic programming equations for the history-dependent value function of history-dependent policies + \begin{enumerate} + \item Define the value function + \item Define an optimal value function + \end{enumerate} +\item Show that value functions decompose to equivalence classes +\item Show that the value function for the equivalence classes can be computed efficiently +\end{enumerate} + +\subsection{Definitions} + +\begin{definition} \label{def:ObjectiveFH} + A finite horizon objective definition is given by $O := (s_0, T)$ where $s_0\in \mathcal{S}$ is the initial state and $T\in \Nats$ is the horizon. + %%\lean{MDPs.ObjectiveFH} \leanok +\end{definition} + +In the reminder of the section, we assume an objective $O = (s_0, T)$. +\begin{definition} \label{sec:objective-fh} + The \emph{finite horizon objective function} for and objective $O$ is $\pi\in \PiHR$ is defined as + \[ + \rho(\pi, O) + := + \E^{s_0, \pi, T} \left[\rew \right]. + \] + %%\lean{MDPs.objective_fh} \leanok +\end{definition} + +\begin{definition}\label{def:return-optimal-fh} + A policy $\pi\opt\in \PiHR$ is \emph{return optimal} for an objective $O$ if + \[ + \rho(\pi\opt, O) \ge \rho(\pi, O), \quad \forall \pi \in \PiHR. + \] + %%\lean{MDPs.OptimalVF_fh} \leanok +\end{definition} + +\begin{definition} \label{def:values-h} + The \emph{set of history-dependent value functions} $\mathcal{U}$ is defined as + \[ + \mathcal{U} := \Real^{\mathcal{H}}. + \] + %%\lean{MDPs.ValuesH} \leanok +\end{definition} + +\begin{definition} \label{def:u-pi} +A \emph{history-dependent policy value function} $\hat{u}^{\pi}_t\colon \mathcal{H} \to \Real$ for each $h\in \mathcal{H}$, $\pi \in \PiHR$, and $t\in \Nats$ is defined as + \[ + \hat{u}^{\pi}_t(h) := \E^{h, \pi, t} \left[ \rew_{\ge |h|} \right], + \] + %%\lean{MDPs.hvalue_π} \leanok +\end{definition} + + +\begin{definition} +The optimal history-dependent value function $\hat{u}_t\opt\colon \mathcal{H} \to \Real$ is defined for a horizon $t\in \Nats$ as + \[ + \hat{u}\opt_t(h) := \sup_{\pi\in \PiHR} \hat{u}_t^{\pi}(h). + \] +\cc{This definition is never used again: nothing in the document relates $\hat{u}\opt_t$ to the DP value function $u\opt_t$ of \cref{def:u-dp-opt}. \Cref{thm:dph-opt-vf-opt} proves only the inequality $u\opt_t \ge \hat{u}^{\pi}_t$ for all $\pi$; the converse direction $u\opt_t \le \sup_{\pi}\hat{u}^{\pi}_t$ --- equivalently, the existence of a policy attaining $u\opt_t$ --- is the missing half of the optimality argument and is never proved. Note also that $\PiHR$ contains randomized policies, so $\sup$ cannot be replaced by $\max$ without an argument; the theorem after \cref{def:pi-opt} is presumably intended to supply the attainment but does not state it in this form.} +\end{definition} + +The following definition is another way of defining an optimal policy. +\begin{definition} \label{def:optimal-fh} +For each $t\in \Nats$, a policy $\pi\opt \in \PiHR$ is optimal if +\[ + \hat{u}_t^{\pi\opt}(h) \ge \hat{u}_t^{\pi}(h), \quad \forall \pi \in \PiHR, h\in \mathcal{H}. + \] + %%\lean{MDPs.Optimal_fh}\leanok +\end{definition} + +\begin{theorem} \label{def:optimalvf-imp-optimal} +A policy $\pi\opt \in \PiHR$ optimal in \cref{def:optimal-fh} is also return optimal in \cref{def:return-optimal-fh} for any initial state $s_0$ and horizon $T$. +%%\lean{MDPs.optimalvf_imp_optimal} +\cc{No proof is given. It also requires the unstated identity $\rho(\pi,O) = \hat{u}^{\pi}_T(\langle s_0\rangle)$, i.e.\ that $\E^{s_0,\pi,T}[\rew] = \E^{s_0,\pi,T}[\rew_{\ge 0}]$, which depends on the (unspecified) convention of \cref{reward_from}.} +\end{theorem} + +\subsection{History-dependent Dynamic Program} + +The following definitions of history-dependent value functions use a dynamic program formulation. + +\begin{definition} \label{def:DPhpi} +The \emph{history-dependent policy Bellman operator} $L_{\mathrm{h}}^{\pi} \colon \mathcal{U} \to \mathcal{U}$ is defined for each $\pi\in \PiHR$ as + \[ + (L_{\mathrm{h}}^{\pi} \tilde{u}) (h) + := + \E^{h, \pi, 1} \left[ \rew_{|h|} + \tilde{u} \right], \quad \forall h\in \mathcal{H}, \tilde{u} \in \mathcal{U}, + \] + where the value function $\tilde{u}$ is interpreted as a random variable on defined on the sample space $\Omega = \mathcal{H}$. + %%\lean{MDPs.DPhπ} \leanok +\end{definition} + +\begin{definition}\label{def:DPhopt} +The \emph{history-dependent optimal Bellman operator} $L_{\mathrm{h}}\opt \colon \mathcal{U} \to \mathcal{U}$ is defined as + \[ + (L\opt_{\mathrm{h}} \tilde{u}) (h) + := + \max_{a\in \mathcal{A}} \E^{h, a, 1} \left[ \rew_{|h|} + \tilde{u} \right], \quad \forall h\in \mathcal{H}, \tilde{u} \in \mathcal{U}, + \] + where the value function $\tilde{u}$ is interpreted as a random variable on defined on the sample space $\Omega = \mathcal{H}$. + %%\lean{MDPs.DPhopt}\leanok + \cc{$\E^{h,a,1}$ is undefined: \cref{def:expect-h} takes a policy $\pi\in \PiHR$ as its second superscript, and \cref{def:decision-rule} only remarks that an action can be read as a \emph{decision rule}, not as a history-dependent policy. An explicit coercion $a \mapsto \bar{\pi}_a \in \PiHR$ (and the corresponding notation) is needed, since this operator, \cref{def:DPMopt}, \cref{def:pi-opt} and \cref{def:DPMpi} all rely on it.} +\end{definition} + +\begin{definition} \label{def:u-dp-pi} + The history-dependent \emph{DP value function} $u_t^{\pi} \in \mathcal{U}$ for a policy $\pi\in \PiHR$ and $t\in \Nats$ is defined as + \[ + u_t^{\pi} := + \begin{cases} + 0 &\text{if } t = 0, \\ + L_{\mathrm{h}}^{\pi} u_{t-1}^{\pi} &\text{otherwise}. + \end{cases} + \] + %%\lean{MDPs.u_dp_π}\leanok +\end{definition} + +\begin{definition} \label{def:u-dp-opt} + The history-dependent \emph{DP value function} $u_t\opt \in \mathcal{U}$ for $t\in \Nats$ is defined as + \[ + u_t\opt := + \begin{cases} + 0 &\text{if } t = 0, \\ + L_{\mathrm{h}}\opt u_{t-1}\opt &\text{otherwise}. + \end{cases} + \] + %%\lean{MDPs.u_dp_opt}\leanok +\end{definition} + +\begin{lemma}\label{thm:dp-opt-ge-dp-pi} + Suppose that $u^1, u^2 \in \mathcal{U}$ satisfy that $u^1(h) \ge u^2(h), \forall h\in \mathcal{H}$. Then + \[ + (L_{\mathrm{h}}\opt u^1)(h) \ge + (L_{\mathrm{h}}^{\pi} u^2)(h), \quad \forall \pi\in \PiHR, h\in \mathcal{H}. + \] +\end{lemma} +%%\lean{MDPs.dp_opt_ge_dp_pi} +\begin{proof} + From \cref{thm:exp-monotone}. + \cc{Incomplete: monotonicity of $\E$ only handles the step from $u^2$ to $u^1$ inside a \emph{fixed} expectation. The remaining --- and essential --- step is that the $\pi$-average over actions is bounded by the maximum over actions, $\E_{a\sim \pi(h)}[\cdot] \le \max_{a\in\mathcal{A}}[\cdot]$. That averaging argument is not proved anywhere in the document.} +\end{proof} + +The following theorem shows the history-dependent value function can be computed by the dynamic program. The following theorem is akin to \citep[theorem~4.2.1]{Puterman2005}. +\begin{theorem} \label{thm:dph-correct-vf} +For each $\pi\in \PiHR$ and $t\in \Nats$: + \[ + \hat{u}^{\pi}_t(h) + = + u^{\pi}_t(h), \quad \forall h\in \mathcal{H}. + \] + %%\lean{MDPs.dph_correct_vf } +\end{theorem} +\begin{proof} +By induction on $t$. The base case for $t=0$ follows from the definition. The inductive case for $t + 1$ follows for each $h\in \mathcal{H}$ when $|h| = k_0$ as +{\allowdisplaybreaks +\begin{align*} +\hat{u}_{t+1}^{\pi}(h) +&= \E^{h, \pi, t+1} \left[ \rew_{\ge k_0} \right] && \why{\cref{def:u-pi}}\\ +&= \E^{h, \pi, t+1} \left[ \E^{h, \pi, t+1} \left[ \rew_{\ge k_0} \mid \tilde{a}_{k_0}, \tilde{s}_{k_0+1} \right] \right] && \why{\cref{thm:total-expectation-h}} \\ +&= \E^{h, \pi, t+1} \left[\rew_{k_0} + \E^{h, \pi, t+1} \left[ \rew_{\ge k_0+1} \mid \tilde{a}_{k_0}, \tilde{s}_{k_0+1} \right] \right] && \why{\cref{thm:sum-rew-cnd}} \\ +&= \E^{h, \pi, t+1} \left[\rew_{k_0} + \E^{\langle h,\tilde{a}_{k_0},\tilde{s}_{k_0+1}\rangle, \pi, t} \left[ \rew_{\ge k_0+1} \right]\right] && \why{\cref{thm:exph-cond-eq-hist}} \\ +&= \E^{h, \pi, t+1} \left[\rew_{k_0} + \hat{u}_t(\langle h,\tilde{a}_{k_0},\tilde{s}_{k_0+1}\rangle; \pi) \right] && \why{\cref{def:u-pi}} \\ +&= \E^{h, \pi, t+1} \left[\rew_{k_0} + u^{\pi}_t(\langle h,\tilde{a}_{k_0},\tilde{s}_{k_0+1}\rangle) \right] && \why{inductive assm} \\ +&= \E^{h, \pi, 1} \left[\rew + \tilde{u}^{\pi}_t \right] && \why{\cref{thm:exp-horizon-cut}} \\ +&= L_{\mathrm{h}}^{\pi} u_t^{\pi} && \why{\cref{def:DPhpi}} \\ +&= u^{\pi}_t(h). && \why{\cref{def:u-dp-pi}} +\end{align*} +} +Also, we use $\tilde{u}_t^{\pi}$ to emphasize when we treat $u^{\pi}_t$ as a random variable. +\cc{Several gaps in this chain. (i) The final line reads $u^{\pi}_t(h)$ but must be $u^{\pi}_{t+1}(h)$ to close the induction; likewise the penultimate line is missing its argument $(h)$. (ii) Step 2 applies \cref{thm:total-expectation-h} conditioning on the \emph{pair} $(\tilde{a}_{k_0},\tilde{s}_{k_0+1})$, whereas that theorem (and \cref{def:expect-h-cnd-rv}) is stated for a single conditioning variable. (iii) Step 3 cites \cref{thm:sum-rew-cnd}, which is stated for the pair $(\tilde{s}_{k_0},\tilde{a}_{k_0})$ --- a different pair (see the comment there) --- and additionally uses linearity of \emph{conditional} expectation, which is never established. (iv) Step 4 substitutes random variables $\tilde{a}_{k_0},\tilde{s}_{k_0+1}$ into the superscript of $\E$; \cref{thm:exph-cond-eq-hist} only justifies this for fixed $a,s$ with $\PP[\cdot]>0$, so the substitution needs \cref{def:expect-h-cnd-rv} plus a check that the zero-probability branches contribute nothing --- neither is done. (v) Step 7 cites \cref{thm:exp-horizon-cut}, whose hypothesis is about absolute prefix length $k$, but here the cut is at $|h|+1$; it also silently changes $\rew_{k_0}$ to $\rew$.} +\end{proof} + +The following theorem is akin to \citep[theorem~4.3.2]{Puterman2005}. +\begin{theorem}\label{thm:dph-opt-vf-opt} +For each $t\in \Nats$: + \[ + u\opt_t(h) + \ge + \hat{u}_t(h; \pi), \quad \forall h\in \mathcal{H}, \pi\in \PiHR. + \] + %%\lean{MDPs.dph_opt_vf_opt} +\end{theorem} +\begin{proof} + By induction on $t$. The base case is immediate. The inductive case follows for $t+1$ as follows. For each $\pi\in \PiHR$: + \begin{align*} + u_{t+1}\opt(h) + &= (L_{\mathrm{h}}\opt u_t\opt)(h) && \why{\cref{def:DPhopt}} \\ + &\ge (L_{\mathrm{h}}^{\pi} \hat{u}_t(\cdot; \pi))(h) && \why{ind asm, \cref{thm:dp-opt-ge-dp-pi}} \\ + &= (L_{\mathrm{h}}^{\pi} u_t^{\pi})(h) && \why{\cref{thm:dph-correct-vf}} \\ + &= u_t^{\pi}(h) && \why{\cref{def:u-pi}} \\ + &= \hat{u}_t(h; \pi). && \why{\cref{thm:dph-correct-vf}} \\ + \end{align*} +\cc{Off-by-one: the chain should end at $u^{\pi}_{t+1}(h) = \hat{u}_{t+1}(h;\pi)$, but the last two lines are written with subscript $t$. The step ``$= u^{\pi}_t(h)$'' also cites \cref{def:u-pi} where \cref{def:u-dp-pi} is meant. In addition, the base case is dismissed as ``immediate'' but $\hat{u}^{\pi}_0(h) = \E^{h,\pi,0}[\rew_{\ge|h|}]$ equals $0$ only under the reward convention that is left unspecified in \cref{reward_from}. Finally, note the notation clash: $\hat{u}_t(h;\pi)$ here versus $\hat{u}^{\pi}_t(h)$ in \cref{def:u-pi}.} +\end{proof} + +\section{Expected Dynamic Program: Markov Policy} + +\subsection{Optimality} +We discuss results needed to prove the optimality of Markov policies. + +\begin{definition} \label{def:Values} + The set of \emph{independent value functions} is defined as $\mathcal{V} := \Real^{\mathcal{S}}$. + %%\lean{MDPs.Values}\leanok +\end{definition} + +\begin{definition}\label{def:DPMopt} +A \emph{Markov Bellman operator} $L\opt\colon \mathcal{V} \to \mathcal{V}$ is defined as +\[ +(L\opt v)(h) := +\max_{a\in \mathcal{A}} \E^{h, a, 1} \left[ \rew + v(\tilde{s}_{|h|}) \right], \quad \forall \tilde{u} \in \mathcal{U}, +\] +%%\lean{MDPs.DPMopt}\leanok +\cc{Type errors: $L\opt$ maps $\mathcal{V}=\Real^{\mathcal{S}}$ to $\mathcal{V}$, so its argument should be a state $s\in \mathcal{S}$, not a history $h$; the quantifier ``$\forall \tilde{u}\in \mathcal{U}$'' quantifies a variable that does not occur and should be $\forall v\in \mathcal{V}, s\in \mathcal{S}$; and $\tilde{s}_{|h|}$ refers to the free $h$ --- for a one-step expectation from $\langle s\rangle$ the successor state is $\tilde{s}_1$.} +\end{definition} + +\begin{definition} \label{def:v-dp-opt} +The \emph{optimal value function} $v\opt_t\in \mathcal{V}, t\in \Nats$ is defined as +\[ +v\opt_t := +\begin{cases} + 0 &\text{if } t = 0 \\ + (L\opt v\opt _{t-1}) &\text{otherwise}. +\end{cases} +\] +%%\lean{MDPs.v_dp_opt}\leanok +\end{definition} + +\begin{theorem} +Suppose that $t\in \Nats $. Then: +\[ +v\opt_t(s_{|h|}(h)) = u\opt_t(h), \quad \forall h\in \mathcal{H}. +\] +%%\lean{MDPs.v_dp_opt_eq_u_opt} +\end{theorem} +\begin{proof} +By induction on $t$. The base case follows immediately from the definition. The inductive step for $t+1$ follows as: +\begin{align*} + u\opt_{t+1}(h) + &= \max_{a\in \mathcal{A}} \E^{h, a, 1} \left[ \rew_{|h|} + \tilde{u}\opt_t \right] + && \why{\cref{def:u-dp-opt}} \\ + &= \max_{a\in \mathcal{A}} \E^{h, a, 1} \left[ \rew_{|h|} + v\opt_t(\tilde{s}_l) \right] + && \why{inductive asm.} \\ + &= \max_{a\in \mathcal{A}} \E^{s_0, a, 1} \left[ \rew_{|h|} + v\opt_t(\tilde{s}_l) \right] + && \why{\cref{thm:exph-horizon-trim}} \\ + &= \max_{a\in \mathcal{A}} \E^{s_0, a, 1} \left[ \rew + v\opt_t(\tilde{s}_l) \right] + && \why{\cref{thm:exph-congr}} \\ + &= v\opt_{t+1}(s_{|h|}(h)) && \why{\cref{def:v-dp-opt}}. +\end{align*} +\cc{Problems in this chain: the theorem has no label and reuses the Lean name \texttt{v\_dp\_opt\_eq\_u\_opt} of the later theorem; the index $l$ in $\tilde{s}_l$ is never defined (it should be $|h|+1$, and $\tilde{s}_1$ after the horizon trim); the third step invokes \cref{thm:exph-horizon-trim}, which as stated is false for policies and whose hypothesis (that the integrand depends only on the last state of $h$) is not verified here; and the fourth step cites \cref{thm:exph-congr}, which is about adding a \emph{constant} and does not justify replacing $\rew_{|h|}$ by $\rew$ --- what is needed is $\rew(\langle s_0\rangle)=0$, i.e.\ the missing base case of \cref{def:reward}.} +\end{proof} + + +\begin{definition} \label{def:pi-opt} + The \emph{optimal finite-horizon policy} $\pi\opt_t, t\in \Nats$ is defined as + \[ + \pi\opt_t(k,s) := + \begin{cases} + \arg\max_{a\in \mathcal{A}} \E^{s, a, 1} \left[ \rew + v_{t-k}\opt (\tilde{s}_{|h|}) \right] + &\text{if } k \le t, \\ + a_0 &\text{otherwise}, + \end{cases} + \] + where $a_0$ is an arbitrary action. + \cc{$\arg\max$ is set-valued and may contain ties, so a choice function must be fixed for $\pi\opt_t$ to be well defined. Also $\tilde{s}_{|h|}$ uses a free $h$ (it should be $\tilde{s}_1$, the successor of $s$), $\rew$ should presumably be $\rew_0$ relative to $\langle s\rangle$, and $t-k$ must be read as truncated subtraction on $\Nats$. The subscript $t$ on $\pi\opt_t$ plays no role in the definition of the two branches other than the guard $k\le t$.} + %%\lean{MDPs.πopt}\leanok +\end{definition} + +\begin{theorem} +Assume a horizon $T\in \Nats$. Then: +\[ + v\opt_{T-|h|}(s_{|h|}(h)) = u^{\pi\opt_{T-h}}_{T-|h|}(h), \quad + \forall h\in \left\{ h\in \mathcal{H} \mid |h| \le T \right\}. +\] +%%\lean{MDPs.v_dp_opt_eq_u_opt} +\end{theorem} +\begin{proof} +Fix some $T \in \Nats$. By induction on $k$ from $k = T$ to $k=0$. The base case is immediate from the definition. We prove the inductive case for $k-1$ from $k$ as +\begin{align*} +u^{\pi\opt_T}_{T-k + 1}(h) + &= \E^{h, \pi\opt_T, 1} \left[ \rew_k + \tilde{u}^{\pi\opt_T}_{T-k} \right] + && \why{\cref{def:DPhpi}} \\ + &= \E^{h, a\opt, 1} \left[ \rew_k + \tilde{u}^{\pi\opt_T}_{T-k} \right] + && \why{???} \\ + &= \E^{h, a\opt, 1} \left[ \rew_{k} + v_{T-k}\opt(\tilde{s}_1) \right] + && \why{ind asm} \\ + &= \E^{s_0, a\opt, 1} \left[ \rew + v_{T-k}\opt(\tilde{s}_1) \right] + && \why{\cref{thm:exph-horizon-trim}} \\ + &= \max_{a\in \mathcal{A}} \E^{s_0, a, 1} \left[ \rew + v_{T-k}\opt(\tilde{s}_1) \right] + && \why{???} \\ + &= v_{T-k+1}\opt(s_0). && \why{\cref{def:v-dp-opt}} +\end{align*} +Here, $k := |h|$, $a\opt := \pi\opt_T(k, s_0)$, and $s_0 := s_{|h|}(h)$ +\cc{This proof has two steps explicitly marked ``???'', i.e.\ unproved, and they are the substantive ones: (i) replacing $\pi\opt_T$ by the single action $a\opt$ requires a lemma that $\E^{h,\bar{\pi},1} = \E^{h,a,1}$ when $\bar{\pi}$ puts a Dirac mass on $a$ at $h$ (see the comment on \cref{def:DPhopt}); (ii) the step $\E^{s_0,a\opt,1}[\cdot] = \max_{a}\E^{s_0,a,1}[\cdot]$ is precisely the defining property of the $\arg\max$ in \cref{def:pi-opt} and needs to be stated as a lemma. Further problems: the theorem statement writes $\pi\opt_{T-h}$, subtracting a history from a number; the induction is described as running downward from $k=T$ to $k=0$ but the displayed step derives the case $T-k+1$ from $T-k$, i.e.\ it is an upward induction on $T-k$, and no base case is verified; the first line cites \cref{def:DPhpi} where \cref{def:u-dp-pi} is also needed; and $\tilde{s}_1$ here is written $\tilde{s}_{|h|}$ in \cref{def:pi-opt}. Finally, this theorem never gets combined with \cref{thm:dph-opt-vf-opt} into the conclusion the section is aiming at --- that the Markov deterministic policy $\pi\opt$ is optimal within $\PiHR$; that corollary is missing.} +\end{proof} + +\subsection{Evaluation} +We discuss results pertinent to the evaluation of Markov policies. + +Markov value functions depend on the length of the history. +\begin{definition} \label{def:ValuesM} + The set of \emph{independent value functions} is defined as $\mathcal{V}_{\mathrm{M}} := \Real^{\Nats \times \mathcal{S}}$. + %%\lean{MDPs.ValuesM}\leanok +\end{definition} + + +\begin{definition}\label{def:DPMpi} +A \emph{Markov policy Bellman operator} $L_k^{\pi}\colon \mathcal{V} \to \mathcal{V}$ for $\pi\in \Pi$ is defined as +\[ +(L^{\pi} v)(k, s) := +\max_{a\in \mathcal{A}} \E^{s, a, 1} \left[ \rew + v(k + 1, \tilde{s}_{|h|}) \right], \quad \forall v\in \mathcal{V}_{\mathrm{M}}, k\in \Nats, s\in \mathcal{S}. +\] +\cc{Error: this is meant to be the \emph{policy} Bellman operator, but it takes $\max_{a\in\mathcal{A}}$ and never uses $\pi$; it should evaluate at $a := \pi(k)(s)$ (or average over $\pi(k,s)$ for randomized policies). Additionally, the operator is declared with type $\mathcal{V}\to\mathcal{V}$ although it acts on $\mathcal{V}_{\mathrm{M}} = \Real^{\Nats\times\mathcal{S}}$; the subscript $k$ in $L^{\pi}_k$ is never used; ``$\pi\in \Pi$'' refers to an undefined set $\Pi$ (presumably $\PiMD$); and $\tilde{s}_{|h|}$ again refers to a free $h$.} +%%\lean{MDPs.DPMopt}\leanok +\end{definition} + +\begin{definition} \label{def:v-dp-pi} + The \emph{Markov policy value function} $v^{\pi}_t\in \mathcal{V}_{\mathrm{M}}, t \in \Nats$ for $\pi\in \PiMD$ is defined as + \[ + v^{\pi}_t := + \begin{cases} + 0 &\text{if } t = 0, \\ + (L^{\pi} v^{\pi}_{t-1}) &\text{otherwise}. + \end{cases} + \] + %%\lean{MDPs.v_dp_π}\leanok + \cc{The ``Evaluation'' subsection ends here with no results at all: there is no counterpart of \cref{thm:dph-correct-vf} relating $v^{\pi}_t$ to $u^{\pi}_t$ or $\hat{u}^{\pi}_t$, so nothing justifies calling $v^{\pi}_t$ the value function of $\pi$. Also, iterating $L^{\pi}$ (which already carries its own time index $k$) $t$ times leaves the relationship between the two time indices unspecified. Finally, \cref{def:Values} and \cref{def:ValuesM} are both named ``independent value functions''.} +\end{definition} + + + +\end{document} + +%%% Local Variables: +%%% mode: LaTeX +%%% TeX-master: t +%%% End: From 4365fef2762d1fb7d05fa8ac75479a4219fa5e71 Mon Sep 17 00:00:00 2001 From: Marek Petrik Date: Fri, 4 Sep 2026 16:42:13 -0400 Subject: [PATCH 9/9] CI: build library, API docs, and LaTeX paper; publish docs to GitHub Pages Replace the stale leanblueprint workflow (which built a nonexistent Probability target against a missing blueprint/ directory) with a simple workflow that builds MDPLib, its doc-gen4 documentation, and latex/main.tex, then deploys the docs and paper to GitHub Pages via the standard actions. Link the published documentation from the README. --- .github/pages-index.html | 13 ++++ .github/workflows/blueprint.yml | 128 -------------------------------- .github/workflows/ci.yml | 69 +++++++++++++++++ README.md | 6 +- 4 files changed, 87 insertions(+), 129 deletions(-) create mode 100644 .github/pages-index.html delete mode 100644 .github/workflows/blueprint.yml create mode 100644 .github/workflows/ci.yml diff --git a/.github/pages-index.html b/.github/pages-index.html new file mode 100644 index 0000000..4b8f168 --- /dev/null +++ b/.github/pages-index.html @@ -0,0 +1,13 @@ + + +MDPLib + +

MDPLib

+

Formalizing Markov Decision Processes in Lean.

+ diff --git a/.github/workflows/blueprint.yml b/.github/workflows/blueprint.yml deleted file mode 100644 index 4f2ffdc..0000000 --- a/.github/workflows/blueprint.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: Compile blueprint - -on: - push: - branches: - - main # Trigger on pushes to the default branch - workflow_dispatch: # Allow manual triggering of the workflow from the GitHub Actions interface - -# Cancel previous runs if a new commit is pushed to the same PR or branch -concurrency: - group: ${{ github.ref }} # Group runs by the ref (branch or PR) - cancel-in-progress: true # Cancel any ongoing runs in the same group - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read # Read access to repository contents - pages: write # Write access to GitHub Pages - id-token: write # Write access to ID tokens - -jobs: - build_project: - runs-on: ubuntu-latest - name: Build project - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history for all branches and tags - - - name: Install elan - run: curl https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh -sSf | sh -s -- -y - - - name: Get Mathlib cache - run: ~/.elan/bin/lake exe cache get || true - - - name: Build project - run: ~/.elan/bin/lake build Probability - - - name: Cache API docs - uses: actions/cache@v4 - with: - path: | - .lake/build/doc/Aesop - .lake/build/doc/Batteries - .lake/build/doc/find - .lake/build/doc/Init - .lake/build/doc/Lake - .lake/build/doc/Lean - .lake/build/doc/Mathlib - .lake/build/doc/Std - key: Docs-${{ hashFiles('lake-manifest.json') }} - - - name: Build project API documentation - run: ~/.elan/bin/lake -R -Kenv=dev build Probability:docs - - - name: Check for `home_page` folder # this is meant to detect a Jekyll-based website - id: check_home_page - run: | - if [ -d home_page ]; then - echo "The 'home_page' folder exists in the repository." - echo "HOME_PAGE_EXISTS=true" >> $GITHUB_ENV - else - echo "The 'home_page' folder does not exist in the repository." - echo "HOME_PAGE_EXISTS=false" >> $GITHUB_ENV - fi - - - name: Build blueprint and copy to `home_page/blueprint` - uses: xu-cheng/texlive-action@v2 - with: - docker_image: ghcr.io/xu-cheng/texlive-full:20250101 - run: | - # Install necessary dependencies and build the blueprint - apk update - apk add --update make py3-pip git pkgconfig graphviz graphviz-dev gcc musl-dev - git config --global --add safe.directory $GITHUB_WORKSPACE - git config --global --add safe.directory `pwd` - python3 -m venv env - source env/bin/activate - pip install --upgrade pip requests wheel - pip install pygraphviz \ - --config-settings="--global-option=build_ext" \ - --config-settings="--global-option=-L/usr/lib/graphviz/" \ - --config-settings="--global-option=-R/usr/lib/graphviz/" - pip install leanblueprint - leanblueprint pdf - mkdir -p home_page - cp blueprint/print/print.pdf home_page/blueprint.pdf - leanblueprint web - cp -r blueprint/web home_page/blueprint - - - name: Check declarations mentioned in the blueprint exist in Lean code - run: | - ~/.elan/bin/lake exe checkdecls blueprint/lean_decls - - - name: Copy API documentation to `home_page/docs` - run: cp -r .lake/build/doc home_page/docs - - - name: Remove unnecessary lake files from documentation in `home_page/docs` - run: | - find home_page/docs -name "*.trace" -delete - find home_page/docs -name "*.hash" -delete - - - name: Bundle dependencies - uses: ruby/setup-ruby@v1 - with: - working-directory: home_page - ruby-version: "3.0" # Specify Ruby version - bundler-cache: true # Enable caching for bundler - - - name: Build website using Jekyll - if: env.HOME_PAGE_EXISTS == 'true' - working-directory: home_page - env: - JEKYLL_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: JEKYLL_ENV=production bundle exec jekyll build # Note this will also copy the blueprint and API doc into home_page/_site - - - name: "Upload website (API documentation, blueprint and any home page)" - uses: actions/upload-pages-artifact@v3 - with: - path: ${{ env.HOME_PAGE_EXISTS == 'true' && 'home_page/_site' || 'home_page/' }} - - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 - - - name: Make sure the API documentation cache works - run: mv home_page/docs .lake/build/doc - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1a8996c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,69 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pages: write + id-token: write + +jobs: + build: + name: Build library, docs and paper + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build the Lean library + uses: leanprover/lean-action@v1 + with: + build-args: "MDPLib" + + - name: Build the API documentation + run: lake -R build MDPLib:docs + + - name: Build the LaTeX paper + uses: xu-cheng/latex-action@v3 + with: + root_file: main.tex + working_directory: latex + + - name: Assemble the website + run: | + mkdir -p _site + cp -r .lake/build/doc _site/docs + find _site/docs \( -name '*.trace' -o -name '*.hash' \) -delete + cp latex/main.pdf _site/main.pdf + cp .github/pages-index.html _site/index.html + + - name: Upload the paper as a build artifact + uses: actions/upload-artifact@v4 + with: + name: paper + path: latex/main.pdf + + - name: Upload the GitHub Pages artifact + if: github.ref == 'refs/heads/main' + uses: actions/upload-pages-artifact@v3 + with: + path: _site + + deploy: + name: Deploy documentation to GitHub Pages + needs: build + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/README.md b/README.md index 810d339..4ec7d94 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Formalizing Markov Decision Processes in Lean -[![Blueprint](https://img.shields.io/badge/Blueprint-WIP-blue)](https://formalproofs.github.io/MDPLib) +[![Documentation](https://img.shields.io/badge/docs-online-blue)](https://formalproofs.github.io/MDPLib/) + +**Documentation:** the [API documentation](https://formalproofs.github.io/MDPLib/docs/) +and the [companion paper](https://formalproofs.github.io/MDPLib/main.pdf) are built by CI and +published to [GitHub Pages](https://formalproofs.github.io/MDPLib/) on every push to `main`. Verified Lean algorithms for solving tabular MDPs and proving their properties. The focus of this project is on two main goals: