diff --git a/.github/workflows/bar.yml b/.github/workflows/bar.yml new file mode 100644 index 0000000..32457ed --- /dev/null +++ b/.github/workflows/bar.yml @@ -0,0 +1,21 @@ +name: verified bar + +on: + push: + pull_request: + # Rerun when the spec reviewer approves, or an approval is dismissed. + pull_request_review: + types: [submitted, dismissed] + +jobs: + bar: + # pull-requests: read lets the check confirm the spec approval on GitHub. + permissions: + contents: read + pull-requests: read + # Both lines pin the same commit of Proven-Commons/mission: bar v2, as + # proposed in its pull request #2. Move both to a release tag once + # mission publishes one. + uses: Proven-Commons/mission/.github/workflows/verified-bar.yml@6ba85c27abaaf07905c9316aa96dc29f7b5847a2 + with: + bar-ref: 6ba85c27abaaf07905c9316aa96dc29f7b5847a2 diff --git a/.gitignore b/.gitignore index 01f8cdb..ae73679 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .lake/ +__pycache__/ diff --git a/Main.lean b/Main.lean new file mode 100644 index 0000000..b237426 --- /dev/null +++ b/Main.lean @@ -0,0 +1,201 @@ +/- + Command line front end. + + sqlite dump FILE + + reads a database file and prints, one per line: + + header F1 ... F18 its header, as `header` gives it; + page N PAGE, page N FAIL each whole page of the file in turn, N from + 1 to the file's length divided by the page + size, as `page` gives it, or FAIL when it is + not a b-tree page this reader accepts; + record N P ROWID VALUES, after a page, for each cell whose payload is + record N P ROWID FAIL all on its page and not empty, in the order + of the cell pointers: P is the cell's + pointer, ROWID its rowid (`-` on an index + page), and VALUES the record the payload + holds, as `record` gives it, or FAIL when it + is not a record. An empty payload holds no + record and gets no line. + + It exits with 0. When the file does not start with a header this + reader accepts, it prints nothing on standard output and exits with + 1; on wrong arguments or a file it cannot read, it prints nothing on + standard output and exits with 2. + + sqlite serve + + answers requests on standard input, one per line, for the + differential tests. Bytes are hex digits, two per byte, and an empty + field is written `-`. Numbers are decimal. + + varint HEX prints OK, the value, the length and the hex of the + rest, or FAIL (readVarint) + header HEX prints OK and the 18 fields, or FAIL (readHeader) + page N HEX prints OK and page N of the file HEX, read with the + file's own header, or FAIL (readPage) + record HEX prints OK and the values, or FAIL (readRecord) + local T U P prints OK and how many bytes of a payload of P + bytes stay on a page of type T (2, 5, 10 + or 13) and usable size U (localOf) + + A page is written as its type, first freeblock, cell count, content + start, fragmented bytes and right-most pointer (`-` on a leaf page), + then `|`, then its cells, separated by spaces. A cell is written as + its pointer, left child, payload size, rowid, payload hex and + overflow page, separated by `/`, with `-` for a field the cell does + not have. A value is written `null`, `int:Z`, `real:BITS` (the 64 + bits as a decimal number), `blob:HEX` or `text:HEX`. + + A malformed request prints `error`: an unknown operation, the wrong + number of fields, hex digits that are not in pairs or not lower case, + or a number that is not decimal. A request line may end in LF or + CR LF. + + Every answer comes from the proved functions in `Sqlite/`. The hex + coding, the output and the request handling here are covered by the + tests, not by a proof. +-/ + +import Sqlite + +open Sqlite Spec + +def hexValue (c : Char) : Option Nat := + if '0' ≤ c ∧ c ≤ '9' then some (c.toNat - '0'.toNat) + else if 'a' ≤ c ∧ c ≤ 'f' then some (c.toNat - 'a'.toNat + 10) + else none + +/-- Bytes from hex digits in pairs. Tail recursive, so that long inputs + do not use the stack. -/ +def hexBytes (cs : List Char) (acc : Array UInt8) : Option (Array UInt8) := + match cs with + | [] => some acc + | a :: b :: rest => + match hexValue a, hexValue b with + | some hi, some lo => hexBytes rest (acc.push (hi * 16 + lo).toUInt8) + | _, _ => none + | [_] => none + +/-- The bytes a field gives: `-` for none, else hex digits in pairs. -/ +def fromHex (f : String) : Option (List UInt8) := + if f = "-" then some [] else if f.isEmpty then none else (hexBytes f.toList #[]).map (·.toList) + +def hexDigit (n : Nat) : Char := + if n < 10 then Char.ofNat ('0'.toNat + n) else Char.ofNat ('a'.toNat + n - 10) + +/-- A field for a list of bytes: `-` when it is empty. -/ +def toHex (bs : List UInt8) : String := + if bs.isEmpty then "-" + else String.ofList (bs.flatMap fun b => [hexDigit (b.toNat / 16), hexDigit (b.toNat % 16)]) + +/-- A decimal number, without sign or leading `+`. -/ +def decimal (s : String) : Option Nat := + if !s.isEmpty && s.all Char.isDigit then s.toNat? else none + +def opt {α} [ToString α] : Option α → String + | some a => toString a + | none => "-" + +def showHeader (h : Header) : String := + " ".intercalate (List.map toString [h.pageSize, h.writeVersion, h.readVersion, h.reserved, + h.changeCounter, h.databaseSize, h.freelistTrunk, h.freelistCount, h.schemaCookie, + h.schemaFormat, h.cacheSize, h.largestRoot, h.textEncoding, h.userVersion, + h.incrementalVacuum, h.applicationId, h.versionValidFor, h.sqliteVersion]) + +def showCell (p : Nat) (c : Cell) : String := + s!"{p}/{opt c.leftChild}/{opt c.payloadSize}/{opt c.rowid}/{toHex c.payload}/{opt c.overflow}" + +def showPage (pg : Page) : String := + let ph := pg.header + let cells := (pg.pointers.zip pg.cells).map fun (p, c) => showCell p c + s!"{ph.type.code} {ph.firstFreeblock} {ph.cellCount} {ph.contentStart} {ph.fragmented} " ++ + s!"{opt ph.rightmost} |" ++ String.join (cells.map (" " ++ ·)) + +def showValue : Value → String + | .null => "null" + | .int z => s!"int:{z}" + | .real b => s!"real:{b}" + | .blob bs => s!"blob:{toHex bs}" + | .text bs => s!"text:{toHex bs}" + +def showValues (vs : List Value) : String := " ".intercalate (vs.map showValue) + +/-- The answer to one request line, or `none` when it is malformed. -/ +def request (line : String) : Option String := + match line.splitOn " " with + | ["varint", h] => do + let b ← fromHex h + pure (match readVarint b with + | some (v, k, rest) => s!"OK {v} {k} {toHex rest}" + | none => "FAIL") + | ["header", h] => do + let b ← fromHex h + pure (match readHeader b with + | some hd => s!"OK {showHeader hd}" + | none => "FAIL") + | ["page", n, h] => do + let n ← decimal n + let b ← fromHex h + pure (match readHeader b >>= fun hd => readPage hd b n with + | some pg => s!"OK {showPage pg}" + | none => "FAIL") + | ["record", h] => do + let b ← fromHex h + pure (match readRecord b with + | some vs => s!"OK {showValues vs}" + | none => "FAIL") + | ["local", t, u, p] => do + let t ← (decimal t).bind pageType? + let u ← decimal u + let p ← decimal p + pure s!"OK {localOf t u p}" + | _ => none + +/-- A request line without its line ending, LF or CR LF. -/ +def chomp (line : String) : String := + let cs := line.toList + if cs.getLast? = some '\n' then + let cs := cs.dropLast + String.ofList (if cs.getLast? = some '\r' then cs.dropLast else cs) + else line + +def serve : IO Unit := do + let stdin ← IO.getStdin + let stdout ← IO.getStdout + repeat + let line ← stdin.getLine + if line.isEmpty then break + stdout.putStrLn ((request (chomp line)).getD "error") + stdout.flush + +/-- The header, then each page and the records of its cells. -/ +def dump (file : List UInt8) : IO UInt32 := do + match readHeader file with + | none => IO.eprintln "not a database header this reader accepts"; return 1 + | some h => + IO.println s!"header {showHeader h}" + for i in [0:file.length / h.pageSize] do + let n := i + 1 + match readPage h file n with + | none => IO.println s!"page {n} FAIL" + | some pg => + IO.println s!"page {n} {showPage pg}" + for (p, c) in pg.pointers.zip pg.cells do + if c.overflow.isNone && !c.payload.isEmpty then + match readRecord c.payload with + | some vs => IO.println s!"record {n} {p} {opt c.rowid} {showValues vs}" + | none => IO.println s!"record {n} {p} {opt c.rowid} FAIL" + return 0 + +def usage : String := "usage: sqlite dump FILE\n sqlite serve" + +def main (args : List String) : IO UInt32 := do + match args with + | ["dump", path] => + match ← (IO.FS.readBinFile path).toBaseIO with + | .error e => IO.eprintln s!"cannot read {path}: {e}"; return 2 + | .ok bytes => dump bytes.toList + | ["serve"] => serve; return 0 + | _ => IO.eprintln usage; return 2 diff --git a/Proofs.lean b/Proofs.lean new file mode 100644 index 0000000..27ea521 --- /dev/null +++ b/Proofs.lean @@ -0,0 +1,6 @@ +import Proofs.Bytes +import Proofs.Varint +import Proofs.Header +import Proofs.Record +import Proofs.Page +import Proofs.Main diff --git a/Proofs/Bytes.lean b/Proofs/Bytes.lean new file mode 100644 index 0000000..547c5a4 --- /dev/null +++ b/Proofs/Bytes.lean @@ -0,0 +1,144 @@ +import Sqlite + +/-! + Fixed-width fields: `take?`, big-endian integers, and `readBE`. +-/ + +namespace SqliteProofs + +open Spec Sqlite + +theorem take?_eq (w : Nat) (b c r : List UInt8) : + take? w b = some (c, r) ↔ c.length = w ∧ b = c ++ r := by + unfold take? + dsimp only + constructor + · intro h + split at h + · rename_i hl + simp only [Option.some.injEq, Prod.mk.injEq] at h + obtain ⟨rfl, rfl⟩ := h + exact ⟨hl, (List.take_append_drop w b).symm⟩ + · exact absurd h (by simp) + · rintro ⟨hl, rfl⟩ + subst hl + simp + +theorem take?_append (c r : List UInt8) : take? c.length (c ++ r) = some (c, r) := + (take?_eq _ _ _ _).mpr ⟨rfl, rfl⟩ + +theorem bigEndian_eq (bs : List UInt8) : bigEndian bs = beNat bs := rfl + +theorem beBytes_length (w n : Nat) : (beBytes w n).length = w := by + simp [beBytes] + +theorem foldl_be (acc : Nat) (bs : List UInt8) : + bs.foldl (fun n b => n * 256 + b.toNat) acc = + acc * 256 ^ bs.length + bs.foldl (fun n b => n * 256 + b.toNat) 0 := by + induction bs generalizing acc with + | nil => simp + | cons b bs ih => + simp only [List.foldl_cons, List.length_cons] + rw [ih, ih (0 * 256 + b.toNat)] + simp [Nat.pow_succ, Nat.mul_add, Nat.mul_assoc, Nat.add_assoc, Nat.mul_comm] + +theorem beNat_append (a b : List UInt8) : beNat (a ++ b) = beNat a * 256 ^ b.length + beNat b := by + unfold beNat + rw [List.foldl_append, foldl_be] + +theorem beNat_cons (b : UInt8) (bs : List UInt8) : + beNat (b :: bs) = b.toNat * 256 ^ bs.length + beNat bs := by + unfold beNat + rw [List.foldl_cons, foldl_be] + simp + +theorem beNat_lt (bs : List UInt8) : beNat bs < 256 ^ bs.length := by + induction bs with + | nil => simp [beNat] + | cons b bs ih => + rw [beNat_cons, List.length_cons, Nat.pow_succ] + have hb : b.toNat ≤ 255 := by have := b.toNat_lt; omega + have h1 : b.toNat * 256 ^ bs.length ≤ 255 * 256 ^ bs.length := Nat.mul_le_mul_right _ hb + omega + +theorem beBytes_succ (w n : Nat) : + beBytes (w + 1) n = (n / 256 ^ w % 256).toUInt8 :: beBytes w n := by + simp [beBytes, List.range_succ] + +theorem toNat_toUInt8 (n : Nat) (h : n < 256) : n.toUInt8.toNat = n := by + simp [Nat.toUInt8, Nat.mod_eq_of_lt h] + +theorem beNat_beBytes (w n : Nat) : beNat (beBytes w n) = n % 256 ^ w := by + induction w with + | zero => simp [beBytes, beNat, Nat.mod_one] + | succ w ih => + rw [beBytes_succ, beNat_cons, ih, beBytes_length, + toNat_toUInt8 _ (Nat.mod_lt _ (by decide)), Nat.pow_succ, Nat.mod_mul, + Nat.mul_comm (n / 256 ^ w % 256)] + omega + +theorem beNat_beBytes_of_lt (w n : Nat) (h : n < 256 ^ w) : beNat (beBytes w n) = n := by + rw [beNat_beBytes, Nat.mod_eq_of_lt h] + +theorem beBytes_add (w a n : Nat) : beBytes w (a * 256 ^ w + n) = beBytes w n := by + unfold beBytes + apply List.map_congr_left + intro i hi + simp only [List.mem_reverse, List.mem_range] at hi + congr 1 + obtain ⟨d, rfl⟩ : ∃ d, w = i + d + 1 := ⟨w - i - 1, by omega⟩ + rw [show i + d + 1 = i + (d + 1) by omega, Nat.pow_add, ← Nat.mul_assoc, + Nat.mul_right_comm a, Nat.add_comm, Nat.add_mul_div_right _ _ (Nat.pow_pos (by decide)), + Nat.pow_succ, ← Nat.mul_assoc, Nat.add_mul_mod_self_right] + +theorem beBytes_beNat (c : List UInt8) : beBytes c.length (beNat c) = c := by + induction c with + | nil => rfl + | cons b c ih => + rw [List.length_cons, beBytes_succ, beNat_cons] + have hc := beNat_lt c + have hdiv : (b.toNat * 256 ^ c.length + beNat c) / 256 ^ c.length = b.toNat := by + rw [Nat.add_comm, Nat.add_mul_div_right _ _ (Nat.pow_pos (by decide)), + Nat.div_eq_of_lt hc, Nat.zero_add] + rw [hdiv, Nat.mod_eq_of_lt b.toNat_lt, beBytes_add, ih] + simp + +theorem readBE_iff (w : Nat) (b : List UInt8) (n : Nat) (r : List UInt8) : + readBE w b = some (n, r) ↔ n < 256 ^ w ∧ b = beBytes w n ++ r := by + unfold readBE + constructor + · intro h + simp only [Option.map_eq_some_iff, Prod.exists] at h + obtain ⟨c, r', hc, he⟩ := h + simp only [Prod.mk.injEq] at he + obtain ⟨rfl, rfl⟩ := he + obtain ⟨hl, rfl⟩ := (take?_eq _ _ _ _).mp hc + subst hl + exact ⟨beNat_lt c, by rw [bigEndian_eq, beBytes_beNat]⟩ + · rintro ⟨hn, rfl⟩ + have := (take?_eq w (beBytes w n ++ r) (beBytes w n) r).mpr ⟨beBytes_length _ _, rfl⟩ + rw [this] + simp [bigEndian_eq, beNat_beBytes_of_lt _ _ hn] + +theorem readBE_append (w n : Nat) (r : List UInt8) (hn : n < 256 ^ w) : + readBE w (beBytes w n ++ r) = some (n, r) := + (readBE_iff _ _ _ _).mpr ⟨hn, rfl⟩ + +theorem guard_some (p : Prop) [Decidable p] (u : Unit) : + (guard p : Option Unit) = some u ↔ p := by + unfold guard + split <;> simp_all [failure, pure] + +theorem take?_append' (w : Nat) (c r : List UInt8) (h : c.length = w) : + take? w (c ++ r) = some (c, r) := + (take?_eq _ _ _ _).mpr ⟨h, rfl⟩ + +theorem beBytes_one (a : Nat) (h : a < 256) : beBytes 1 a = [a.toUInt8] := by + simp [beBytes, Nat.mod_eq_of_lt h] + +theorem readBE_one (a : UInt8) (r : List UInt8) : readBE 1 (a :: r) = some (a.toNat, r) := by + have := readBE_append 1 a.toNat r (by simpa using a.toNat_lt) + rw [beBytes_one _ a.toNat_lt] at this + simpa using this + +end SqliteProofs diff --git a/Proofs/Header.lean b/Proofs/Header.lean new file mode 100644 index 0000000..4291d6b --- /dev/null +++ b/Proofs/Header.lean @@ -0,0 +1,105 @@ +import Proofs.Bytes + +/-! + The header reader against `Spec.headerBytes` and `Spec.HeaderValid`. +-/ + +namespace SqliteProofs + +open Spec Sqlite + +theorem pageSizeOk_iff (n : Nat) : + pageSizeOk n = true ↔ ∃ k, 9 ≤ k ∧ k ≤ 16 ∧ n = 2 ^ k := by + constructor + · intro h + simp only [pageSizeOk, List.contains_iff_mem, List.mem_cons, List.mem_nil_iff, or_false] at h + rcases h with rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl + · exact ⟨9, by decide⟩ + · exact ⟨10, by decide⟩ + · exact ⟨11, by decide⟩ + · exact ⟨12, by decide⟩ + · exact ⟨13, by decide⟩ + · exact ⟨14, by decide⟩ + · exact ⟨15, by decide⟩ + · exact ⟨16, by decide⟩ + · rintro ⟨k, h1, h2, rfl⟩ + have : k = 9 ∨ k = 10 ∨ k = 11 ∨ k = 12 ∨ k = 13 ∨ k = 14 ∨ k = 15 ∨ k = 16 := by omega + rcases this with rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl <;> decide + +theorem guard_true : (guard True : Option Unit) = some () := rfl + +theorem readBE_one' (a : UInt8) (l r : List UInt8) : + readBE 1 ((a :: l) ++ r) = some (a.toNat, l ++ r) := + readBE_one a (l ++ r) + +/-- The page size field gives back the page size. -/ +theorem pageSize_code (ps : Nat) (h : ps < 65536) : + (if (if ps = 1 then 65536 else ps) = 65536 then 1 else (if ps = 1 then 65536 else ps)) = ps := by + by_cases h1 : ps = 1 + · simp [h1] + · rw [ite_eq_right h1, ite_eq_right (by omega)] + +theorem pageSize_code' (n : Nat) (h : ∃ k, 9 ≤ k ∧ k ≤ 16 ∧ n = 2 ^ k) : + (if (if n = 65536 then 1 else n) = 1 then 65536 else (if n = 65536 then 1 else n)) = n := by + obtain ⟨k, h1, h2, rfl⟩ := h + have : k = 9 ∨ k = 10 ∨ k = 11 ∨ k = 12 ∨ k = 13 ∨ k = 14 ∨ k = 15 ∨ k = 16 := by omega + rcases this with rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl <;> decide + +theorem readHeader_some (file : List UInt8) (h : Header) (H : readHeader file = some h) : + HeaderValid h ∧ ∃ rest, file = headerBytes h ++ rest := by + unfold readHeader at H + simp only [bind, Option.bind_eq_some_iff, Prod.exists, take?_eq, readBE_iff, guard_some, + exists_and_left, exists_const, pure, Option.some.injEq, Bool.and_eq_true, Bool.or_eq_true, + beq_iff_eq, decide_eq_true_eq, Nat.reducePow] at H + obtain ⟨m, r, ⟨-, rfl⟩, rfl, ps, r, ⟨hps, rfl⟩, wv, r, ⟨hwv, rfl⟩, rv, r, ⟨hrv, rfl⟩, + res, r, ⟨hres, rfl⟩, fr, r, ⟨-, rfl⟩, rfl, + cc, r, ⟨hcc, rfl⟩, ds, r, ⟨hds, rfl⟩, ft, r, ⟨hft, rfl⟩, fc, r, ⟨hfc, rfl⟩, + sc, r, ⟨hsc, rfl⟩, sf, r, ⟨hsf, rfl⟩, cs, r, ⟨hcs, rfl⟩, lr, r, ⟨hlr, rfl⟩, + te, r, ⟨hte, rfl⟩, uv, r, ⟨huv, rfl⟩, iv, r, ⟨hiv, rfl⟩, ai, r, ⟨hai, rfl⟩, + z, r, ⟨-, rfl⟩, rfl, vv, r, ⟨hvv, rfl⟩, sv, r, ⟨hsv, rfl⟩, + ⟨⟨⟨⟨hok, hrv'⟩, hus⟩, hsf'⟩, hte'⟩, rfl⟩ := H + refine ⟨?_, r, ?_⟩ + · simp only [HeaderValid, Nat.reducePow] + exact ⟨(pageSizeOk_iff _).1 hok, hwv, hrv', hres, hus, hcc, hds, hft, hfc, hsc, hsf', hcs, + hlr, by omega, huv, hiv, hai, hvv, hsv⟩ + · simp only [headerBytes, List.append_assoc, beBytes_one _ hwv, beBytes_one _ hrv, + beBytes_one _ hres, pageSize_code ps hps, List.cons_append, List.nil_append] + +theorem readHeader_bytes (h : Header) (hv : HeaderValid h) (rest : List UInt8) : + readHeader (headerBytes h ++ rest) = some h := by + have hv' := hv + simp only [HeaderValid] at hv' + obtain ⟨hps, hwv, hrv, hres, hus, hcc, hds, hft, hfc, hsc, hsf, hcs, hlr, hte, huv, hiv, hai, + hvv, hsv⟩ := hv' + have hps2 : (if h.pageSize = 65536 then 1 else h.pageSize) < 256 ^ 2 := by + obtain ⟨k, h1, h2, h3⟩ := hps + have : h.pageSize ≤ 65536 := by + rw [h3]; exact Nat.pow_le_pow_right (by decide) h2 + split <;> simp only [Nat.reducePow] <;> omega + have hrv1 : h.readVersion < 256 := by omega + have hte' : ((h.textEncoding = 1 ∨ h.textEncoding = 2) ∨ h.textEncoding = 3) ∨ + h.textEncoding = 0 ∧ h.schemaFormat = 0 := by omega + have b4 : ∀ n, n < 2 ^ 32 → n < 256 ^ 4 := fun n h => by simpa using h + unfold readHeader headerBytes + simp only [bind, List.append_assoc, take?_append' 16 magic _ rfl, Option.bind_some, + readBE_append 2 _ _ hps2, readBE_one', take?_append' 3 [64, 32, 32] _ rfl, + readBE_append 4 _ _ (b4 _ hcc), readBE_append 4 _ _ (b4 _ hds), readBE_append 4 _ _ (b4 _ hft), + readBE_append 4 _ _ (b4 _ hfc), readBE_append 4 _ _ (b4 _ hsc), + readBE_append 4 _ _ (b4 _ (by omega : h.schemaFormat < 2 ^ 32)), + readBE_append 4 _ _ (b4 _ hcs), readBE_append 4 _ _ (b4 _ hlr), + readBE_append 4 _ _ (b4 _ (by omega : h.textEncoding < 2 ^ 32)), + readBE_append 4 _ _ (b4 _ huv), readBE_append 4 _ _ (b4 _ hiv), readBE_append 4 _ _ (b4 _ hai), + take?_append' 20 (List.replicate 20 0) _ (by simp), + readBE_append 4 _ _ (b4 _ hvv), readBE_append 4 _ _ (b4 _ hsv), + toNat_toUInt8 _ hwv, toNat_toUInt8 _ hrv1, toNat_toUInt8 _ hres, pageSize_code' _ hps, + (pageSizeOk_iff _).2 hps, guard_true, Bool.and_eq_true, Bool.or_eq_true, + beq_iff_eq, decide_eq_true_eq, hrv, hus, hsf, hte', true_and, pure] + +theorem readHeader_iff (file : List UInt8) (h : Header) : + Sqlite.readHeader file = some h ↔ HeaderValid h ∧ ∃ rest, file = headerBytes h ++ rest := by + constructor + · exact readHeader_some file h + · rintro ⟨hv, rest, rfl⟩ + exact readHeader_bytes h hv rest + +end SqliteProofs diff --git a/Proofs/Main.lean b/Proofs/Main.lean new file mode 100644 index 0000000..89c7ffc --- /dev/null +++ b/Proofs/Main.lean @@ -0,0 +1,71 @@ +/- + The statements of `Spec.lean`, each proved from the lemmas in the + other files of `Proofs/`. `lake comparator` checks that each theorem + here is exactly the one in `Spec.lean`. +-/ + +import Proofs.Page +import Spec.Examples + +namespace Spec + +theorem readVarint_iff (b : List UInt8) (v k : Nat) (rest : List UInt8) : + Sqlite.readVarint b = some (v, k, rest) ↔ + ∃ bs, Varint bs ∧ varintValue bs = v ∧ bs.length = k ∧ b = bs ++ rest := + SqliteProofs.readVarint_iff b v k rest + +theorem readHeader_iff (file : List UInt8) (h : Header) : + Sqlite.readHeader file = some h ↔ HeaderValid h ∧ ∃ rest, file = headerBytes h ++ rest := + SqliteProofs.readHeader_iff file h + +theorem readPage_iff (h : Header) (file : List UInt8) (n : Nat) (pg : Page) : + Sqlite.readPage h file n = some pg ↔ + 1 ≤ n ∧ n * h.pageSize ≤ file.length ∧ PageEncodes h n pg (pageOf h file n) := + SqliteProofs.readPage_iff h file n pg + +theorem readPage_inside (h : Header) (file : List UInt8) (n : Nat) (pg : Page) + (hp : Sqlite.readPage h file n = some pg) : + (n - 1) * h.pageSize + h.usable ≤ file.length ∧ + ∀ p ∈ pg.pointers, + headerOffset n + pg.header.type.headerSize + 2 * pg.pointers.length ≤ p ∧ p < h.usable := + SqliteProofs.readPage_inside h file n pg hp + +theorem readRecord_iff (p : List UInt8) (vs : List Value) : + Sqlite.readRecord p = some vs ↔ RecordEncodes vs p := + SqliteProofs.readRecord_iff p vs + +theorem examples : + Sqlite.readVarint [0x00] = some (0, 1, []) ∧ + Sqlite.readVarint [0x7F, 0x55] = some (127, 1, [0x55]) ∧ + Sqlite.readVarint [0x81, 0x00] = some (128, 2, []) ∧ + Sqlite.readVarint (List.replicate 9 0xFF) = some (2 ^ 64 - 1, 9, []) ∧ + Sqlite.localOf .tableLeaf 4096 10000 = 1816 ∧ + Sqlite.readRecord [0x03, 0x01, 0x13, 0x07, 0x61, 0x62, 0x63] = + some [.int 7, .text [0x61, 0x62, 0x63]] ∧ + Sqlite.readHeader sqliteHeaderBytes = some sqliteHeader ∧ + Sqlite.readPage sqliteHeader onePageFile 1 = + some { header := { type := .tableLeaf, firstFreeblock := 0, cellCount := 1, + contentStart := 507, fragmented := 0, rightmost := none }, + pointers := [507], + cells := [{ leftChild := none, payloadSize := some 3, rowid := some 1, + payload := [0x02, 0x01, 0x05], overflow := none }] } ∧ + Sqlite.readRecord [0x02, 0x01, 0x05] = some [.int 5] := by + decide +kernel + +theorem rejects : + Sqlite.readVarint [0x81, 0x80] = none ∧ + Sqlite.readHeader (List.replicate 16 0 ++ sqliteHeaderBytes.drop 16) = none ∧ + Sqlite.readHeader (headerBytes { sqliteHeader with pageSize := 768 }) = none ∧ + Sqlite.readHeader (headerBytes { sqliteHeader with readVersion := 3 }) = none ∧ + Sqlite.readHeader (headerBytes { sqliteHeader with reserved := 33 }) = none ∧ + Sqlite.readHeader (headerBytes { sqliteHeader with textEncoding := 0 }) = none ∧ + Sqlite.readHeader (sqliteHeaderBytes.set 21 63) = none ∧ + Sqlite.readPage sqliteHeader onePageFile 2 = none ∧ + Sqlite.readPage sqliteHeader onePageFile 0 = none ∧ + Sqlite.readRecord [0x02, 0x0A] = none ∧ + Sqlite.readRecord [0x02, 0x01, 0x07, 0x00] = none ∧ + Sqlite.readRecord [0x05, 0x01] = none ∧ + Sqlite.readRecord [0x01] = none := by + decide +kernel + +end Spec diff --git a/Proofs/Page.lean b/Proofs/Page.lean new file mode 100644 index 0000000..f51a0cd --- /dev/null +++ b/Proofs/Page.lean @@ -0,0 +1,410 @@ +import Proofs.Varint +import Proofs.Header +import Proofs.Record + +/-! + The page reader against `Spec.PageEncodes`: the page header, the cell + pointers, payloads, cells, and the page. +-/ + +namespace SqliteProofs + +open Spec Sqlite + +theorem pageType?_eq (n : Nat) (t : PageType) : pageType? n = some t ↔ t.code = n := by + constructor + · intro h + unfold pageType? at h + split at h <;> simp only [Option.some.injEq, reduceCtorEq] at h <;> subst h <;> rfl + · rintro rfl + cases t <;> rfl + +theorem code_lt (t : PageType) : t.code < 256 := by cases t <;> decide + +theorem contentStart_code (cs : Nat) (h : cs < 65536) : + (if cs = 0 then 65536 else cs) % 65536 = cs := by + by_cases h0 : cs = 0 + · simp [h0] + · rw [ite_eq_right h0, Nat.mod_eq_of_lt h] + +theorem contentStart_code' (cs : Nat) (h1 : 1 ≤ cs) (h2 : cs ≤ 65536) : + (if cs % 65536 = 0 then 65536 else cs % 65536) = cs := by + by_cases h : cs = 65536 + · subst h; rfl + · rw [Nat.mod_eq_of_lt (by omega), ite_eq_right (by omega)] + +theorem readPageHeader_some (b : List UInt8) (ph : PageHeader) (r : List UInt8) + (H : readPageHeader b = some (ph, r)) : PageHeaderValid ph ∧ b = pageHeaderBytes ph ++ r := by + unfold readPageHeader at H + simp only [bind, Option.bind_eq_some_iff, Prod.exists, readBE_iff, pageType?_eq, + Nat.reducePow] at H + obtain ⟨fl, r1, ⟨hfl, rfl⟩, ty, rfl, ff, r2, ⟨hff, rfl⟩, cc, r3, ⟨hcc, rfl⟩, + cs, r4, ⟨hcs, rfl⟩, fr, r5, ⟨hfr, rfl⟩, H⟩ := H + split at H + · rename_i hi + simp only [Option.bind_eq_some_iff, Prod.exists, readBE_iff, Nat.reducePow, pure, + Option.some.injEq, Prod.mk.injEq] at H + obtain ⟨p, r6, ⟨hp, rfl⟩, rfl, rfl⟩ := H + refine ⟨⟨hff, hcc, by dsimp only; split <;> omega, by dsimp only; split <;> omega, hfr, hi, hp⟩, ?_⟩ + simp only [pageHeaderBytes, beBytes_one _ (code_lt ty), beBytes_one _ hfr, + contentStart_code _ hcs, List.append_assoc, List.cons_append, List.nil_append] + · rename_i hi + simp only [pure, Option.some.injEq, Prod.mk.injEq] at H + obtain ⟨rfl, rfl⟩ := H + refine ⟨⟨hff, hcc, by dsimp only; split <;> omega, by dsimp only; split <;> omega, hfr, by simpa using hi⟩, ?_⟩ + simp only [pageHeaderBytes, beBytes_one _ (code_lt ty), beBytes_one _ hfr, + contentStart_code _ hcs, List.append_assoc, List.cons_append, List.nil_append, + List.append_nil] + +theorem readPageHeader_bytes (ph : PageHeader) (hv : PageHeaderValid ph) (r : List UInt8) : + readPageHeader (pageHeaderBytes ph ++ r) = some (ph, r) := by + obtain ⟨ty, ff, cc, cs, fr, rm⟩ := ph + obtain ⟨hff, hcc, hcs1, hcs2, hfr, hrm⟩ := hv + dsimp only at hff hcc hcs1 hcs2 hfr hrm + have hff' : ff < 256 ^ 2 := by simpa using hff + have hcc' : cc < 256 ^ 2 := by simpa using hcc + have hcs' : cs % 65536 < 256 ^ 2 := by simp only [Nat.reducePow]; omega + unfold readPageHeader pageHeaderBytes + cases rm with + | some p => + obtain ⟨hi, hp⟩ := hrm + have hp' : p < 256 ^ 4 := by simpa using hp + simp only [List.append_assoc, List.cons_append, List.nil_append, readBE_one, bind, + Option.bind_some, + toNat_toUInt8 _ (code_lt ty), (pageType?_eq _ ty).2 rfl, readBE_append 2 _ _ hff', + readBE_append 2 _ _ hcc', readBE_append 2 _ _ hcs', toNat_toUInt8 _ hfr, + contentStart_code' _ hcs1 hcs2, hi, ite_true, readBE_append 4 _ _ hp', pure] + | none => + have hi : ty.interior = false := hrm + simp only [List.append_assoc, List.cons_append, List.nil_append, readBE_one, bind, + Option.bind_some, + toNat_toUInt8 _ (code_lt ty), (pageType?_eq _ ty).2 rfl, readBE_append 2 _ _ hff', + readBE_append 2 _ _ hcc', readBE_append 2 _ _ hcs', toNat_toUInt8 _ hfr, + contentStart_code' _ hcs1 hcs2, hi, pure] + rfl + +theorem readPageHeader_iff (b : List UInt8) (ph : PageHeader) (r : List UInt8) : + readPageHeader b = some (ph, r) ↔ PageHeaderValid ph ∧ b = pageHeaderBytes ph ++ r := by + constructor + · exact readPageHeader_some b ph r + · rintro ⟨hv, rfl⟩ + exact readPageHeader_bytes ph hv r + +theorem readPointers_iff (k : Nat) : + ∀ b ps, readPointers k b = some ps ↔ + ps.length = k ∧ (∀ p ∈ ps, p < 2 ^ 16) ∧ ∃ rest, b = pointerBytes ps ++ rest := by + induction k with + | zero => + intro b ps + simp only [readPointers, Option.some.injEq] + constructor + · rintro rfl; exact ⟨rfl, by simp, b, by simp [pointerBytes]⟩ + · rintro ⟨h, -, -⟩; exact (List.eq_nil_of_length_eq_zero h).symm + | succ k ih => + intro b ps + rw [readPointers.eq_2] + constructor + · intro h + simp only [bind, Option.bind_eq_some_iff, Prod.exists, readBE_iff, ih, pure, + Option.some.injEq] at h + obtain ⟨p, r, ⟨hp, rfl⟩, ps', ⟨hl, hb, rest, rfl⟩, rfl⟩ := h + refine ⟨by simp [hl], ?_, rest, by simp [pointerBytes]⟩ + simp only [List.mem_cons, forall_eq_or_imp] + exact ⟨by simpa using hp, hb⟩ + · rintro ⟨hl, hb, rest, rfl⟩ + cases ps with + | nil => simp at hl + | cons p ps => + simp only [List.length_cons, Nat.add_right_cancel_iff] at hl + simp only [List.mem_cons, forall_eq_or_imp] at hb + have hp : p < 256 ^ 2 := by simpa using hb.1 + have := (ih (pointerBytes ps ++ rest) ps).2 ⟨hl, hb.2, rest, rfl⟩ + simp only [pointerBytes, List.flatMap_cons, List.append_assoc] at this ⊢ + simp only [readBE_append 2 _ _ hp, bind, Option.bind_some, this, pure] + +theorem localOf_eq (t : PageType) (U P : Nat) : localOf t U P = localSize t U P := by + cases t <;> rfl + +theorem int64_eq (n : Nat) : int64 n = toInt64 n := by + simp only [int64, toInt64, Nat.reducePow, Int.reducePow] + +theorem readPayload_iff (t : PageType) (U P : Nat) (b kept : List UInt8) (ovf : Option Nat) + (k : Nat) : + readPayload t U P b = some (kept, ovf, k) ↔ + ∃ bs rest, PayloadEncodes t U P kept ovf bs ∧ k = bs.length ∧ b = bs ++ rest := by + unfold readPayload + simp only [localOf_eq] + constructor + · intro H + simp only [bind, Option.bind_eq_some_iff, Prod.exists, take?_eq] at H + obtain ⟨loc, r, ⟨hl, rfl⟩, H⟩ := H + split at H + · rename_i hlt + simp only [Option.bind_eq_some_iff, Prod.exists, readBE_iff, pure, Option.some.injEq, + Prod.mk.injEq] at H + obtain ⟨o, r', ⟨ho, rfl⟩, rfl, rfl, rfl⟩ := H + refine ⟨loc ++ beBytes 4 o, r', ⟨hl, ?_⟩, by simp [beBytes_length, hl], by simp⟩ + rw [ite_eq_left hlt] + exact ⟨o, by simpa using ho, rfl, rfl⟩ + · rename_i hlt + simp only [pure, Option.some.injEq, Prod.mk.injEq] at H + obtain ⟨rfl, rfl, rfl⟩ := H + refine ⟨loc, r, ⟨hl, ?_⟩, hl.symm, rfl⟩ + rw [ite_eq_right hlt] + exact ⟨rfl, rfl⟩ + · rintro ⟨bs, rest, ⟨hl, hif⟩, rfl, rfl⟩ + by_cases hlt : localSize t U P < P + · rw [ite_eq_left hlt] at hif + obtain ⟨o, ho, rfl, rfl⟩ := hif + have ho' : o < 256 ^ 4 := by simpa using ho + simp only [List.append_assoc, bind, take?_append' _ kept _ hl, Option.bind_some, hlt, + ite_true, readBE_append 4 o rest ho', pure, List.length_append, beBytes_length, hl] + · rw [ite_eq_right hlt] at hif + obtain ⟨rfl, rfl⟩ := hif + simp only [bind, take?_append' _ bs _ hl, Option.bind_some, hlt, ite_false, pure, hl] + +theorem readCell_iff (t : PageType) (U : Nat) (b : List UInt8) (c : Cell) (size : Nat) : + readCell t U b = some (c, size) ↔ + ∃ bs rest, CellEncodes t U c bs ∧ size = bs.length ∧ b = bs ++ rest := by + cases t with + | tableLeaf => + constructor + · intro H + simp only [readCell, bind, Option.bind_eq_some_iff, Prod.exists, readVarint_iff, + readPayload_iff, pure, Option.some.injEq, Prod.mk.injEq] at H + obtain ⟨P, n1, r1, ⟨vp, hvp, rfl, rfl, rfl⟩, rowid, n2, r2, ⟨vr, hvr, rfl, rfl, rfl⟩, + loc, ovf, n3, ⟨pb, rest, hpe, rfl, rfl⟩, rfl, rfl⟩ := H + refine ⟨vp ++ vr ++ pb, rest, ?_, by simp [Nat.add_assoc], by simp⟩ + unfold CellEncodes + exact ⟨vp, vr, pb, hvp, hvr, rfl, rfl, rfl, by simp [int64_eq], hpe⟩ + · rintro ⟨bs, rest, hc, rfl, rfl⟩ + simp only [CellEncodes] at hc + obtain ⟨vp, vr, pb, hvp, hvr, rfl, h1, h2, h3, hpe⟩ := hc + have hp := (readPayload_iff _ _ _ _ _ _ _).2 ⟨pb, rest, hpe, rfl, rfl⟩ + simp only [readCell, List.append_assoc, readVarint_append _ _ hvp, readVarint_append _ _ hvr, + bind, Option.bind_some, hp, pure, Option.some.injEq, Prod.mk.injEq, List.length_append] + refine ⟨?_, by omega⟩ + cases c + simp_all [int64_eq] + | tableInterior => + constructor + · intro H + simp only [readCell, bind, Option.bind_eq_some_iff, Prod.exists, readBE_iff, readVarint_iff, + pure, Option.some.injEq, Prod.mk.injEq] at H + obtain ⟨k, r1, ⟨hk, rfl⟩, rowid, n2, r2, ⟨vr, hvr, rfl, rfl, rfl⟩, rfl, rfl⟩ := H + refine ⟨beBytes 4 k ++ vr, r2, ?_, by simp [beBytes_length], by simp⟩ + unfold CellEncodes + exact ⟨k, vr, by simpa using hk, hvr, rfl, by simp [int64_eq]⟩ + · rintro ⟨bs, rest, hc, rfl, rfl⟩ + simp only [CellEncodes] at hc + obtain ⟨k, vr, hk, hvr, rfl, rfl⟩ := hc + have hk' : k < 256 ^ 4 := by simpa using hk + simp only [readCell, List.append_assoc, readBE_append 4 k _ hk', readVarint_append _ _ hvr, + bind, Option.bind_some, pure, int64_eq, List.length_append, beBytes_length] + | indexLeaf => + constructor + · intro H + simp only [readCell, bind, Option.bind_eq_some_iff, Prod.exists, readVarint_iff, + readPayload_iff, pure, Option.some.injEq, Prod.mk.injEq] at H + obtain ⟨P, n1, r1, ⟨vp, hvp, rfl, rfl, rfl⟩, loc, ovf, n3, ⟨pb, rest, hpe, rfl, rfl⟩, + rfl, rfl⟩ := H + refine ⟨vp ++ pb, rest, ?_, by simp, by simp⟩ + unfold CellEncodes + exact ⟨vp, pb, hvp, rfl, rfl, rfl, rfl, hpe⟩ + · rintro ⟨bs, rest, hc, rfl, rfl⟩ + simp only [CellEncodes] at hc + obtain ⟨vp, pb, hvp, rfl, h1, h2, h3, hpe⟩ := hc + have hp := (readPayload_iff _ _ _ _ _ _ _).2 ⟨pb, rest, hpe, rfl, rfl⟩ + simp only [readCell, List.append_assoc, readVarint_append _ _ hvp, + bind, Option.bind_some, hp, pure, Option.some.injEq, Prod.mk.injEq, List.length_append] + refine ⟨?_, by simp⟩ + cases c + simp_all + | indexInterior => + constructor + · intro H + simp only [readCell, bind, Option.bind_eq_some_iff, Prod.exists, readBE_iff, readVarint_iff, + readPayload_iff, pure, Option.some.injEq, Prod.mk.injEq] at H + obtain ⟨k, r0, ⟨hk, rfl⟩, P, n1, r1, ⟨vp, hvp, rfl, rfl, rfl⟩, loc, ovf, n3, + ⟨pb, rest, hpe, rfl, rfl⟩, rfl, rfl⟩ := H + refine ⟨beBytes 4 k ++ vp ++ pb, rest, ?_, by simp [beBytes_length, Nat.add_assoc], + by simp⟩ + unfold CellEncodes + exact ⟨k, vp, pb, by simpa using hk, hvp, rfl, rfl, rfl, rfl, hpe⟩ + · rintro ⟨bs, rest, hc, rfl, rfl⟩ + simp only [CellEncodes] at hc + obtain ⟨k, vp, pb, hk, hvp, rfl, h1, h2, h3, hpe⟩ := hc + have hk' : k < 256 ^ 4 := by simpa using hk + have hp := (readPayload_iff _ _ _ _ _ _ _).2 ⟨pb, rest, hpe, rfl, rfl⟩ + simp only [readCell, List.append_assoc, readBE_append 4 k _ hk', readVarint_append _ _ hvp, + bind, Option.bind_some, hp, pure, Option.some.injEq, Prod.mk.injEq, List.length_append, + beBytes_length] + refine ⟨?_, by omega⟩ + cases c + simp_all + +/-- What `readCells` checks of each cell, as `PageEncodes` says it. -/ +def CellsAt (t : PageType) (U start : Nat) (page : List UInt8) (ptrs : List Nat) + (cells : List Cell) : Prop := + cells.length = ptrs.length ∧ ∀ pc ∈ ptrs.zip cells, ∃ bs, + CellEncodes t U pc.2 bs ∧ start ≤ pc.1 ∧ pc.1 + bs.length ≤ U ∧ + (page.drop pc.1).take bs.length = bs + +theorem readCells_iff (t : PageType) (U start : Nat) (page : List UInt8) (ptrs : List Nat) : + ∀ cells, readCells t U start page ptrs = some cells ↔ CellsAt t U start page ptrs cells := by + induction ptrs with + | nil => + intro cells + simp only [readCells, Option.some.injEq, CellsAt, List.length_nil, List.zip_nil_left, + List.not_mem_nil, false_implies, implies_true, and_true] + constructor + · rintro rfl; rfl + · intro h; exact (List.eq_nil_of_length_eq_zero h).symm + | cons p ps ih => + intro cells + rw [readCells.eq_2] + constructor + · intro H + simp only [bind, Option.bind_eq_some_iff, Prod.exists, guard_some, exists_const, + readCell_iff, ih, pure, Option.some.injEq] at H + obtain ⟨hs, c, size, ⟨bs, rest, hc, rfl, hd⟩, hU, cs, ⟨hl, hall⟩, rfl⟩ := H + refine ⟨by simp [hl], ?_⟩ + intro pc hpc + simp only [List.zip_cons_cons, List.mem_cons] at hpc + rcases hpc with rfl | hpc + · exact ⟨bs, hc, hs, hU, by rw [hd]; exact List.take_left⟩ + · exact hall pc hpc + · rintro ⟨hl, hall⟩ + cases cells with + | nil => simp at hl + | cons c cs => + obtain ⟨bs, hc, hs, hU, ht⟩ := hall (p, c) (by simp) + have hd : page.drop p = bs ++ (page.drop p).drop bs.length := by + conv => lhs; rw [← List.take_append_drop bs.length (page.drop p)] + rw [ht] + have h1 := (readCell_iff t U (page.drop p) c bs.length).2 ⟨bs, _, hc, rfl, hd⟩ + have h2 := (ih cs).2 ⟨by simpa using hl, fun pc h => hall pc (by simp [h])⟩ + simp only [bind, guard_of _ hs, Option.bind_some, h1, guard_of _ hU, h2, pure] + +theorem pageHeaderBytes_length (ph : PageHeader) (hv : PageHeaderValid ph) : + (pageHeaderBytes ph).length = ph.type.headerSize := by + obtain ⟨-, -, -, -, -, hrm⟩ := hv + unfold pageHeaderBytes PageType.headerSize + cases h : ph.rightmost with + | some p => + rw [h] at hrm + simp [beBytes_length, hrm.1] + | none => + rw [h] at hrm + simp [beBytes_length, hrm] + +theorem pointerBytes_length (ps : List Nat) : (pointerBytes ps).length = 2 * ps.length := by + induction ps with + | nil => rfl + | cons p ps ih => + simp only [pointerBytes, List.flatMap_cons, List.length_append, beBytes_length] at ih ⊢ + rw [ih, List.length_cons] + omega + +theorem page_length (file : List UInt8) (n ps : Nat) (hn : 1 ≤ n) : + ((file.drop ((n - 1) * ps)).take ps).length = ps ↔ n * ps ≤ file.length := by + obtain ⟨m, rfl⟩ : ∃ m, n = m + 1 := ⟨n - 1, by omega⟩ + simp only [Nat.add_sub_cancel, List.length_take, List.length_drop, Nat.succ_mul] + rcases Nat.eq_zero_or_pos ps with rfl | hps + · simp + · omega + +theorem readPage_iff (h : Header) (file : List UInt8) (n : Nat) (pg : Page) : + Sqlite.readPage h file n = some pg ↔ + 1 ≤ n ∧ n * h.pageSize ≤ file.length ∧ PageEncodes h n pg (pageOf h file n) := by + constructor + · intro H + unfold readPage at H + simp only [bind, Option.bind_eq_some_iff, Prod.exists, guard_some, exists_const, + readPageHeader_iff, readPointers_iff, readCells_iff, Bool.and_eq_true, decide_eq_true_eq, + pure, Option.some.injEq] at H + obtain ⟨hn, hlen, ph, r, ⟨hv, hd⟩, ptrs, ⟨hpl, hpb, rest, rfl⟩, ⟨hc1, hc2⟩, cells, + ⟨hcl, hcells⟩, rfl⟩ := H + refine ⟨hn, (page_length _ _ _ hn).1 hlen, hv, hpl.symm, hcl, hpb, ?_, ?_, hc2, hcells⟩ + rotate_left + · show headerOffset n + ph.type.headerSize + 2 * ptrs.length ≤ ph.contentStart + unfold headerOffset PageType.headerSize + rw [hpl] + exact hc1 + show List.take _ (List.drop (headerOffset n) (pageOf h file n)) = _ + unfold headerOffset pageOf + rw [hd, ← List.append_assoc] + apply List.take_left' + simp [pageHeaderBytes_length _ hv, pointerBytes_length, PageType.headerSize] + · rintro ⟨hn, hlen, hv, hpl, hcl, hpb, ht, hc1, hc2, hcells⟩ + unfold readPage + have hlen' := (page_length file n h.pageSize hn).2 hlen + have hd : (pageOf h file n).drop (headerOffset n) = pageHeaderBytes pg.header ++ + (pointerBytes pg.pointers ++ + ((pageOf h file n).drop (headerOffset n)).drop + (pg.header.type.headerSize + 2 * pg.pointers.length)) := by + rw [← List.append_assoc, ← ht, List.take_append_drop] + unfold pageOf headerOffset at hd + generalize List.drop (pg.header.type.headerSize + 2 * pg.pointers.length) _ = R at hd + have h1 := (readPageHeader_iff _ _ _).2 ⟨hv, hd⟩ + have h2 := (readPointers_iff pg.header.cellCount (pointerBytes pg.pointers ++ R) pg.pointers).2 + ⟨hpl.symm, hpb, R, rfl⟩ + have h3 := (readCells_iff pg.header.type (h.pageSize - h.reserved) pg.header.contentStart + ((file.drop ((n - 1) * h.pageSize)).take h.pageSize) pg.pointers pg.cells).2 ⟨hcl, hcells⟩ + have hc1' : (if n = 1 then 100 else 0) + (if pg.header.type.interior = true then 12 else 8) + + 2 * pg.header.cellCount ≤ pg.header.contentStart := by + rw [hpl]; exact hc1 + simp only [bind, guard_of _ hn, Option.bind_some, guard_of _ hlen', h1, h2, h3, pure, + Bool.and_eq_true, decide_eq_true_eq] + rw [guard_of _ ⟨hc1', hc2⟩, Option.bind_some] + +theorem cellEncodes_length (t : PageType) (U : Nat) (c : Cell) (bs : List UInt8) + (h : CellEncodes t U c bs) : 1 ≤ bs.length := by + cases t <;> unfold CellEncodes at h + · obtain ⟨k, vp, rest, -, -, rfl, -⟩ := h + simp only [List.length_append, beBytes_length] + omega + · obtain ⟨k, vr, -, -, rfl, -⟩ := h + simp only [List.length_append, beBytes_length] + omega + · obtain ⟨vp, rest, hv, rfl, -⟩ := h + have := hv.1 + simp only [List.length_append] + omega + · obtain ⟨vp, vr, rest, hv, -, rfl, -⟩ := h + have := hv.1 + simp only [List.length_append] + omega + +theorem mem_zip_left {α β} (as : List α) (bs : List β) (a : α) (ha : a ∈ as) + (hl : bs.length = as.length) : ∃ b, (a, b) ∈ as.zip bs := by + induction as generalizing bs with + | nil => simp at ha + | cons x as ih => + cases bs with + | nil => simp at hl + | cons y bs => + simp only [List.mem_cons] at ha + rcases ha with rfl | ha + · exact ⟨y, by simp⟩ + · obtain ⟨b, hb⟩ := ih bs ha (by simpa using hl) + exact ⟨b, by simp [hb]⟩ + +theorem readPage_inside (h : Header) (file : List UInt8) (n : Nat) (pg : Page) + (hp : Sqlite.readPage h file n = some pg) : + (n - 1) * h.pageSize + h.usable ≤ file.length ∧ + ∀ p ∈ pg.pointers, + headerOffset n + pg.header.type.headerSize + 2 * pg.pointers.length ≤ p ∧ p < h.usable := by + obtain ⟨hn, hlen, -, -, hcl, -, -, hc1, -, hcells⟩ := (readPage_iff h file n pg).1 hp + constructor + · obtain ⟨m, rfl⟩ : ∃ m, n = m + 1 := ⟨n - 1, by omega⟩ + simp only [Nat.add_sub_cancel, Header.usable] + rw [Nat.succ_mul] at hlen + omega + · intro p hmem + obtain ⟨c, hc⟩ := mem_zip_left _ _ p hmem hcl + obtain ⟨bs, hce, hs, hU, -⟩ := hcells (p, c) hc + have := cellEncodes_length _ _ _ _ hce + exact ⟨by omega, by omega⟩ + +end SqliteProofs diff --git a/Proofs/Record.lean b/Proofs/Record.lean new file mode 100644 index 0000000..cc1672f --- /dev/null +++ b/Proofs/Record.lean @@ -0,0 +1,197 @@ +import Proofs.Varint +import Proofs.Header + +/-! + The record reader against `Spec.RecordEncodes`. +-/ + +namespace SqliteProofs + +open Spec Sqlite + +theorem guard_of (p : Prop) [Decidable p] (hp : p) : (guard p : Option Unit) = some () := + (guard_some p ()).mpr hp + +theorem varint_ne_nil (bs : List UInt8) (h : Varint bs) : bs ≠ [] := by + intro e; subst e; exact absurd h.1 (by simp) + +theorem readVarint_append (bs rest : List UInt8) (h : Varint bs) : + readVarint (bs ++ rest) = some (varintValue bs, bs.length, rest) := + (readVarint_iff _ _ _ _).2 ⟨bs, h, rfl, rfl, rfl⟩ + +theorem small (t : Nat) (h : t < 12) : + t = 0 ∨ t = 1 ∨ t = 2 ∨ t = 3 ∨ t = 4 ∨ t = 5 ∨ t = 6 ∨ t = 7 ∨ t = 8 ∨ t = 9 ∨ t = 10 ∨ + t = 11 := by + omega + +theorem sizeOf_eq (t : Nat) : Sqlite.sizeOf t = contentSize t := by + rcases Nat.lt_or_ge t 12 with h | h + · rcases small t h with rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl <;> + rfl + · obtain ⟨n, rfl⟩ : ∃ n, t = n + 12 := ⟨t - 12, by omega⟩ + show _ = some (if (n + 12) % 2 = 0 then (n + 12 - 12) / 2 else (n + 12 - 13) / 2) + unfold Sqlite.sizeOf + rw [ite_eq_right (by omega), ite_eq_right (by omega), ite_eq_right (by omega), + ite_eq_right (by omega), ite_eq_right (by omega), Option.some.injEq] + split <;> omega + +theorem signed_eq (c : List UInt8) (h : 1 ≤ c.length) : signed c = signedOf c := by + have e1 : (256 : Nat) ^ c.length = 2 ^ (8 * c.length) := by rw [Nat.pow_mul] + have e2 : 2 ^ (8 * c.length) = 2 * 2 ^ (8 * c.length - 1) := by + rw [← Nat.pow_succ']; congr 1; omega + have e3 : (256 : Int) ^ c.length = 2 ^ (8 * c.length) := by rw [Int.pow_mul]; rfl + unfold signed signedOf + simp only [bigEndian_eq, e3] + by_cases hc : beNat c < 2 ^ (8 * c.length - 1) + · rw [ite_eq_left (by omega), ite_eq_left hc] + · rw [ite_eq_right (by omega), ite_eq_right hc] + +theorem columnValue_eq (t : Nat) (c : List UInt8) (h : Sqlite.sizeOf t = some c.length) : + columnValue t c = valueOf t c := by + rcases Nat.lt_or_ge t 12 with ht | ht + · rcases small t ht with rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl | rfl <;> + simp [Sqlite.sizeOf] at h <;> simp [columnValue, valueOf, bigEndian_eq] <;> rw [signed_eq _ (by omega)] + · obtain ⟨n, rfl⟩ : ∃ n, t = n + 12 := ⟨t - 12, by omega⟩ + show _ = if (n + 12) % 2 = 0 then Value.blob c else Value.text c + unfold columnValue + rw [ite_eq_right (by omega), ite_eq_right (by omega), ite_eq_right (by omega), + ite_eq_right (by omega), ite_eq_right (by omega)] + +/-- The columns, by the values of their serial types. -/ +def ColsN : List Nat → List (List UInt8) → List Value → Prop + | [], [], [] => True + | n :: ns, c :: cs, v :: vs => contentSize n = some c.length ∧ v = valueOf n c ∧ ColsN ns cs vs + | _, _, _ => False + +theorem colsN_length (ns : List Nat) : + ∀ cs vs, ColsN ns cs vs → vs.length = ns.length := by + induction ns with + | nil => intro cs vs h; cases cs <;> cases vs <;> simp_all [ColsN] + | cons n ns ih => + intro cs vs h + cases cs <;> cases vs <;> simp only [ColsN] at h + simp [ih _ _ h.2.2] + +theorem columnsEncode_iff (ts : List (List UInt8)) : + ∀ cs vs, ColumnsEncode ts cs vs ↔ (∀ t ∈ ts, Varint t) ∧ ColsN (ts.map varintValue) cs vs := by + induction ts with + | nil => intro cs vs; cases cs <;> cases vs <;> simp [ColumnsEncode, ColsN] + | cons t ts ih => + intro cs vs + cases cs <;> cases vs <;> simp only [ColumnsEncode, ColsN, List.map_cons, and_false] + rw [ih] + simp only [List.mem_cons, forall_eq_or_imp] + constructor + · rintro ⟨h1, h2, h3, h4, h5⟩; exact ⟨⟨h1, h4⟩, h2, h3, h5⟩ + · rintro ⟨⟨h1, h4⟩, h2, h3, h5⟩; exact ⟨h1, h2, h3, h4, h5⟩ + +theorem readColumns_iff (ns : List Nat) : + ∀ body vs, readColumns ns body = some vs ↔ ∃ cs, body = cs.flatten ∧ ColsN ns cs vs := by + induction ns with + | nil => + intro body vs + constructor + · intro h + cases body with + | nil => + rw [readColumns.eq_1, Option.some.injEq] at h + exact ⟨[], rfl, by subst h; trivial⟩ + | cons x xs => rw [readColumns.eq_2] at h; exact absurd h (by simp) + · rintro ⟨cs, rfl, hc⟩ + cases cs <;> cases vs <;> simp_all [ColsN, readColumns.eq_1] + | cons n ns ih => + intro body vs + rw [readColumns.eq_3] + constructor + · intro h + simp only [bind, Option.bind_eq_some_iff, Prod.exists, take?_eq, ih, pure, + Option.some.injEq] at h + obtain ⟨size, hsize, c, r, ⟨hcl, rfl⟩, vs', ⟨cs, rfl, hcs⟩, rfl⟩ := h + rw [← hcl] at hsize + refine ⟨c :: cs, rfl, ?_, ?_, hcs⟩ + · rw [← sizeOf_eq]; exact hsize + · exact columnValue_eq _ _ hsize + · rintro ⟨cs, rfl, hc⟩ + cases cs with + | nil => cases vs <;> simp [ColsN] at hc + | cons c cs => + cases vs with + | nil => simp [ColsN] at hc + | cons v vs => + obtain ⟨h1, rfl, h3⟩ := hc + have hs : Sqlite.sizeOf n = some c.length := by rw [sizeOf_eq]; exact h1 + have hr := (ih cs.flatten vs).2 ⟨cs, rfl, h3⟩ + simp only [List.flatten_cons, hs, bind, Option.bind_some, + take?_append' _ c _ rfl, hr, pure, columnValue_eq _ _ hs] + +theorem readTypes_some (fuel : Nat) : + ∀ b ns, readTypes fuel b = some ns → + ∃ ts : List (List UInt8), (∀ t ∈ ts, Varint t) ∧ b = ts.flatten ∧ + ns = ts.map varintValue := by + induction fuel with + | zero => + intro b ns h + cases b with + | nil => rw [readTypes.eq_1, Option.some.injEq] at h; exact ⟨[], by simp, rfl, h.symm⟩ + | cons x xs => rw [readTypes.eq_2] at h; exact absurd h (by simp) + | succ f ih => + intro b ns h + cases b with + | nil => rw [readTypes.eq_1, Option.some.injEq] at h; exact ⟨[], by simp, rfl, h.symm⟩ + | cons x xs => + rw [readTypes.eq_3 _ _ (by simp)] at h + simp only [bind, Option.bind_eq_some_iff, Prod.exists, pure, Option.some.injEq] at h + obtain ⟨t, k, r, hv, ts', hts, rfl⟩ := h + obtain ⟨bs, hbs, rfl, rfl, he⟩ := (readVarint_iff _ _ _ _).1 hv + obtain ⟨ts, htv, rfl, rfl⟩ := ih _ _ hts + refine ⟨bs :: ts, ?_, by simp [he], rfl⟩ + simp only [List.mem_cons, forall_eq_or_imp] + exact ⟨hbs, htv⟩ + +theorem readTypes_flatten (ts : List (List UInt8)) : + ∀ fuel, (∀ t ∈ ts, Varint t) → ts.flatten.length ≤ fuel → + readTypes fuel ts.flatten = some (ts.map varintValue) := by + induction ts with + | nil => intro fuel _ _; exact readTypes.eq_1 _ + | cons t ts ih => + intro fuel hv hl + simp only [List.mem_cons, forall_eq_or_imp] at hv + have hne := varint_ne_nil t hv.1 + have h1 := hv.1.1 + simp only [List.flatten_cons, List.length_append] at hl + obtain ⟨f, rfl⟩ : ∃ f, fuel = f + 1 := ⟨fuel - 1, by omega⟩ + rw [List.flatten_cons, readTypes.eq_3 _ _ (by simp [hne]), readVarint_append _ _ hv.1] + simp only [bind, Option.bind_some, ih f hv.2 (by omega), pure, List.map_cons] + +theorem readRecord_iff (p : List UInt8) (vs : List Value) : + Sqlite.readRecord p = some vs ↔ RecordEncodes vs p := by + constructor + · intro H + unfold readRecord at H + simp only [bind, Option.bind_eq_some_iff, Prod.exists, guard_some, take?_eq, + exists_and_left, exists_const] at H + obtain ⟨hsize, n, r, hv, hle, hdr, body, ⟨hl, rfl⟩, ns, hts, hne, hcols⟩ := H + obtain ⟨hs, hvar, rfl, rfl, rfl⟩ := (readVarint_iff _ _ _ _).1 hv + obtain ⟨ts, htv, rfl, rfl⟩ := readTypes_some _ _ _ hts + obtain ⟨cs, rfl, hc⟩ := (readColumns_iff _ _ _).1 hcols + refine ⟨?_, hs, ts, cs, hvar, by omega, (columnsEncode_iff _ _ _).2 ⟨htv, hc⟩, by simp⟩ + intro e + have := colsN_length _ _ _ hc + rw [e] at this + exact hne (List.eq_nil_of_length_eq_zero this.symm) + · intro H + obtain ⟨hne, hs, ts, cs, hvar, hval, hcols, rfl⟩ := H + obtain ⟨htv, hc⟩ := (columnsEncode_iff _ _ _).1 hcols + have h4 : ts.map varintValue ≠ [] := by + intro e + have := colsN_length _ _ _ hc + rw [e] at this + exact hne (List.eq_nil_of_length_eq_zero this) + unfold readRecord + simp only [List.append_assoc, readVarint_append _ _ hvar, bind, Option.bind_some, + guard_of _ (by omega : hs.length ≤ varintValue hs), + take?_append' (varintValue hs - hs.length) ts.flatten cs.flatten (by omega), + readTypes_flatten ts _ htv (Nat.le_refl _), guard_of _ h4, + (readColumns_iff _ _ _).2 ⟨cs, rfl, hc⟩] + +end SqliteProofs diff --git a/Proofs/Varint.lean b/Proofs/Varint.lean new file mode 100644 index 0000000..b8ecd13 --- /dev/null +++ b/Proofs/Varint.lean @@ -0,0 +1,139 @@ +import Proofs.Bytes + +/-! + The varint reader against `Spec.Varint` and `Spec.varintValue`. +-/ + +namespace SqliteProofs + +open Spec Sqlite + +/-- The seven-bit digits of the bytes, after `acc`. -/ +def f7 (acc : Nat) (bs : List UInt8) : Nat := + bs.foldl (fun n b => n * 128 + b.toNat % 128) acc + +/-- What `go` does on bytes with the high bit set, then one more. -/ +theorem go_run (pre : List UInt8) (x : UInt8) (rest : List UInt8) : + ∀ acc i, (∀ y ∈ pre, 128 ≤ y.toNat) → i + pre.length ≤ 8 → + readVarint.go acc i (pre ++ x :: rest) = + if i + pre.length = 8 then some (f7 acc pre * 256 + x.toNat, 9, rest) + else if x.toNat < 128 then some (f7 acc pre * 128 + x.toNat, i + pre.length + 1, rest) + else readVarint.go (f7 acc pre * 128 + x.toNat % 128) (i + pre.length + 1) rest := by + induction pre with + | nil => + intro acc i _ _ + show readVarint.go acc i (x :: rest) = _ + rw [readVarint.go.eq_2] + rfl + | cons y pre ih => + intro acc i hpre hlen + simp only [List.length_cons] at hlen + have hy : 128 ≤ y.toNat := hpre y (by simp) + rw [List.cons_append, readVarint.go.eq_2, ite_eq_right (by omega), ite_eq_right (by omega), + ih _ _ (fun z hz => hpre z (by simp [hz])) (by omega)] + have e1 : i + 1 + pre.length = i + (pre.length + 1) := by omega + have e2 : f7 (acc * 128 + y.toNat % 128) pre = f7 acc (y :: pre) := rfl + simp only [e1, e2, List.length_cons] + +/-- What `go` gives back, it read from bytes with the high bit set, then + one more. -/ +theorem go_some (b : List UInt8) : + ∀ acc i v k rest, i ≤ 8 → readVarint.go acc i b = some (v, k, rest) → + ∃ pre x, b = pre ++ x :: rest ∧ (∀ y ∈ pre, 128 ≤ y.toNat) ∧ i + pre.length ≤ 8 ∧ + (i + pre.length = 8 ∧ v = f7 acc pre * 256 + x.toNat ∧ k = 9 ∨ + i + pre.length < 8 ∧ x.toNat < 128 ∧ v = f7 acc pre * 128 + x.toNat ∧ + k = i + pre.length + 1) := by + induction b with + | nil => intro acc i v k rest _ h; simp [readVarint.go] at h + | cons y b ih => + intro acc i v k rest hi h + rw [readVarint.go.eq_2] at h + by_cases h8 : i = 8 + · rw [ite_eq_left h8] at h + simp only [Option.some.injEq, Prod.mk.injEq] at h + obtain ⟨rfl, rfl, rfl⟩ := h + exact ⟨[], y, rfl, by simp, by simp [h8], Or.inl ⟨by simp [h8], by simp [f7], rfl⟩⟩ + · rw [ite_eq_right h8] at h + by_cases hy : y.toNat < 128 + · rw [ite_eq_left hy] at h + simp only [Option.some.injEq, Prod.mk.injEq] at h + obtain ⟨rfl, rfl, rfl⟩ := h + exact ⟨[], y, rfl, by simp, by simp; omega, + Or.inr ⟨by simp; omega, hy, by simp [f7], by simp⟩⟩ + · rw [ite_eq_right hy] at h + obtain ⟨pre, x, rfl, hpre, hlen, hcase⟩ := ih _ _ _ _ _ (by omega) h + refine ⟨y :: pre, x, rfl, ?_, by simp; omega, ?_⟩ + · intro z hz + simp only [List.mem_cons] at hz + rcases hz with rfl | hz + · omega + · exact hpre z hz + · simp only [List.length_cons, f7, List.foldl_cons] + simp only [f7] at hcase + rcases hcase with ⟨h1, h2, h3⟩ | ⟨h1, h2, h3, h4⟩ + · exact Or.inl ⟨by omega, h2, h3⟩ + · exact Or.inr ⟨by omega, h2, h3, by omega⟩ + +/-- A varint is bytes with the high bit set, then one more: the last of + nine, or a byte with the high bit clear. -/ +theorem varint_split (bs : List UInt8) : + Varint bs ↔ ∃ pre x, bs = pre ++ [x] ∧ (∀ y ∈ pre, 128 ≤ y.toNat) ∧ + (pre.length = 8 ∨ pre.length < 8 ∧ x.toNat < 128) := by + constructor + · rintro ⟨h1, h9, hpre, hlast⟩ + obtain ⟨pre, x, hbs⟩ := List.eq_nil_or_concat bs |>.resolve_left (by + intro h; subst h; simp at h1) + rw [List.concat_eq_append] at hbs + subst hbs + refine ⟨pre, x, rfl, by simpa using hpre, ?_⟩ + simp only [List.length_append, List.length_singleton] at h9 hlast + by_cases h : pre.length = 8 + · exact Or.inl h + · exact Or.inr ⟨by omega, hlast (by omega) x (by simp)⟩ + · rintro ⟨pre, x, rfl, hpre, hcase⟩ + refine ⟨by simp, by simp; omega, by simpa using hpre, ?_⟩ + intro hl y hy + simp only [List.getLast?_concat, Option.mem_def, Option.some.injEq] at hy + subst hy + simp only [List.length_append, List.length_singleton] at hl + omega + +theorem varintValue_split (pre : List UInt8) (x : UInt8) + (hcase : pre.length = 8 ∨ pre.length < 8 ∧ x.toNat < 128) : + varintValue (pre ++ [x]) = + if pre.length = 8 then f7 0 pre * 256 + x.toNat else f7 0 pre * 128 + x.toNat := by + unfold varintValue + simp only [List.length_append, List.length_singleton] + by_cases h : pre.length = 8 + · rw [ite_eq_left (by omega), ite_eq_left h, List.take_append_of_le_length (by omega), + List.take_of_length_le (by omega)] + simp [f7, List.getD_eq_getElem?_getD, h] + · rw [ite_eq_right (by omega), ite_eq_right h] + have hx : x.toNat < 128 := by omega + simp [f7, List.foldl_append, Nat.mod_eq_of_lt hx] + +theorem readVarint_iff (b : List UInt8) (v k : Nat) (rest : List UInt8) : + Sqlite.readVarint b = some (v, k, rest) ↔ + ∃ bs, Varint bs ∧ varintValue bs = v ∧ bs.length = k ∧ b = bs ++ rest := by + constructor + · intro h + obtain ⟨pre, x, rfl, hpre, hlen, hcase⟩ := go_some b 0 0 v k rest (by omega) h + have hc : pre.length = 8 ∨ pre.length < 8 ∧ x.toNat < 128 := by omega + refine ⟨pre ++ [x], (varint_split _).2 ⟨pre, x, rfl, hpre, hc⟩, ?_, ?_, by simp⟩ + · rw [varintValue_split pre x hc] + rcases hcase with ⟨h1, h2, _⟩ | ⟨h1, _, h3, _⟩ + · rw [ite_eq_left (by omega), h2] + · rw [ite_eq_right (by omega), h3] + · simp only [List.length_append, List.length_singleton] + omega + · rintro ⟨bs, hv, rfl, rfl, rfl⟩ + obtain ⟨pre, x, rfl, hpre, hc⟩ := (varint_split _).1 hv + show readVarint.go 0 0 _ = _ + rw [List.append_assoc, List.singleton_append, + go_run pre x rest 0 0 hpre (by omega), varintValue_split pre x hc] + simp only [Nat.zero_add, List.length_append, List.length_singleton] + rcases hc with h | ⟨h, hx⟩ + · rw [ite_eq_left h, ite_eq_left h, h] + · rw [ite_eq_right (by omega), ite_eq_left hx, ite_eq_right (by omega)] + +end SqliteProofs diff --git a/README.md b/README.md index 421619c..d19ef74 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,99 @@ -# sqlite +# sqlite: reading the SQLite file format, proved -A Lean reader for the SQLite file format. This is a Proven Commons project: it rebuilds a widely used format in Lean and proves the new code follows its written specification. It is not SQLite itself. It reads SQLite files. It is not a database engine. +SQLite databases are everywhere: in phones, browsers, operating systems and most desktop programs. A database file is often a file someone else wrote, sent as an attachment, synced from another machine, or recovered from a damaged disk, and programs other than SQLite read it: forensic tools, backup tools, viewers, and readers written in other languages. Each of these takes offsets, sizes and lengths from the file and follows them. A reader that trusts one of them reads past the end of a page or of the file. -Status: not verified. No code has landed yet. Plan issue 1 holds the first slice, and it is unclaimed. The specification has not been approved by a reviewer, so do not call this verified. +This library reads three parts of the format, in Lean: -## Wave 1 +- the 100-byte database header (1.3 of "Database File Format", https://www.sqlite.org/fileformat2.html); +- one b-tree page: its page header, its cell pointer array, and its cells, with the part of each payload kept on the page by the rule of 1.6; +- records, the format of every row and index entry (2.1), and varints (1.6). -This is one of the three Wave 1 projects in the founding brief, months 4 to 9, one squad each. The other two are the Cedar style policy engine and the Redis protocol parser. Redis already has a repo. +Machine-checked proofs show: -## First slice +- **The readers accept exactly what the format says, and give back what it holds.** The varint, header, page and record readers each accept an input exactly when it is what `Spec/Defs.lean` writes down from the document, and give the values it holds. The header reader refuses every header that breaks a "must" of 1.3. +- **Nothing the page reader gives lies outside the file or its page.** It refuses page 0 and a page past the end of the file. On a page it gives, the usable part of the page is in the file, the cell pointer array ends before the cell content area starts, and every cell starts in that area and ends within the usable size. +- **The local part of a payload is what the rule of 1.6 gives.** A cell keeps on its page the number of payload bytes the X, M and K rule fixes, and the 4-byte page number of the first overflow page follows exactly when the payload spills. -- The 100 byte header. -- The page header. -- Records. +It is tested against SQLite itself and a model of the document; `tests/TEST-RESULTS.md` holds the record. It is a project of [Proven Commons](https://github.com/Proven-Commons/mission), built to the verified bar published there. -## Claims to prove +This reads SQLite files. It is not SQLite, and it is not a database engine. -- The parser returns an error instead of reading outside the file. -- A record it reads stays inside its page. +Status: in progress. The specification has not yet been approved by a reviewer apart from its author, so this is not verified. -## What is not in this slice +## Use it -- Writing SQLite files. -- Querying. This is a file reader only. -- Any claim about a full database engine. +From Lean, on `List UInt8`, with the structures of `Spec/Defs.lean`: -## Layout, when the first slice lands +- `Sqlite.readVarint : List UInt8 → Option (Nat × Nat × List UInt8)`: the varint at the start of the input, its length, and the rest. +- `Sqlite.readHeader : List UInt8 → Option Header`: the database header at the start of a file. +- `Sqlite.readPage : Header → List UInt8 → Nat → Option Page`: page `n` of a file, as a b-tree page. +- `Sqlite.readRecord : List UInt8 → Option (List Value)`: the values of a record. +- `Sqlite.localOf : PageType → Nat → Nat → Nat`: how many bytes of a payload stay on a page. -The layout follows the Proven Commons template in the mission repo, under bar/template. Spec.lean holds the statements a reviewer approves. Proofs.lean holds the proofs under the same names. Main.lean holds the command line tool used by the tests. The bar check runs in CI on every change. +`SPEC.md` says what each function does. + +## What is proved + +The statements are in `Spec.lean`. The definitions they use are in `Spec/Defs.lean`, written from sections 1.2, 1.3, 1.6 and 2.1 of the document, and the proofs are in `Proofs/`. + +| Claim | Theorem | +|-------|---------| +| The varint reader accepts exactly an input that starts with a varint of one to nine bytes, and gives its value, its length and what follows. | `Spec.readVarint_iff` | +| The header reader accepts exactly a file that starts with the 100 bytes of a header this reader accepts, and gives that header. | `Spec.readHeader_iff` | +| The page reader gives a page exactly when page `n` is in the file and holds it: the page header, then one cell pointer per cell, then each cell's bytes where its pointer says, within the usable size. | `Spec.readPage_iff` | +| The usable part of every page the reader gives is in the file, and every cell pointer lies after the cell pointer array and before the end of the usable size. | `Spec.readPage_inside` | +| The record reader accepts exactly a record of 2.1 and gives its values. | `Spec.readRecord_iff` | +| Examples worked by hand, and the header SQLite 3.45.1 wrote for a real database, read field by field. Thirteen inputs that each break a rule are refused. | `Spec.examples`, `Spec.rejects` | + +The headers this reader accepts are those whose page size is a power of two from 512 to 65536, whose read version is 1 or 2, whose usable size is at least 480, whose payload fractions are 64, 32 and 32, whose schema format is 0 to 4, whose text encoding is 1, 2 or 3 (or 0 in a database with no schema yet, as SQLite writes it), and whose 20 reserved bytes are zero. + +## What is not proved + +- **Overflow pages.** A payload that spills is given as its local part and the number of its first overflow page. The chain is not followed, so a record that spills is not read; the tests follow it themselves. +- **The b-tree.** Each page is read alone. Nothing checks that child pointers name pages in the file, that keys are in order, or that each page is reached once. +- **Free space.** The freeblock list and the count of fragmented bytes are read but not followed or checked, and nothing checks that cells do not overlap each other or the freeblocks. +- **Other pages.** The freelist, pointer-map pages, the lock-byte page, the WAL and the rollback journal. A page of another kind is refused when it does not read as a b-tree page, but one may happen to read as one. +- **Header fields that need the rest of the file.** The in-header database size, the freelist fields, the schema cookie and the largest root page are read but not checked against the file. +- **Values.** Strings are given as bytes, not checked as UTF-8 or UTF-16; floats are given as their 64 bits. +- **The schema**, and writing a database. +- **Time and memory.** No proof bounds them. The reader works on a list of bytes, so reading page `n` walks the file up to that page. +- **Reading files, requests and printing answers in `Main.lean`.** The differential tests cover them. + +## What you must trust + +- The Lean kernel and the five external kernels shipped with Lean, which all check the proofs. +- `lake comparator`, which builds the statements and the proofs in a sandbox, exports them, and checks that each proof proves exactly its statement. +- bubblewrap and the Linux kernel's namespaces, which keep the untrusted build from tampering with that check. +- The Lean compiler, with the C compiler and linker shipped with it, and the runtime, which turn the proved definitions into the program that runs. +- Lean's core library, including the compiled code it puts in place of some definitions the proofs reason about, such as natural number and integer arithmetic, `UInt8`, and lists. +- The operating system and hardware it runs on. +- For the tests only: the model in `tests/run_tests.py`, and SQLite, which the record names. + +This project has no dependencies beyond Lean itself. + +## Re-check it + +Needs Linux, [elan](https://github.com/leanprover/elan), bubblewrap (`apt install bubblewrap`) and Python 3 with its `sqlite3` module. + +``` +lake build +lake comparator --paranoid +python3 tests/run_tests.py +python3 tests/plant_bugs.py +``` + +`lake comparator --paranoid` builds the statements and the proofs in a sandbox. It checks that each theorem in `Proofs/` proves exactly the statement in `Spec.lean`, using only the axioms `propext`, `Classical.choice`, and `Quot.sound`, and replays it through every kernel shipped with Lean. + +## Layout + +- `Sqlite/Basic.lean`: the code under proof. +- `Spec.lean`: the statements, with `sorry` for proofs. A reviewer approves this file. +- `Spec/Defs.lean`: the definitions the statements use, written from the document. +- `Spec/Examples.lean`: the inputs of the examples: a header SQLite wrote, and a one-page file. +- `Proofs.lean` and `Proofs/`: the proofs. +- `Main.lean`: the command-line tool the tests drive. +- `lakefile.toml`: the build. It names the package and its version, its libraries and executable, and the default targets, and sets nothing else. +- `comparator.json`: which theorems `lake comparator` checks, and which axioms it allows. +- `bar.toml`: the record for the bar check: status, authors, approvals, and where things are. +- `SPEC.md`: the specification in plain words. +- `tests/`: the differential tests and the planted bugs. diff --git a/SPEC.md b/SPEC.md new file mode 100644 index 0000000..1f82873 --- /dev/null +++ b/SPEC.md @@ -0,0 +1,112 @@ +# Specification: sqlite + +The SQLite database file format ("Database File Format", https://www.sqlite.org/fileformat2.html) is how every SQLite database is stored. This library reads three parts of it: + +- the 100-byte database header (1.3); +- one b-tree page: its page header, its cell pointer array and its cells, with the part of each payload kept on the page (1.6); +- records, the format of every row and index entry (2.1), and the varints they and the cells are made of (1.6). + +It does not follow overflow chains, read the freelist, pointer-map pages or the WAL, or check that the pages form a b-tree. "Not in this slice" below lists what else it leaves out. + +A reviewer who did not write this specification approves it. The approval also covers `Spec.lean`, `Spec/Defs.lean`, `Spec/Examples.lean`, and the other files that `bar.toml` lists. + +## Varints + +`Spec.Varint` is a varint of 1.6: "either zero or more bytes which have the high-order bit set followed by a single byte with the high-order bit clear, or nine bytes, whichever is shorter". So one to nine bytes, and a ninth byte may have its high bit set. `Spec.varintValue` is its value: "The lower seven bits of each of the first eight bytes and all 8 bits of the ninth byte are used to reconstruct the 64-bit twos-complement integer", most significant first. The value is kept as a number from 0 to 2^64 - 1; `Spec.toInt64` reads it as a signed 64-bit integer where 2.1 and 1.6 ask for one (a rowid). + +**`Sqlite.readVarint`** accepts exactly the inputs that start with a varint, and gives its value, its length and the bytes after it. + +## The database header + +`Spec.Header` has the fields of the table in 1.3, in its order, and `Spec.headerBytes` writes them as their 100 bytes: the header string "SQLite format 3" and a NUL, the page size in two bytes (65536 written as 1), the file format write and read versions, the reserved space, the payload fractions 64, 32 and 32, the 4-byte fields, 20 zero bytes, the version-valid-for number and the SQLite version number. Every multibyte field is big-endian. + +`Spec.HeaderValid` says which headers this reader accepts. Each rule is a "must" of 1.3, or a value 1.3 says a reader cannot use: + +- the page size is a power of two from 512 to 65536 (1.3.2); +- the read version is 1 or 2: "If a database file with a read version greater than 2 is encountered, then that database cannot be read" (1.3.3); any write version is accepted, since it only makes the file read-only; +- the usable size, the page size less the reserved space, is at least 480 (1.3.4); +- the payload fractions are 64, 32 and 32 (1.3.5); +- the schema format is 0 to 4 (1.3.10: formats 1 to 4, and 0 for a database with no schema); +- the text encoding is 1, 2 or 3, for UTF-8, UTF-16le and UTF-16be (1.3.13), or 0 when the schema format is 0: SQLite writes both as 0 while a database has no schema, so no text has been stored yet, and reads such a file; +- the 20 bytes "reserved for expansion" are zero (1.3.17); +- every field is small enough for its bytes. + +**`Sqlite.readHeader`** accepts exactly the inputs that start with `Spec.headerBytes` of a header in `Spec.HeaderValid`, and gives that header. Since `Spec.headerBytes` is a function of the header, each accepted header comes from exactly one string of 100 bytes. + +## B-tree pages + +A file is a list of pages of the page size, numbered from 1 (1.2); `Spec.pageOf` is page `n`. Page 1 starts with the database header, so its b-tree page header starts at offset 100; on every other page it starts at 0 (1.6). + +**The page header.** `Spec.PageHeader` is the b-tree page header of 1.6: the page type (2, 5, 10 or 13: interior index, interior table, leaf index, leaf table), the first freeblock, the number of cells, the start of the cell content area ("A zero value for this integer is interpreted as 65536"), the number of fragmented free bytes, and the right-most pointer, which interior pages have and leaf pages do not. `Spec.pageHeaderBytes` writes it as its 8 bytes, or 12 on an interior page. + +**The cell pointer array.** One 2-byte offset per cell, from the start of the page, after the page header. + +**Cells.** `Spec.CellEncodes` writes each kind of cell of "B-tree Cell Format": + +- a table leaf cell: the payload size as a varint, the rowid as a varint, the part of the payload kept on the page, and the first overflow page when the payload spills; +- a table interior cell: a 4-byte left child page number and the rowid as a varint; +- an index leaf cell: the payload size, the part kept on the page, and the overflow page; +- an index interior cell: the left child, the payload size, the part kept on the page, and the overflow page. + +How much of a payload of `P` bytes stays on the page is `Spec.localSize`, the rule of 1.6 in its own words: with `U` the usable size, `X` is `U-35` on a table leaf page and `((U-12)*64/255)-23` on an index page, `M` is `((U-12)*32/255)-23`, and `K` is `M+((P-M)%(U-4))`. All `P` bytes stay when `P<=X`; else the first `K` when `K<=X`; else the first `M`. When some of the payload spills, a 4-byte page number follows the part on the page. + +`Spec.PageEncodes` says that page `n`, with the file's header, holds a page: + +- the page header is valid: every field fits its bytes, and the right-most pointer is there exactly on interior pages; +- the page header and then the cell pointer array are at the page header's offset, one pointer per cell; +- the array ends no later than the cell content area starts, and that area starts no later than the end of the usable size; +- each cell starts in the cell content area, its bytes, at the offset its pointer gives, are those of `Spec.CellEncodes`, and they end within the usable size, before the reserved space. + +**`Sqlite.readPage`** gives a page exactly when page `n` is in the file and holds it, as `Spec.PageEncodes` says; it refuses page 0 and a page past the end of the file. So nothing it gives lies outside the file or its page: the usable part of the page is in the file, and every cell starts after the cell pointer array and ends within the usable size. A cell's payload is its local part only, with the number of the first overflow page when the payload spills. + +## Records + +A record (2.1) is a header and a body. The header is a varint giving the size of the header in bytes, the varint itself included, then one varint per column, its serial type. The body is the contents of the columns, one after another. `Spec.contentSize` gives the content size of each serial type, from the table in 2.1, and `Spec.valueOf` the value: + +| Serial type | Content size | Value | +|---|---|---| +| 0 | 0 | NULL | +| 1, 2, 3, 4, 5, 6 | 1, 2, 3, 4, 6, 8 | a big-endian two's complement integer | +| 7 | 8 | a 64-bit IEEE 754 float, kept as its 64 bits | +| 8, 9 | 0 | the integers 0 and 1 | +| 10, 11 | | "will never appear in a well-formed database file": refused | +| N ≥ 12, even | (N-12)/2 | a BLOB | +| N ≥ 13, odd | (N-13)/2 | a string, as its bytes in the database's text encoding | + +`Spec.RecordEncodes vs p` says that `p` is a record of the values `vs`: at least one column ("one or more additional varints, one per column"), a header size that is the size of the header, and a body that is exactly the contents of the columns. + +**`Sqlite.readRecord`** accepts exactly the records and gives their values. + +## What must always hold + +Each property is a theorem in `Spec.lean`, under the name given. + +| Property | Theorem | +|----------|---------| +| The varint reader accepts exactly an input that starts with a varint, and gives its value, its length and what follows. | `Spec.readVarint_iff` | +| The header reader accepts exactly a file that starts with the 100 bytes of a header in `Spec.HeaderValid`, and gives that header. | `Spec.readHeader_iff` | +| The page reader gives a page exactly when page `n` is in the file and holds it, as `Spec.PageEncodes` says. | `Spec.readPage_iff` | +| The usable part of every page the reader gives is in the file, and every cell pointer lies after the cell pointer array and before the end of the usable size. | `Spec.readPage_inside` | +| The record reader accepts exactly a record and gives its values. | `Spec.readRecord_iff` | +| Examples worked by hand: varints of one, two and nine bytes; the local size of a payload of 10000 bytes on a table leaf page of usable size 4096; a record of an integer and a string; the header SQLite 3.45.1 wrote for a database, field by field; and a one-page file with one row. | `Spec.examples` | +| Inputs that each break a rule: a varint cut short; a header without its header string, with a page size of 768, a read version of 3, a usable size of 479, a text encoding of 0 with a schema, or a payload fraction of 63; page 2 of a one-page file, and page 0; serial type 10, a body longer than its columns, a header size larger than the record, and a record with no column. | `Spec.rejects` | + +## Not in this slice + +- **Overflow pages.** A payload that spills is given as its local part and the number of its first overflow page; the chain is not followed, so a record that spills is not read. +- **The b-tree.** Each page is read alone. Nothing checks that child pointers name pages in the file, that keys are in order, or that each page is reached once. +- **Free space.** The freeblock list and the count of fragmented bytes are read but not followed or checked, and nothing checks that cells do not overlap each other or the freeblocks. +- **Other pages.** The freelist, pointer-map pages, the lock-byte page, and the WAL and rollback journal. A page of another kind is refused when it does not read as a b-tree page, but one may happen to read as one. +- **Header fields that need the rest of the file.** The in-header database size, the freelist fields, the schema cookie and the largest root page are read but not checked against the file. +- **Values.** Strings are given as bytes and not checked as UTF-8 or UTF-16; floats are given as their 64 bits. +- **The schema.** Which b-tree belongs to which table, and the SQL in `sqlite_schema`. +- **Writing a database.** + +## The command-line tool + +`Main.lean` has two modes: + +- `sqlite dump FILE` prints the header of a database file, then each page as a b-tree page (or `page N FAIL`), and the record of each cell whose payload is all on its page. It exits with 1 when the file does not start with a header this reader accepts, and with 2 on wrong arguments or a file it cannot read. +- `sqlite serve` answers requests from the differential tests, one per line. The header comment of `Main.lean` gives the protocol. + +Every answer comes from the functions above. diff --git a/Spec.lean b/Spec.lean new file mode 100644 index 0000000..c1b95ff --- /dev/null +++ b/Spec.lean @@ -0,0 +1,110 @@ +import Sqlite +import Spec.Defs +import Spec.Examples + +/-! + The statements a reviewer approves. Each `sorry` is proved under the + same name in `Proofs/`, and `lake comparator` checks that each proof + proves exactly the statement here. + + `Spec/Defs.lean` writes the file format down from "Database File + Format" (https://www.sqlite.org/fileformat2.html). The code is in + `Sqlite/Basic.lean`. +-/ + +namespace Spec + +/-- The varint reader accepts exactly a varint of 1.6 at the start of + its input, and gives its value, its length and what follows. -/ +theorem readVarint_iff (b : List UInt8) (v k : Nat) (rest : List UInt8) : + Sqlite.readVarint b = some (v, k, rest) ↔ + ∃ bs, Varint bs ∧ varintValue bs = v ∧ bs.length = k ∧ b = bs ++ rest := by + sorry + +/-- The header reader accepts exactly a file that starts with the 100 + bytes of a header this reader accepts, and gives that header. -/ +theorem readHeader_iff (file : List UInt8) (h : Header) : + Sqlite.readHeader file = some h ↔ HeaderValid h ∧ ∃ rest, file = headerBytes h ++ rest := by + sorry + +/-- The page reader gives a b-tree page exactly when the `n`th page is + in the file and holds it, as `PageEncodes` says: the page header, + the cell pointers, and each cell's bytes where its pointer says. -/ +theorem readPage_iff (h : Header) (file : List UInt8) (n : Nat) (pg : Page) : + Sqlite.readPage h file n = some pg ↔ + 1 ≤ n ∧ n * h.pageSize ≤ file.length ∧ PageEncodes h n pg (pageOf h file n) := by + sorry + +/-- Nothing the page reader gives lies outside the file or its page: + the page is in the file, and every cell starts after the cell pointer + array and before the end of the usable size. -/ +theorem readPage_inside (h : Header) (file : List UInt8) (n : Nat) (pg : Page) + (hp : Sqlite.readPage h file n = some pg) : + (n - 1) * h.pageSize + h.usable ≤ file.length ∧ + ∀ p ∈ pg.pointers, + headerOffset n + pg.header.type.headerSize + 2 * pg.pointers.length ≤ p ∧ p < h.usable := by + sorry + +/-- The record reader accepts exactly a record of 2.1, and gives its + values. -/ +theorem readRecord_iff (p : List UInt8) (vs : List Value) : + Sqlite.readRecord p = some vs ↔ RecordEncodes vs p := by + sorry + +/-- Worked by hand: + - varints: 0, 127, 128 in two bytes, and nine bytes 0xFF, which is + 2^64 - 1 (1.6); + - a payload of 10000 bytes on a table leaf page with a usable size of + 4096: X = 4061, M = 489, and K = 489 + 9511 mod 4092 = 1816, which + is no more than X, so 1816 bytes stay on the page (1.6); + - the record whose header is 03 01 13 (a header of 3 bytes, an 8-bit + integer and a string of 3 bytes) and whose body is 07 61 62 63 is + 7 and "abc" (2.1); + - the header SQLite wrote, in `sqliteHeaderBytes`, read field by + field (1.3); + - the page of `onePageFile`: a table leaf page with one cell, of + rowid 1, whose record is the integer 5 (1.6, 2.1). -/ +theorem examples : + Sqlite.readVarint [0x00] = some (0, 1, []) ∧ + Sqlite.readVarint [0x7F, 0x55] = some (127, 1, [0x55]) ∧ + Sqlite.readVarint [0x81, 0x00] = some (128, 2, []) ∧ + Sqlite.readVarint (List.replicate 9 0xFF) = some (2 ^ 64 - 1, 9, []) ∧ + Sqlite.localOf .tableLeaf 4096 10000 = 1816 ∧ + Sqlite.readRecord [0x03, 0x01, 0x13, 0x07, 0x61, 0x62, 0x63] = + some [.int 7, .text [0x61, 0x62, 0x63]] ∧ + Sqlite.readHeader sqliteHeaderBytes = some sqliteHeader ∧ + Sqlite.readPage sqliteHeader onePageFile 1 = + some { header := { type := .tableLeaf, firstFreeblock := 0, cellCount := 1, + contentStart := 507, fragmented := 0, rightmost := none }, + pointers := [507], + cells := [{ leftChild := none, payloadSize := some 3, rowid := some 1, + payload := [0x02, 0x01, 0x05], overflow := none }] } ∧ + Sqlite.readRecord [0x02, 0x01, 0x05] = some [.int 5] := by + sorry + +/-- Inputs the readers refuse, each for a rule: + - a varint that ends before its last byte (1.6); + - headers with the header string zeroed, a page size of 768, a read + version of 3, a usable size of 479, a text encoding of 0 with a schema, and a + maximum embedded payload fraction of 63 (1.3); + - page 2 of a one-page file, and page 0 (1.2); + - serial type 10, which "will never appear in a well-formed database + file", a body one byte longer than its columns, a header size + larger than the record, and a header with no column (2.1). -/ +theorem rejects : + Sqlite.readVarint [0x81, 0x80] = none ∧ + Sqlite.readHeader (List.replicate 16 0 ++ sqliteHeaderBytes.drop 16) = none ∧ + Sqlite.readHeader (headerBytes { sqliteHeader with pageSize := 768 }) = none ∧ + Sqlite.readHeader (headerBytes { sqliteHeader with readVersion := 3 }) = none ∧ + Sqlite.readHeader (headerBytes { sqliteHeader with reserved := 33 }) = none ∧ + Sqlite.readHeader (headerBytes { sqliteHeader with textEncoding := 0 }) = none ∧ + Sqlite.readHeader (sqliteHeaderBytes.set 21 63) = none ∧ + Sqlite.readPage sqliteHeader onePageFile 2 = none ∧ + Sqlite.readPage sqliteHeader onePageFile 0 = none ∧ + Sqlite.readRecord [0x02, 0x0A] = none ∧ + Sqlite.readRecord [0x02, 0x01, 0x07, 0x00] = none ∧ + Sqlite.readRecord [0x05, 0x01] = none ∧ + Sqlite.readRecord [0x01] = none := by + sorry + +end Spec diff --git a/Spec/Defs.lean b/Spec/Defs.lean new file mode 100644 index 0000000..deeddd4 --- /dev/null +++ b/Spec/Defs.lean @@ -0,0 +1,338 @@ +/- + The SQLite database file format, written from "Database File Format" + (https://www.sqlite.org/fileformat2.html): the database header (1.3), + b-tree pages and their cells (1.6), and the record format (2.1). + + Bytes are `List UInt8`. Multibyte integers in the header, the page + header, cell pointers and child pointers are big-endian (1.3, 1.6). + This file uses nothing from the code. +-/ + +namespace Spec + +/-! ## Integers -/ + +/-- `n` as `w` bytes, most significant first. -/ +def beBytes (w n : Nat) : List UInt8 := + (List.range w).reverse.map fun i => (n / 256 ^ i % 256).toUInt8 + +/-- The bytes as an unsigned integer, most significant first. -/ +def beNat (bs : List UInt8) : Nat := + bs.foldl (fun n b => n * 256 + b.toNat) 0 + +/-- A varint (1.6): "either zero or more bytes which have the high-order + bit set followed by a single byte with the high-order bit clear, or + nine bytes, whichever is shorter". So one to nine bytes; a ninth byte + may have its high bit set. -/ +def Varint (bs : List UInt8) : Prop := + 1 ≤ bs.length ∧ bs.length ≤ 9 ∧ (∀ b ∈ bs.dropLast, 128 ≤ b.toNat) ∧ + (bs.length < 9 → ∀ b ∈ bs.getLast?, b.toNat < 128) + +/-- A varint's value: "The lower seven bits of each of the first eight + bytes and all 8 bits of the ninth byte are used to reconstruct the + 64-bit twos-complement integer. Varints are big-endian." -/ +def varintValue (bs : List UInt8) : Nat := + if bs.length = 9 then + (bs.take 8).foldl (fun n b => n * 128 + b.toNat % 128) 0 * 256 + (bs.getD 8 0).toNat + else bs.foldl (fun n b => n * 128 + b.toNat % 128) 0 + +/-- A 64-bit two's complement integer, from its 64 bits as a number. -/ +def toInt64 (n : Nat) : Int := + if n < 2 ^ 63 then n else (n : Int) - 2 ^ 64 + +/-! ## The database header (1.3) -/ + +/-- The header string: "SQLite format 3" and a NUL (1.3.1). -/ +def magic : List UInt8 := + [0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00] + +/-- The fields of the database header, in the order of the table in + 1.3. The page size is the size in bytes. -/ +structure Header where + pageSize : Nat + writeVersion : Nat + readVersion : Nat + reserved : Nat + changeCounter : Nat + databaseSize : Nat + freelistTrunk : Nat + freelistCount : Nat + schemaCookie : Nat + schemaFormat : Nat + cacheSize : Nat + largestRoot : Nat + textEncoding : Nat + userVersion : Nat + incrementalVacuum : Nat + applicationId : Nat + versionValidFor : Nat + sqliteVersion : Nat + deriving DecidableEq, Repr + +/-- The usable size of a page: the page size less the reserved space + (1.3.4). -/ +def Header.usable (h : Header) : Nat := h.pageSize - h.reserved + +/-- The header as its 100 bytes (1.3). The page size 65536 is written + as 1 (1.3.2); the payload fractions are 64, 32 and 32 (1.3.5); the + 20 bytes at offset 72 are zero (1.3.17). -/ +def headerBytes (h : Header) : List UInt8 := + magic ++ beBytes 2 (if h.pageSize = 65536 then 1 else h.pageSize) ++ + [h.writeVersion.toUInt8, h.readVersion.toUInt8, h.reserved.toUInt8, 64, 32, 32] ++ + beBytes 4 h.changeCounter ++ beBytes 4 h.databaseSize ++ beBytes 4 h.freelistTrunk ++ + beBytes 4 h.freelistCount ++ beBytes 4 h.schemaCookie ++ beBytes 4 h.schemaFormat ++ + beBytes 4 h.cacheSize ++ beBytes 4 h.largestRoot ++ beBytes 4 h.textEncoding ++ + beBytes 4 h.userVersion ++ beBytes 4 h.incrementalVacuum ++ beBytes 4 h.applicationId ++ + List.replicate 20 0 ++ beBytes 4 h.versionValidFor ++ beBytes 4 h.sqliteVersion + +/-- The headers this reader accepts: + - a page size that is a power of two from 512 to 65536 (1.2, 1.3.2); + - a read version of 1 or 2: "If a database file with a read version + greater than 2 is encountered, then that database cannot be read" + (1.3.3); any write version, which only makes the file read-only; + - a usable size of at least 480 (1.3.4); + - schema format 0 to 4 (1.3.10: formats 1 to 4, and 0 for a database + with no schema); + - text encoding 1, 2 or 3 (1.3.13), or 0 when the schema format is 0: + SQLite writes both as 0 while a database has no schema, so no + text has been stored yet; + - every field small enough for its bytes. -/ +def HeaderValid (h : Header) : Prop := + (∃ k, 9 ≤ k ∧ k ≤ 16 ∧ h.pageSize = 2 ^ k) ∧ + h.writeVersion < 256 ∧ (h.readVersion = 1 ∨ h.readVersion = 2) ∧ + h.reserved < 256 ∧ 480 ≤ h.pageSize - h.reserved ∧ + h.changeCounter < 2 ^ 32 ∧ h.databaseSize < 2 ^ 32 ∧ h.freelistTrunk < 2 ^ 32 ∧ + h.freelistCount < 2 ^ 32 ∧ h.schemaCookie < 2 ^ 32 ∧ h.schemaFormat ≤ 4 ∧ + h.cacheSize < 2 ^ 32 ∧ h.largestRoot < 2 ^ 32 ∧ + (h.textEncoding = 1 ∨ h.textEncoding = 2 ∨ h.textEncoding = 3 ∨ + h.textEncoding = 0 ∧ h.schemaFormat = 0) ∧ + h.userVersion < 2 ^ 32 ∧ h.incrementalVacuum < 2 ^ 32 ∧ h.applicationId < 2 ^ 32 ∧ + h.versionValidFor < 2 ^ 32 ∧ h.sqliteVersion < 2 ^ 32 + +/-! ## B-tree pages (1.6) -/ + +/-- The four kinds of b-tree page, by their flag byte. -/ +inductive PageType where + | indexInterior | tableInterior | indexLeaf | tableLeaf + deriving DecidableEq, Repr + +/-- "A value of 2 (0x02) means the page is an interior index b-tree + page. A value of 5 (0x05) means the page is an interior table b-tree + page. A value of 10 (0x0a) means the page is a leaf index b-tree + page. A value of 13 (0x0d) means the page is a leaf table b-tree + page." -/ +def PageType.code : PageType → Nat + | .indexInterior => 2 + | .tableInterior => 5 + | .indexLeaf => 10 + | .tableLeaf => 13 + +def PageType.interior : PageType → Bool + | .indexInterior | .tableInterior => true + | .indexLeaf | .tableLeaf => false + +/-- The b-tree page header (1.6). The start of the cell content area is + 1 to 65536; "A zero value for this integer is interpreted as 65536". + The right-most pointer is there for interior pages only. -/ +structure PageHeader where + type : PageType + firstFreeblock : Nat + cellCount : Nat + contentStart : Nat + fragmented : Nat + rightmost : Option Nat + deriving DecidableEq, Repr + +/-- The page header as its 8 bytes, or 12 for an interior page. -/ +def pageHeaderBytes (ph : PageHeader) : List UInt8 := + [ph.type.code.toUInt8] ++ beBytes 2 ph.firstFreeblock ++ beBytes 2 ph.cellCount ++ + beBytes 2 (ph.contentStart % 65536) ++ [ph.fragmented.toUInt8] ++ + match ph.rightmost with + | some p => beBytes 4 p + | none => [] + +/-- Every field small enough for its bytes, and a right-most pointer + exactly on interior pages. -/ +def PageHeaderValid (ph : PageHeader) : Prop := + ph.firstFreeblock < 2 ^ 16 ∧ ph.cellCount < 2 ^ 16 ∧ + 1 ≤ ph.contentStart ∧ ph.contentStart ≤ 65536 ∧ ph.fragmented < 256 ∧ + (match ph.rightmost with + | some p => ph.type.interior = true ∧ p < 2 ^ 32 + | none => ph.type.interior = false) + +/-- "Let X be U-35" on a table leaf page, "Let X be ((U-12)*64/255)-23" + on an index page: the most payload a cell keeps on its page. -/ +def maxLocal (t : PageType) (U : Nat) : Nat := + if t = .tableLeaf then U - 35 else (U - 12) * 64 / 255 - 23 + +/-- "Let M be ((U-12)*32/255)-23". -/ +def minLocal (U : Nat) : Nat := (U - 12) * 32 / 255 - 23 + +/-- How much of a payload of `P` bytes a cell keeps on its page (1.6): + "If P<=X then all P bytes of payload are stored directly on the + btree page without overflow. If P>X and K<=X then the first K bytes + of P are stored on the btree page and the remaining P-K bytes are + stored on overflow pages. If P>X and K>X then the first M bytes of P + are stored on the btree page", where K is M+((P-M)%(U-4)). -/ +def localSize (t : PageType) (U P : Nat) : Nat := + if P ≤ maxLocal t U then P + else + let M := minLocal U + let K := M + (P - M) % (U - 4) + if K ≤ maxLocal t U then K else M + +/-- A cell, with what it holds on its page. A table leaf cell has a + payload size, a rowid and the payload; a table interior cell a left + child and a rowid; an index leaf cell a payload size and the payload; + an index interior cell a left child, a payload size and the payload. + `payload` is the part of the payload on the page, and `overflow` the + first overflow page when the rest spills. -/ +structure Cell where + leftChild : Option Nat + payloadSize : Option Nat + rowid : Option Int + payload : List UInt8 + overflow : Option Nat + deriving DecidableEq, Repr + +/-- The payload part of a cell, after its payload size: the bytes kept + on the page, then "A 4-byte big-endian integer page number for the + first page of the overflow page list - omitted if all payload fits on + the b-tree page". -/ +def PayloadEncodes (t : PageType) (U P : Nat) (kept : List UInt8) (overflow : Option Nat) + (bs : List UInt8) : Prop := + kept.length = localSize t U P ∧ + if localSize t U P < P then ∃ o, o < 2 ^ 32 ∧ overflow = some o ∧ bs = kept ++ beBytes 4 o + else overflow = none ∧ bs = kept + +/-- `bs` are the bytes of the cell `c` on a page of type `t` and usable + size `U` (1.6, "B-tree Cell Format"). -/ +def CellEncodes (t : PageType) (U : Nat) (c : Cell) (bs : List UInt8) : Prop := + match t with + | .tableLeaf => + ∃ vp vr rest, Varint vp ∧ Varint vr ∧ bs = vp ++ vr ++ rest ∧ + c.leftChild = none ∧ c.payloadSize = some (varintValue vp) ∧ + c.rowid = some (toInt64 (varintValue vr)) ∧ + PayloadEncodes t U (varintValue vp) c.payload c.overflow rest + | .tableInterior => + ∃ k vr, k < 2 ^ 32 ∧ Varint vr ∧ bs = beBytes 4 k ++ vr ∧ + c = { leftChild := some k, payloadSize := none, rowid := some (toInt64 (varintValue vr)), + payload := [], overflow := none } + | .indexLeaf => + ∃ vp rest, Varint vp ∧ bs = vp ++ rest ∧ + c.leftChild = none ∧ c.payloadSize = some (varintValue vp) ∧ c.rowid = none ∧ + PayloadEncodes t U (varintValue vp) c.payload c.overflow rest + | .indexInterior => + ∃ k vp rest, k < 2 ^ 32 ∧ Varint vp ∧ bs = beBytes 4 k ++ vp ++ rest ∧ + c.leftChild = some k ∧ c.payloadSize = some (varintValue vp) ∧ c.rowid = none ∧ + PayloadEncodes t U (varintValue vp) c.payload c.overflow rest + +/-- A b-tree page as this reader gives it: its header, its cell + pointers, and its cells in the order of the pointers. -/ +structure Page where + header : PageHeader + pointers : List Nat + cells : List Cell + deriving DecidableEq, Repr + +/-- The cell pointer array: "K 2-byte integer offsets to the cell + contents". -/ +def pointerBytes (ps : List Nat) : List UInt8 := + ps.flatMap (beBytes 2) + +/-- Where the page header starts: after the 100-byte database header on + page 1, at the start of every other page (1.6). -/ +def headerOffset (n : Nat) : Nat := if n = 1 then 100 else 0 + +/-- The size of the page header: "8 bytes in size for leaf pages and 12 + bytes for interior pages". -/ +def PageType.headerSize (t : PageType) : Nat := if t.interior then 12 else 8 + +/-- `bytes`, the `n`th page of a file with header `h`, holds the b-tree + page `pg` (1.6): + - the page header at its offset, then the cell pointer array, with + one pointer per cell; + - the array ends no later than the cell content area starts, and that + area starts no later than the end of the usable size, the "reserved + region" coming after it; + - each cell starts in the cell content area, its bytes are those of + the cell, and they end within the usable size. -/ +def PageEncodes (h : Header) (n : Nat) (pg : Page) (bytes : List UInt8) : Prop := + let ph := pg.header + let off := headerOffset n + let U := h.usable + PageHeaderValid ph ∧ ph.cellCount = pg.pointers.length ∧ + pg.cells.length = pg.pointers.length ∧ + (∀ p ∈ pg.pointers, p < 2 ^ 16) ∧ + (bytes.drop off).take (ph.type.headerSize + 2 * pg.pointers.length) = + pageHeaderBytes ph ++ pointerBytes pg.pointers ∧ + off + ph.type.headerSize + 2 * pg.pointers.length ≤ ph.contentStart ∧ ph.contentStart ≤ U ∧ + ∀ pc ∈ pg.pointers.zip pg.cells, ∃ bs, + CellEncodes ph.type U pc.2 bs ∧ + ph.contentStart ≤ pc.1 ∧ pc.1 + bs.length ≤ U ∧ + (bytes.drop pc.1).take bs.length = bs + +/-- The `n`th page of a file, numbered from 1 (1.2). -/ +def pageOf (h : Header) (file : List UInt8) (n : Nat) : List UInt8 := + (file.drop ((n - 1) * h.pageSize)).take h.pageSize + +/-! ## The record format (2.1) -/ + +/-- A value of a record. A float is kept as its 64 bits, a string as its + bytes in the database's text encoding. -/ +inductive Value where + | null + | int (z : Int) + | real (bits : Nat) + | blob (bs : List UInt8) + | text (bs : List UInt8) + deriving DecidableEq, Repr + +/-- The content size of each serial type ("Serial Type Codes Of The + Record Format"). Types 10 and 11 are "Reserved for internal use" and + "will never appear in a well-formed database file". -/ +def contentSize (t : Nat) : Option Nat := + match t with + | 0 => some 0 | 1 => some 1 | 2 => some 2 | 3 => some 3 | 4 => some 4 + | 5 => some 6 | 6 => some 8 | 7 => some 8 | 8 => some 0 | 9 => some 0 + | 10 | 11 => none + | n => some (if n % 2 = 0 then (n - 12) / 2 else (n - 13) / 2) + +/-- A big-endian two's complement integer of the bytes. -/ +def signedOf (bs : List UInt8) : Int := + if beNat bs < 2 ^ (8 * bs.length - 1) then beNat bs else (beNat bs : Int) - 2 ^ (8 * bs.length) + +/-- The value of a column with serial type `t` and content `c`: NULL; + integers of 1, 2, 3, 4, 6 and 8 bytes; "a big-endian IEEE 754-2008 + 64-bit floating point number"; the integers 0 and 1; a BLOB; a + string. -/ +def valueOf (t : Nat) (c : List UInt8) : Value := + match t with + | 0 => .null + | 1 | 2 | 3 | 4 | 5 | 6 => .int (signedOf c) + | 7 => .real (beNat c) + | 8 => .int 0 + | 9 => .int 1 + | n => if n % 2 = 0 then .blob c else .text c + +/-- The serial types `ts` and contents `cs` encode the values `vs`, one + column each. -/ +def ColumnsEncode : List (List UInt8) → List (List UInt8) → List Value → Prop + | [], [], [] => True + | t :: ts, c :: cs, v :: vs => + Varint t ∧ contentSize (varintValue t) = some c.length ∧ v = valueOf (varintValue t) c ∧ + ColumnsEncode ts cs vs + | _, _, _ => False + +/-- `p` is a record of the values `vs` (2.1): "A record contains a + header and a body, in that order. The header begins with a single + varint which determines the total number of bytes in the header. The + varint value is the size of the header in bytes including the size + varint itself. Following the size varint are one or more additional + varints, one per column." The body is the contents of the columns, + one after another, and nothing else. -/ +def RecordEncodes (vs : List Value) (p : List UInt8) : Prop := + vs ≠ [] ∧ ∃ hs ts cs, Varint hs ∧ varintValue hs = hs.length + ts.flatten.length ∧ + ColumnsEncode ts cs vs ∧ p = hs ++ ts.flatten ++ cs.flatten + +end Spec diff --git a/Spec/Examples.lean b/Spec/Examples.lean new file mode 100644 index 0000000..eec99b0 --- /dev/null +++ b/Spec/Examples.lean @@ -0,0 +1,37 @@ +import Spec.Defs + +/-! + The inputs of `Spec.examples` and `Spec.rejects`. +-/ + +namespace Spec + +/-- The first 100 bytes of a database that SQLite 3.45.1 wrote: 512-byte + pages, one table of six rows, three pages in all. -/ +def sqliteHeaderBytes : List UInt8 := + [0x53, 0x51, 0x4C, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6F, 0x72, 0x6D, 0x61, 0x74, 0x20, 0x33, 0x00, + 0x02, 0x00, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x2E, 0x76, 0x89] + +/-- That header's fields: change counter 2, three pages, schema cookie 1, + schema format 4, UTF-8, valid for change 2, written by SQLite + 3045001. -/ +def sqliteHeader : Header := + { pageSize := 512, writeVersion := 1, readVersion := 1, reserved := 0, changeCounter := 2, + databaseSize := 3, freelistTrunk := 0, freelistCount := 0, schemaCookie := 1, + schemaFormat := 4, cacheSize := 0, largestRoot := 0, textEncoding := 1, userVersion := 0, + incrementalVacuum := 0, applicationId := 0, versionValidFor := 2, sqliteVersion := 3045001 } + +/-- A one-page file, written by hand: that header, then on page 1 a + table leaf page with one cell at offset 507, the last five bytes of + the page. The cell has a payload of 3 bytes, rowid 1, and the payload + 02 01 05: a record of one 8-bit integer, 5. -/ +def onePageFile : List UInt8 := + sqliteHeaderBytes ++ [0x0D, 0x00, 0x00, 0x00, 0x01, 0x01, 0xFB, 0x00] ++ [0x01, 0xFB] ++ + List.replicate 397 0 ++ [0x03, 0x01, 0x02, 0x01, 0x05] + +end Spec diff --git a/Sqlite.lean b/Sqlite.lean new file mode 100644 index 0000000..258e37e --- /dev/null +++ b/Sqlite.lean @@ -0,0 +1 @@ +import Sqlite.Basic diff --git a/Sqlite/Basic.lean b/Sqlite/Basic.lean new file mode 100644 index 0000000..4c967a3 --- /dev/null +++ b/Sqlite/Basic.lean @@ -0,0 +1,258 @@ +/- + Reading the SQLite database file format: the database header, one + b-tree page with its cells, and records. The structures the reader + gives back are those of `Spec/Defs.lean`, and so is the header string; + the functions here are the code under proof, and share nothing else + with the specification. +-/ + +import Spec.Defs + +namespace Sqlite + +open Spec (Header PageType PageHeader Cell Page Value) + +/-- Splits off the first `w` bytes, if there are that many. Measures + only those `w`. -/ +def take? (w : Nat) (b : List UInt8) : Option (List UInt8 × List UInt8) := + let c := b.take w + if c.length = w then some (c, b.drop w) else none + +/-- The bytes as a big-endian unsigned integer. -/ +def bigEndian (bs : List UInt8) : Nat := + bs.foldl (fun n b => n * 256 + b.toNat) 0 + +/-- A `w`-byte big-endian integer at the start of the input, and the + rest. -/ +def readBE (w : Nat) (b : List UInt8) : Option (Nat × List UInt8) := + (take? w b).map fun (c, r) => (bigEndian c, r) + +/-- A varint at the start of the input (1.6): its value, how many bytes + it takes, and the rest. Each of the first eight bytes gives its low + seven bits and ends the varint when its high bit is clear; a ninth + byte gives all eight. -/ +def readVarint (b : List UInt8) : Option (Nat × Nat × List UInt8) := + go 0 0 b +where + go (acc i : Nat) : List UInt8 → Option (Nat × Nat × List UInt8) + | [] => none + | x :: rest => + if i = 8 then some (acc * 256 + x.toNat, 9, rest) + else if x.toNat < 128 then some (acc * 128 + x.toNat, i + 1, rest) + else go (acc * 128 + x.toNat % 128) (i + 1) rest + +/-! ## The database header -/ + +/-- Whether a page size is a power of two from 512 to 65536. -/ +def pageSizeOk (n : Nat) : Bool := + [512, 1024, 2048, 4096, 8192, 16384, 32768, 65536].contains n + +/-- The database header at the start of a file (1.3), or `none` when + the file does not start with one this reader accepts. -/ +def readHeader (file : List UInt8) : Option Header := do + let (m, r) ← take? 16 file + guard (m = Spec.magic) + let (ps, r) ← readBE 2 r + let pageSize := if ps = 1 then 65536 else ps + let (wv, r) ← readBE 1 r + let (rv, r) ← readBE 1 r + let (res, r) ← readBE 1 r + let (fr, r) ← take? 3 r + guard (fr = [64, 32, 32]) + let (changeCounter, r) ← readBE 4 r + let (databaseSize, r) ← readBE 4 r + let (freelistTrunk, r) ← readBE 4 r + let (freelistCount, r) ← readBE 4 r + let (schemaCookie, r) ← readBE 4 r + let (schemaFormat, r) ← readBE 4 r + let (cacheSize, r) ← readBE 4 r + let (largestRoot, r) ← readBE 4 r + let (textEncoding, r) ← readBE 4 r + let (userVersion, r) ← readBE 4 r + let (incrementalVacuum, r) ← readBE 4 r + let (applicationId, r) ← readBE 4 r + let (z, r) ← take? 20 r + guard (z = List.replicate 20 0) + let (versionValidFor, r) ← readBE 4 r + let (sqliteVersion, _) ← readBE 4 r + guard (pageSizeOk pageSize && (rv == 1 || rv == 2) && 480 ≤ pageSize - res && + schemaFormat ≤ 4 && (textEncoding == 1 || textEncoding == 2 || textEncoding == 3 || + textEncoding == 0 && schemaFormat == 0)) + pure { pageSize, writeVersion := wv, readVersion := rv, reserved := res, changeCounter, + databaseSize, freelistTrunk, freelistCount, schemaCookie, schemaFormat, cacheSize, + largestRoot, textEncoding, userVersion, incrementalVacuum, applicationId, + versionValidFor, sqliteVersion } + +/-! ## B-tree pages -/ + +/-- The page type of a flag byte (1.6). -/ +def pageType? : Nat → Option PageType + | 2 => some .indexInterior + | 5 => some .tableInterior + | 10 => some .indexLeaf + | 13 => some .tableLeaf + | _ => none + +/-- The b-tree page header at the start of the input, and the rest. -/ +def readPageHeader (b : List UInt8) : Option (PageHeader × List UInt8) := do + let (flag, r) ← readBE 1 b + let type ← pageType? flag + let (firstFreeblock, r) ← readBE 2 r + let (cellCount, r) ← readBE 2 r + let (cs, r) ← readBE 2 r + let (fragmented, r) ← readBE 1 r + if type.interior then + let (p, r) ← readBE 4 r + pure ({ type, firstFreeblock, cellCount, contentStart := if cs = 0 then 65536 else cs, + fragmented, rightmost := some p }, r) + else + pure ({ type, firstFreeblock, cellCount, contentStart := if cs = 0 then 65536 else cs, + fragmented, rightmost := none }, r) + +/-- The `k` two-byte cell pointers at the start of the input. -/ +def readPointers : Nat → List UInt8 → Option (List Nat) + | 0, _ => some [] + | k + 1, b => do + let (p, r) ← readBE 2 b + let ps ← readPointers k r + pure (p :: ps) + +/-- How many bytes of a payload of `P` bytes stay on a page of type `t` + and usable size `U` (1.6). -/ +def localOf (t : PageType) (U P : Nat) : Nat := + let x := match t with + | .tableLeaf => U - 35 + | _ => (U - 12) * 64 / 255 - 23 + let m := (U - 12) * 32 / 255 - 23 + if P ≤ x then P + else + let k := m + (P - m) % (U - 4) + if k ≤ x then k else m + +/-- A payload of `P` bytes at the start of the input: the bytes on the + page, the first overflow page if the payload spills, and how many + bytes that takes. -/ +def readPayload (t : PageType) (U P : Nat) (b : List UInt8) : + Option (List UInt8 × Option Nat × Nat) := do + let L := localOf t U P + let (loc, r) ← take? L b + if L < P then + let (o, _) ← readBE 4 r + pure (loc, some o, L + 4) + else pure (loc, none, L) + +/-- A 64-bit two's complement integer from its bits. -/ +def int64 (n : Nat) : Int := + if n < 9223372036854775808 then n else (n : Int) - 18446744073709551616 + +/-- The cell at the start of the input, on a page of type `t` and + usable size `U` (1.6), and how many bytes it takes. -/ +def readCell (t : PageType) (U : Nat) (b : List UInt8) : Option (Cell × Nat) := + match t with + | .tableLeaf => do + let (P, n1, r) ← readVarint b + let (rowid, n2, r) ← readVarint r + let (loc, ovf, n3) ← readPayload t U P r + pure ({ leftChild := none, payloadSize := some P, rowid := some (int64 rowid), + payload := loc, overflow := ovf }, n1 + n2 + n3) + | .tableInterior => do + let (k, r) ← readBE 4 b + let (rowid, n2, _) ← readVarint r + pure ({ leftChild := some k, payloadSize := none, rowid := some (int64 rowid), + payload := [], overflow := none }, 4 + n2) + | .indexLeaf => do + let (P, n1, r) ← readVarint b + let (loc, ovf, n3) ← readPayload t U P r + pure ({ leftChild := none, payloadSize := some P, rowid := none, + payload := loc, overflow := ovf }, n1 + n3) + | .indexInterior => do + let (k, r) ← readBE 4 b + let (P, n1, r) ← readVarint r + let (loc, ovf, n3) ← readPayload t U P r + pure ({ leftChild := some k, payloadSize := some P, rowid := none, + payload := loc, overflow := ovf }, 4 + n1 + n3) + +/-- The cells at the given offsets of a page, each starting no earlier + than `start` and ending within `U`. -/ +def readCells (t : PageType) (U start : Nat) (page : List UInt8) : List Nat → Option (List Cell) + | [] => some [] + | p :: ps => do + guard (start ≤ p) + let (c, size) ← readCell t U (page.drop p) + guard (p + size ≤ U) + let cs ← readCells t U start page ps + pure (c :: cs) + +/-- The `n`th page of a file with header `h`, as a b-tree page (1.6), + or `none` when it is not in the file or is not a b-tree page this + reader accepts. -/ +def readPage (h : Header) (file : List UInt8) (n : Nat) : Option Page := do + guard (1 ≤ n) + let page := (file.drop ((n - 1) * h.pageSize)).take h.pageSize + guard (page.length = h.pageSize) + let off := if n = 1 then 100 else 0 + let U := h.pageSize - h.reserved + let (ph, r) ← readPageHeader (page.drop off) + let ptrs ← readPointers ph.cellCount r + let hs := if ph.type.interior then 12 else 8 + guard (off + hs + 2 * ph.cellCount ≤ ph.contentStart && ph.contentStart ≤ U) + let cells ← readCells ph.type U ph.contentStart page ptrs + pure { header := ph, pointers := ptrs, cells } + +/-! ## Records -/ + +/-- The content size of a serial type (2.1); `none` for 10 and 11. -/ +def sizeOf (t : Nat) : Option Nat := + if t ≤ 4 then some t + else if t = 5 then some 6 + else if t = 6 ∨ t = 7 then some 8 + else if t = 8 ∨ t = 9 then some 0 + else if t = 10 ∨ t = 11 then none + else some ((t - 12) / 2) + +/-- A two's complement integer from big-endian bytes. -/ +def signed (c : List UInt8) : Int := + let n := bigEndian c + if 2 * n < 256 ^ c.length then n else (n : Int) - 256 ^ c.length + +/-- The value of a column of serial type `t` with contents `c`. -/ +def columnValue (t : Nat) (c : List UInt8) : Value := + if t = 0 then .null + else if t ≤ 6 then .int (signed c) + else if t = 7 then .real (bigEndian c) + else if t = 8 then .int 0 + else if t = 9 then .int 1 + else if t % 2 = 0 then .blob c + else .text c + +/-- The serial types in a record header, each a varint, to the end of + the header. `fuel` is the header's length. -/ +def readTypes : Nat → List UInt8 → Option (List Nat) + | _, [] => some [] + | 0, _ :: _ => none + | fuel + 1, b => do + let (t, _, r) ← readVarint b + let ts ← readTypes fuel r + pure (t :: ts) + +/-- The columns of a record body with these serial types; the body must + end with the last. -/ +def readColumns : List Nat → List UInt8 → Option (List Value) + | [], [] => some [] + | [], _ :: _ => none + | t :: ts, b => do + let size ← sizeOf t + let (c, r) ← take? size b + let vs ← readColumns ts r + pure (columnValue t c :: vs) + +/-- The record a payload holds (2.1), or `none` when it is not one. -/ +def readRecord (p : List UInt8) : Option (List Value) := do + let (hsize, n, r) ← readVarint p + guard (n ≤ hsize) + let (hdr, body) ← take? (hsize - n) r + let ts ← readTypes hdr.length hdr + guard (ts ≠ []) + readColumns ts body + +end Sqlite diff --git a/bar.toml b/bar.toml new file mode 100644 index 0000000..364e865 --- /dev/null +++ b/bar.toml @@ -0,0 +1,32 @@ +# Read by bar/check.py in Proven-Commons/mission. +# See playbook/verified-bar.md there for what each field means. + +[project] +name = "sqlite" +# "in-progress" until every bar item passes, then "verified". +status = "in-progress" +page = "README.md" +# GitHub handles of everyone who wrote the specification. It was drafted +# by Claude, an AI agent, for the org: agent fdn-claude, for the fdn-0 +# account, posting as zero-aska on GitHub. +spec_authors = ["claude", "fdn-0", "zero-aska"] +# The program people run. A change to it voids the approval. +entry_points = ["Main.lean"] + +[spec] +files = ["SPEC.md", "Spec.lean", "Spec/Defs.lean", "Spec/Examples.lean"] +# The code under proof: the functions the statements are about. +code = ["Sqlite.readVarint", "Sqlite.readHeader", "Sqlite.readPage", "Sqlite.readRecord", + "Sqlite.localOf"] + +# One entry per approval, added in a pull request the reviewer approves. +# [[spec.approval]] +# reviewer = "github-handle" +# date = "2026-11-02" +# sha256 = "..." +# record = "https://github.com/Proven-Commons/sqlite/pull/" + +[tests] +results = "tests/TEST-RESULTS.md" +command = "python3 tests/run_tests.py" +files = ["tests/run_tests.py", "tests/plant_bugs.py"] diff --git a/comparator.json b/comparator.json new file mode 100644 index 0000000..405fbb1 --- /dev/null +++ b/comparator.json @@ -0,0 +1,9 @@ +{ + "challenge_module": "Spec", + "solution_module": "Proofs", + "theorem_names": [ + "Spec.readVarint_iff", "Spec.readHeader_iff", "Spec.readPage_iff", "Spec.readPage_inside", + "Spec.readRecord_iff", "Spec.examples", "Spec.rejects" + ], + "permitted_axioms": ["propext", "Quot.sound", "Classical.choice"] +} diff --git a/lake-manifest.json b/lake-manifest.json new file mode 100644 index 0000000..abab5ae --- /dev/null +++ b/lake-manifest.json @@ -0,0 +1,6 @@ +{"version": "1.3.0", + "packagesDir": ".lake/packages", + "packages": [], + "name": "sqlite", + "lakeDir": ".lake", + "fixedToolchain": false} diff --git a/lakefile.toml b/lakefile.toml new file mode 100644 index 0000000..516271c --- /dev/null +++ b/lakefile.toml @@ -0,0 +1,16 @@ +name = "sqlite" +version = "0.1.0" +defaultTargets = ["Sqlite", "Spec", "Proofs", "sqlite"] + +[[lean_lib]] +name = "Sqlite" + +[[lean_lib]] +name = "Spec" + +[[lean_lib]] +name = "Proofs" + +[[lean_exe]] +name = "sqlite" +root = "Main" diff --git a/lean-toolchain b/lean-toolchain new file mode 100644 index 0000000..a6aff06 --- /dev/null +++ b/lean-toolchain @@ -0,0 +1 @@ +leanprover/lean4:v4.35.0 diff --git a/tests/PLANTED-BUGS.md b/tests/PLANTED-BUGS.md new file mode 100644 index 0000000..a358566 --- /dev/null +++ b/tests/PLANTED-BUGS.md @@ -0,0 +1,56 @@ +# Planted bugs + +Written by `tests/plant_bugs.py`. Each bug is a change of one line, to `Sqlite/Basic.lean` (40 bugs) or to `Main.lean` (7), tested with `tests/run_tests.py --count 3000 --no-oracles`, which builds only the executable and compares it with the model alone. Every bug must give failures, and the unchanged code none. A bug that does not build, or a run that gives up or dies, counts as missed. So does a run on other inputs than the unchanged code's: every run must give the same `- Inputs SHA-256:` line, which here is 51f296ed8bcf700cf86ce58a62ccd0eeb8291a87cd09c640cb5dd057fda50271. + +| Planted bug | Failures | +|---|---| +| none | 0 | +| the ninth byte of a varint gives only its low seven bits | 858 | +| a byte 0x80 ends a varint, as if its continuation bit were clear | 2187 | +| a varint byte that goes on gives its continuation bit too | 20785 | +| the page size 1 is not read as 65536 | 13370 | +| a read version of 0 is accepted | 13 | +| a usable size of 479 is accepted | 3 | +| only the maximum embedded payload fraction is checked | 5631 | +| the last of the 20 bytes reserved for expansion is not checked | 766 | +| schema format 5 is accepted | 15 | +| text encoding 0 is accepted, whatever the schema format | 48 | +| text encoding 0 is refused, even with schema format 0 | 12839 | +| the index leaf page type is 11, not 10 | 19520 | +| page type 2 is read as a table interior page | 240 | +| the right-most pointer is read as 2 bytes | 458 | +| on a leaf page, a content start of 0 is not read as 65536 | 3 | +| a cell may start one byte before the cell content area | 127 | +| a cell may end one byte past the usable size | 33 | +| the bound on the cell pointer array counts one byte a pointer | 73 | +| the cell content area may start in the reserved region | 46 | +| the page header of an interior page is taken as 8 bytes in the bound on the pointer array | 40 | +| the page may be one byte short of the page size | 193 | +| X is U-35 on index leaf pages, not on table leaf pages | 18358 | +| X on index pages is one more | 2052 | +| M divides by 256, not 255 | 16784 | +| K takes the remainder by U-5, not U-4 | 25782 | +| a K equal to X is not kept on the page | 2064 | +| the overflow page number is read as 3 bytes | 424 | +| a cell that spills is not counted with its overflow page number | 17 | +| rowids of 2^63 and more are not read as negative | 122 | +| the rowid of a table interior cell is not read as a 64-bit two's complement integer | 59 | +| serial type 5 takes 5 bytes, not 6 | 903 | +| serial type 7 is not 8 bytes | 1188 | +| serial type 11 is read | 9 | +| an integer whose top bit is set and the rest clear is read as positive | 170 | +| serial type 7 is read as an integer | 1180 | +| serial type 9 is read as 0 | 476 | +| odd serial types are blobs and even ones text | 2220 | +| the header size is counted after a size varint of one byte, whatever its length | 232 | +| bytes after the last column are allowed | 225 | +| a record with no column is read | 71 | +| Main.lean: the CR of a CR LF line ending is kept | 44306 | +| Main.lean: upper-case hex digits are read | 7 | +| Main.lean: a cell is printed with its rowid before its payload size | 965 | +| Main.lean: the content start 65536 is printed as 0 | 5 | +| Main.lean: `sqlite dump` exits with 0 when the header is refused | 3 | +| Main.lean: `sqlite dump` leaves out the last page | 30 | +| Main.lean: `sqlite dump` prints the record of a cell that spills, from the part on the page | 26 | + +Bugs caught: 47 of 47 diff --git a/tests/TEST-RESULTS.md b/tests/TEST-RESULTS.md new file mode 100644 index 0000000..a3eb469 --- /dev/null +++ b/tests/TEST-RESULTS.md @@ -0,0 +1,46 @@ +# Test results + +Written by `tests/run_tests.py`. Rerun it to check these numbers. + +- Lean: Lean (version 4.35.0, x86_64-unknown-linux-gnu, commit eb6cafb7290e8bd5f2937522d8d804f58f02bafc, Release) +- Executable: built by `lake build sqlite`; SHA-256 of the copy run 0775a7c64c7675eb78281336b584d0e55b5e94508d8e29acc77686a98a324b73 +- Oracle: the model of the file format in `tests/run_tests.py`; SQLite 3.45.1 +- SQLite: 3.45.1 (2024-01-30 16:01:20 e876e51a0ed5c5b3126f52e532044363a014bc594cfefa87ffb5b82257ccalt1), the library of Debian package libsqlite3-0 3.45.1-1ubuntu2.8, through the sqlite3 module of Python 3.13.17, with dbstat +- Oracles left out: none +- Seed: 20261028 +- Inputs: 699318 +- Inputs SHA-256: 431810804d8f98d0f9d4e4515af3fa0315801b62475cf294dde189e819070f79, of the files made and the requests built from them, in order; a rerun with the same seed and count must give the same +- By kind: Spec.lean's examples and rejects (header) 7, Spec.lean's examples and rejects (local) 1, Spec.lean's examples and rejects (page) 3, Spec.lean's examples and rejects (record) 6, Spec.lean's examples and rejects (varint) 5, a built page, a byte changed (page) 1198, a built page, a cell pointer before the content area (page) 910, a built page, a cell running past the usable size (page) 882, a built page, a payload size one more (page) 653, a built page, none (page) 8588, a built page, the cell count one less (page) 919, a built page, the cell count one more (page) 1186, a built page, the content start 0 (page) 1156, a built page, the content start before the pointer array ends (page) 1213, a built page, the content start past the first cell (page) 929, a built page, the content start past the usable size (page) 1230, a built page, the page type changed (page) 1136, a database made by SQLite (dump) 120, a database made by SQLite, bytes changed (dump) 54, a grid of usable and payload sizes (local) 63299, a header from random fields (header) 20000, a page of a database made by SQLite, a bit flipped (page) 3019, a page of a database made by SQLite, a byte of a cell changed (page) 1017, a page of a database made by SQLite, a cell pointer changed (page) 2504, a page of a database made by SQLite, a page header byte changed (page) 2977, a page of a database made by SQLite, a payload size changed (page) 1592, a page of a database made by SQLite, a rowid changed (page) 737, a page of a database made by SQLite, an overflow page number changed (page) 174, a page of a database made by SQLite, as page 2 (page) 2951, a page of a database made by SQLite, as page 2, the file one byte short (page) 762, a page of a database made by SQLite, the cell count changed (page) 3063, a page of a database made by SQLite, the content start changed (page) 2986, a page of a database made by SQLite, the file cut short (page) 3058, a page of a database made by SQLite, the freeblock and fragment fields changed (page) 2888, a page of a database made by SQLite, the page size field changed (page) 2980, a page of a database made by SQLite, the page type changed (page) 2954, a page of a database made by SQLite, the reserved byte of the header changed (page) 3035, a page of a database made by SQLite, the right-most pointer changed (page) 349, a payload that spills, joined from its overflow pages (record) 3964, a real header cut short (header) 100, an edge of a page, a cell ending at U-1, U or U+1 (page) 95, an edge of a page, a cell ending at U-1, U or U+1, the file one byte short (page) 95, an edge of a page, a cell starting one byte before the content area (page) 95, an edge of a page, a cell starting one byte before the content area, the file one byte short (page) 95, an edge of a page, a content start around the end of the pointer array (page) 234, an edge of a page, a content start around the end of the pointer array, the file one byte short (page) 234, an edge of a page, empty, content start U-1 to 65536 (page) 168, an edge of a page, empty, content start U-1 to 65536, the file one byte short (page) 168, each length, at its bounds (varint) 47, each length, at its bounds, cut short (varint) 47, each length, at its bounds, with bytes after (varint) 47, each schema format from 0 to 5 with each text encoding from 0 to 4, in a real header (header) 60, every payload size up to three pages (local) 11912, every two bytes (varint) 65536, every value of bytes 18 to 23, for each page size (header) 12288, every value of every byte of a real header (header) 51000, every value of every byte of the header of a database with no schema (header) 25500, every value of the page size field (header) 65536, every varint of one byte (varint) 256, nine bytes, every ninth byte (varint) 512, page 0 and each of the first 8 pages of a database made by SQLite with pages up to 2048 bytes, the file cut after it (page) 563, random (local) 50000, random bytes (record) 10000, random bytes (varint) 100000, random record (record) 100000, random record, a bit flipped (record) 5039, random record, a header size past the record (record) 4954, random record, a header size smaller than its varint (record) 4990, random record, a huge serial type (record) 4935, random record, a serial type running past the header (record) 4999, random record, no body (record) 5020, random record, no column (record) 4972, random record, serial type 10 or 11 (record) 5006, random record, the body one byte longer (record) 5086, random record, the body one byte shorter (record) 5137, random record, the header size one less (record) 4785, random record, the header size one more (record) 5070, the header of a database made by SQLite (header) 120, the last page of a database made by SQLite of up to 128 KB, and the page after it (page) 112 +- Requests: `dump` 174, `header` 174611, `local` 125212, `page` 58908, `record` 173963, `varint` 166450 +- Answers that differ from the model: 0 +- Spec.lean's `examples` and `rejects`, answered by the model as Spec.lean states them: 22 of 22 +- Varints: 166450, of which the model reads 100910; why it refuses the others: the varint does not end (1.6) 65540 +- Headers: 174611, of which the model reads 49828; why it refuses the others: a page size that is not a power of two from 512 to 65536 (1.3.2) 68811, the 20 bytes reserved for expansion not all zero (1.3.17) 16877, not the header string (1.3.1) 15266, payload fractions other than 64, 32 and 32 (1.3.5) 10264, a schema format above 4 (1.3.10) 4117, a text encoding other than 0, 1, 2 and 3 (1.3.13) 3881, a read version other than 1 and 2 (1.3.3) 3540, a text encoding of 0 with a schema format other than 0 (1.3.13; Spec/Defs.lean HeaderValid allows 0 only with a schema format of 0) 1510, a usable size below 480 (1.3.4) 417, fewer than 100 bytes (1.3) 100 +- Pages asked with `page`: 58908, of which the model reads 21005; why it refuses the others: a page type other than 2, 5, 10 and 13 (1.6) 15271, the page is not all in the file (1.2) 7150, a cell runs past the usable size (1.6) 4544, a cell starts before the cell content area (1.6) 3919, the cell content area starts past the usable size (1.6) 3886, the cell pointer array runs past the start of the cell content area (1.6) 2431, the file's header is refused 637, page 0: pages are numbered from 1 (1.2) 65 +- Records: 173963, of which the model reads 108155; why it refuses the others: the body ends before the last column (2.1) 17845, a header size past the end of the record (2.1) 13608, a serial type runs past the end of the header (2.1) 8474, bytes after the last column (Spec/Defs.lean RecordEncodes) 7959, no column (2.1: "one or more additional varints") 6048, a header size smaller than its own varint (2.1) 5070, serial type 10 or 11, "Reserved for internal use" (2.1) 5030, the header size varint does not end (2.1) 1774 +- Databases made by SQLite: 120 (20 fixed, the rest random), 22648 pages in all. Page sizes: 512 24, 2048 23, 8192 18, 1024 17, 4096 16, 65536 9, 32768 7, 16384 6; with reserved bytes: 21; text encodings: UTF-8 84, UTF-16le 20, UTF-16be 16; auto_vacuum: none 65, incremental 29, full 26; WAL: 24; legacy file format: 11; secure_delete off: 32. Each made twice from the same seed, the same bytes both times: 120 of 120 +- Dumps: 174, of which 54 are of copies with one to three pages changed, each different from its database; 87916 lines; as the model prints them: 174 of 174. Pages in them, by type: not read (FAIL) 19432, page type 13 3335, page type 10 2344, page type 2 338, page type 5 209, records not read (FAIL) 10. Databases made by SQLite with no schema (schema format 0, text encoding 0): 1, of which the tool's dump reads the header as the model does: 1 +- `header` requests with a text encoding of 0, by schema format, as the model answers them (an answer of the tool's that differs is a failure above): schema format 0: read 13715, refused for another rule 11627; schema format 1: refused for the text encoding 378, refused for another rule 193; schema format 2: refused for the text encoding 358, refused for another rule 197; schema format 3: refused for the text encoding 343, refused for another rule 180; schema format 4: refused for the text encoding 431, refused for another rule 199; schema format 5 or more: refused for another rule 1252 +- Agreement with SQLite (the rest are the named differences below, or failures): dbstat rows 17182 of 17191, header fields 1294 of 1440, index entries 31782 of 34910, pages accounted for 120 of 120, pointer-map entries 9232 of 9232, table interior keys 2067 of 2067, table rows 9712 of 18314 +- SQLite's answers that the model's dump, read with SQLite's rules, gives, whether or not the tool agrees (any other is a failure): dbstat rows 17191 of 17191, header fields 1440 of 1440, index entries 34910 of 34910, pointer-map entries 9232 of 9232, table rows 18314 of 18314 +- Payloads that spill, joined from the part the tool keeps on the page and the overflow pages, and read whole by `record`: 3964 +- Pages SQLite uses as overflow, freelist or pointer-map pages: 18075 (overflow pages 12618, freelist trunk pages 53, freelist leaf pages 5298, pointer-map pages 106), of which the tool reads 616 as b-tree pages from their bytes (freelist leaf pages 609, pointer-map pages 7). A count, not a check: SQLite gives no b-tree reading of these pages, and their lines in the dumps are compared with the model's like every other +- Malformed requests answered `error`: 64 of 64; odd but well-formed requests answered as the model says: 14 of 14 +- Requests ending in CR LF: every seventh, 99311 +- Last lines sent alone, with no LF, with CR LF, or with a lone CR, answered right: 4 of 4 +- Command line (`sqlite dump`, `sqlite serve` and wrong arguments), printing and exiting as the model says: 18 of 18 +- Time: 53 s +- Failures: 0 + +## Where SQLite and the tool differ + +Each disagreement is counted under the reason the model confirms: the model's dump, read with SQLite's own rules, gives SQLite's answer, and needs the rules named to do so. The first case that showed it is given. + +- a whole-number float in a column of REAL affinity is kept as an integer; SQLite gives it back as a float (2.3): 5669; for example database 0 (4096-byte pages, as SQLite makes them by default), table t1, rowid 5, column c4: SQLite real:13830554455654793216, the tool int:-1 +- an INTEGER PRIMARY KEY column is NULL in the record; SQLite gives the rowid (2.3): 5568; for example database 6 (65536-byte pages, 200 bytes reserved, long overflow chains), table t0, rowid -6821244590509130105, column c0: SQLite int:-6821244590509130105, the tool null +- a whole-number float in a column of REAL affinity is kept as an integer; SQLite gives it back as a float (2.3); an INTEGER PRIMARY KEY column is NULL in the record; SQLite gives the rowid (2.3): 493; for example database 6 (65536-byte pages, 200 bytes reserved, long overflow chains), table t0, rowid -9223372036854775808, column c3: SQLite real:13974669643728551936, the tool int:-4294967295; database 6 (65536-byte pages, 200 bytes reserved, long overflow chains), table t0, rowid -9223372036854775808, column ... (350 characters) +- PRAGMA default_cache_size gives SQLite's built-in default, -2000, for a field of 0: 88; for example database 0 (4096-byte pages, as SQLite makes them by default): SQLite -2000, the tool 0 +- PRAGMA application_id reads the 4-byte field as a signed integer: 29; for example database 12 (no table, only user_version, application_id and default_cache_size): SQLite -123456789, the tool 4171510507 +- PRAGMA user_version reads the 4-byte field as a signed integer: 28; for example database 22 (random settings, 2048-byte pages): SQLite -1027188348, the tool 3267778948 +- dbstat counts unused bytes from the content start as stored, so 0 counts as 0, not 65536 (1.6): 9; for example database 1 (65536-byte pages: an empty table, an empty index and an empty WITHOUT ROWID table, whose content start 0 means 65536; long overflow chains), page 2: dbstat ('t0', '/', 'leaf', 0, 0, -8, 0), the tool's reading ('t0', '/', 'leaf', 0, 0, 65528, 0) +- PRAGMA encoding gives UTF-8, the default for a database not yet created, for a text encoding of 0, which names none: SQLite writes 0, with a schema format of 0, while a database has no schema: 1; for example database 12 (no table, only user_version, application_id and default_cache_size): SQLite UTF-8, the tool none (text encoding 0) diff --git a/tests/plant_bugs.py b/tests/plant_bugs.py new file mode 100644 index 0000000..7a209c9 --- /dev/null +++ b/tests/plant_bugs.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +"""Check that the differential tests catch planted bugs. + +Each bug is a one-line change to Sqlite/Basic.lean, or to Main.lean +where its description says so. The script copies the project to a +temporary folder, without .lake/build, plants one bug at a time, runs +tests/run_tests.py there with a small count and without the oracle, and +records how many failures the tests report. So every failure counted is +the tool disagreeing with the model of the file format; SQLite only +makes the databases that are read. Every planted bug must give at least +one failure. The unchanged copy must give none. A bug that stops the +executable from building is not a catch: the tests then give up, and +the bug counts as missed. So is any run that gives up (its record shows +`- Failures: 1 (why)`), a run that dies without writing its record, and +one whose exit status does not match the failures it records. Every run +must also test the same inputs as the unchanged copy: its record must +give the same `- Inputs SHA-256:` line, or the bug counts as missed, +since its failures might come from inputs the unchanged copy was not +run on. The tests build only the executable (`lake build sqlite`), so a +bug that breaks a proof still runs. + + python3 tests/plant_bugs.py [--count N] + +The script writes tests/PLANTED-BUGS.md and exits non-zero if any bug +goes uncaught. +""" + +import argparse +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +RECORD = ROOT / "tests" / "PLANTED-BUGS.md" +CODE = "Sqlite/Basic.lean" + +# (what the bug does, the text as written, the text with the bug[, file]) +BUGS = [ + ("the ninth byte of a varint gives only its low seven bits", + " if i = 8 then some (acc * 256 + x.toNat, 9, rest)", + " if i = 8 then some (acc * 128 + x.toNat % 128, 9, rest)"), + ("a byte 0x80 ends a varint, as if its continuation bit were clear", + " else if x.toNat < 128 then some (acc * 128 + x.toNat, i + 1, rest)", + " else if x.toNat ≤ 128 then some (acc * 128 + x.toNat, i + 1, rest)"), + ("a varint byte that goes on gives its continuation bit too", + " else go (acc * 128 + x.toNat % 128) (i + 1) rest", + " else go (acc * 128 + x.toNat) (i + 1) rest"), + ("the page size 1 is not read as 65536", + " let pageSize := if ps = 1 then 65536 else ps", + " let pageSize := ps"), + ("a read version of 0 is accepted", + " guard (pageSizeOk pageSize && (rv == 1 || rv == 2) && 480 ≤ pageSize - res &&", + " guard (pageSizeOk pageSize && rv ≤ 2 && 480 ≤ pageSize - res &&"), + ("a usable size of 479 is accepted", + " guard (pageSizeOk pageSize && (rv == 1 || rv == 2) && 480 ≤ pageSize - res &&", + " guard (pageSizeOk pageSize && (rv == 1 || rv == 2) && 479 ≤ pageSize - res &&"), + ("only the maximum embedded payload fraction is checked", + " guard (fr = [64, 32, 32])", + " guard (fr.head? = some 64)"), + ("the last of the 20 bytes reserved for expansion is not checked", + " guard (z = List.replicate 20 0)", + " guard (z.take 19 = List.replicate 19 0)"), + ("schema format 5 is accepted", + " schemaFormat ≤ 4 && (textEncoding == 1 || textEncoding == 2 || textEncoding == 3 ||", + " schemaFormat ≤ 5 && (textEncoding == 1 || textEncoding == 2 || textEncoding == 3 ||"), + ("text encoding 0 is accepted, whatever the schema format", + " textEncoding == 0 && schemaFormat == 0))", + " textEncoding == 0))"), + ("text encoding 0 is refused, even with schema format 0", + " textEncoding == 0 && schemaFormat == 0))", + " false))"), + ("the index leaf page type is 11, not 10", + " | 10 => some .indexLeaf", + " | 11 => some .indexLeaf"), + ("page type 2 is read as a table interior page", + " | 2 => some .indexInterior", + " | 2 => some .tableInterior"), + ("the right-most pointer is read as 2 bytes", + " let (p, r) ← readBE 4 r", + " let (p, r) ← readBE 2 r"), + ("on a leaf page, a content start of 0 is not read as 65536", + " else\n pure ({ type, firstFreeblock, cellCount, contentStart := if cs = 0 then 65536 else cs,", + " else\n pure ({ type, firstFreeblock, cellCount, contentStart := cs,"), + ("a cell may start one byte before the cell content area", + " guard (start ≤ p)", + " guard (start ≤ p + 1)"), + ("a cell may end one byte past the usable size", + " guard (p + size ≤ U)", + " guard (p + size ≤ U + 1)"), + ("the bound on the cell pointer array counts one byte a pointer", + " guard (off + hs + 2 * ph.cellCount ≤ ph.contentStart && ph.contentStart ≤ U)", + " guard (off + hs + ph.cellCount ≤ ph.contentStart && ph.contentStart ≤ U)"), + ("the cell content area may start in the reserved region", + " guard (off + hs + 2 * ph.cellCount ≤ ph.contentStart && ph.contentStart ≤ U)", + " guard (off + hs + 2 * ph.cellCount ≤ ph.contentStart && ph.contentStart ≤ h.pageSize)"), + ("the page header of an interior page is taken as 8 bytes in the bound on the pointer array", + " let hs := if ph.type.interior then 12 else 8", + " let hs := 8"), + ("the page may be one byte short of the page size", + " guard (page.length = h.pageSize)", + " guard (page.length + 1 ≥ h.pageSize)"), + ("X is U-35 on index leaf pages, not on table leaf pages", + " | .tableLeaf => U - 35", + " | .indexLeaf => U - 35"), + ("X on index pages is one more", + " | _ => (U - 12) * 64 / 255 - 23", + " | _ => (U - 12) * 64 / 255 - 22"), + ("M divides by 256, not 255", + " let m := (U - 12) * 32 / 255 - 23", + " let m := (U - 12) * 32 / 256 - 23"), + ("K takes the remainder by U-5, not U-4", + " let k := m + (P - m) % (U - 4)", + " let k := m + (P - m) % (U - 5)"), + ("a K equal to X is not kept on the page", + " if k ≤ x then k else m", + " if k < x then k else m"), + ("the overflow page number is read as 3 bytes", + " let (o, _) ← readBE 4 r", + " let (o, _) ← readBE 3 r"), + ("a cell that spills is not counted with its overflow page number", + " pure (loc, some o, L + 4)", + " pure (loc, some o, L)"), + ("rowids of 2^63 and more are not read as negative", + " if n < 9223372036854775808 then n else (n : Int) - 18446744073709551616", + " if n < 18446744073709551616 then n else (n : Int) - 18446744073709551616"), + ("the rowid of a table interior cell is not read as a 64-bit two's complement integer", + " pure ({ leftChild := some k, payloadSize := none, rowid := some (int64 rowid),", + " pure ({ leftChild := some k, payloadSize := none, rowid := some (rowid : Int),"), + ("serial type 5 takes 5 bytes, not 6", + " else if t = 5 then some 6", + " else if t = 5 then some 5"), + ("serial type 7 is not 8 bytes", + " else if t = 6 ∨ t = 7 then some 8", + " else if t = 6 then some 8"), + ("serial type 11 is read", + " else if t = 10 ∨ t = 11 then none", + " else if t = 10 then none"), + ("an integer whose top bit is set and the rest clear is read as positive", + " if 2 * n < 256 ^ c.length then n else (n : Int) - 256 ^ c.length", + " if 2 * n ≤ 256 ^ c.length then n else (n : Int) - 256 ^ c.length"), + ("serial type 7 is read as an integer", + " else if t = 7 then .real (bigEndian c)", + " else if t = 7 then .int (signed c)"), + ("serial type 9 is read as 0", + " else if t = 9 then .int 1", + " else if t = 9 then .int 0"), + ("odd serial types are blobs and even ones text", + " else if t % 2 = 0 then .blob c", + " else if t % 2 = 1 then .blob c"), + ("the header size is counted after a size varint of one byte, whatever its length", + " let (hdr, body) ← take? (hsize - n) r", + " let (hdr, body) ← take? (hsize - 1) r"), + ("bytes after the last column are allowed", + " | [], _ :: _ => none", + " | [], _ :: _ => some []"), + ("a record with no column is read", + " guard (ts ≠ [])", + " guard (ts.length ≥ 0)"), + ("Main.lean: the CR of a CR LF line ending is kept", + " String.ofList (if cs.getLast? = some '\\r' then cs.dropLast else cs)", + " String.ofList cs", + "Main.lean"), + ("Main.lean: upper-case hex digits are read", + " else if 'a' ≤ c ∧ c ≤ 'f' then some (c.toNat - 'a'.toNat + 10)", + " else if 'a' ≤ c.toLower ∧ c.toLower ≤ 'f' then some (c.toLower.toNat - 'a'.toNat + 10)", + "Main.lean"), + ("Main.lean: a cell is printed with its rowid before its payload size", + ' s!"{p}/{opt c.leftChild}/{opt c.payloadSize}/{opt c.rowid}/{toHex c.payload}/{opt c.overflow}"', + ' s!"{p}/{opt c.leftChild}/{opt c.rowid}/{opt c.payloadSize}/{toHex c.payload}/{opt c.overflow}"', + "Main.lean"), + ("Main.lean: the content start 65536 is printed as 0", + ' s!"{ph.type.code} {ph.firstFreeblock} {ph.cellCount} {ph.contentStart} {ph.fragmented} " ++', + ' s!"{ph.type.code} {ph.firstFreeblock} {ph.cellCount} {ph.contentStart % 65536} {ph.fragmented} " ++', + "Main.lean"), + ("Main.lean: `sqlite dump` exits with 0 when the header is refused", + ' | none => IO.eprintln "not a database header this reader accepts"; return 1', + ' | none => IO.eprintln "not a database header this reader accepts"; return 0', + "Main.lean"), + ("Main.lean: `sqlite dump` leaves out the last page", + " for i in [0:file.length / h.pageSize] do", + " for i in [0:file.length / h.pageSize - 1] do", + "Main.lean"), + ("Main.lean: `sqlite dump` prints the record of a cell that spills, from the part on the page", + " if c.overflow.isNone && !c.payload.isEmpty then", + " if !c.payload.isEmpty then", + "Main.lean"), +] + + +def failures(copy, count): + """The failures a run of the tests reports, or None, with the run's + output, when it reports none: the code did not build, or the run did + not finish; and the SHA-256 of the inputs the run made. A run that + dies writes no record, so the old one is removed first rather than + read again.""" + record = copy / "tests" / "TEST-RESULTS.md" + record.unlink(missing_ok=True) + run = subprocess.run([sys.executable, "tests/run_tests.py", "--count", str(count), + "--no-oracles"], + cwd=copy, capture_output=True, text=True) + out = run.stdout + run.stderr + if not record.exists(): + return None, f"no record written, exit status {run.returncode}\n{out}", None + text = record.read_text(encoding="utf-8") + m = re.search(r"^- Failures: (\d+)(.*)$", text, re.M) + if m is None or m.group(2).strip(): + return None, out, None # the run gave up, for example the bug did not build: not a catch + d = re.search(r"^- Inputs SHA-256: ([0-9a-f]{64})", text, re.M) + n = int(m.group(1)) + if run.returncode != (1 if n else 0): + return None, f"{n} failures but exit status {run.returncode}\n{out}", None + return n, None, d and d.group(1) + + +def ignore(folder, names): + """Leave out the build, the git folder and old records: the copy + builds its own executable.""" + leave = {"PLANTED-BUGS.md", "__pycache__", ".git"} + if Path(folder).resolve() == (ROOT / ".lake").resolve(): + leave.add("build") + return [n for n in names if n in leave] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--count", type=int, default=3000) + args = ap.parse_args() + rows, missed = [], 0 + with tempfile.TemporaryDirectory() as tmp: + copy = Path(tmp) / "sqlite" + shutil.copytree(ROOT, copy, symlinks=True, ignore=ignore) + n, err, inputs = failures(copy, args.count) + rows.append(("none", n, "")) + if n != 0 or inputs is None: + missed += 1 + print(f"none: {n} failures, inputs {inputs}" + (" (did not build or did not finish)" if err else ""), + flush=True) + if err: + print(err[-2000:], flush=True) + for what, line, bug, *where in BUGS: + name = where[0] if where else CODE + code = copy / name + original = code.read_text(encoding="utf-8") + if original.count(line) != 1: + sys.exit(f"{name} no longer holds, once, the line for {what!r}: {line}") + code.write_text(original.replace(line, bug), encoding="utf-8") + n, err, got = failures(copy, args.count) + code.write_text(original, encoding="utf-8") + note = "" if err or got == inputs else " (other inputs than the unchanged copy's)" + rows.append((what, n, note)) + if not n or note: + missed += 1 + print(f"{what}: {n} failures{note}" + (" (did not build or did not finish)" if err else ""), + flush=True) + if err: + print(err[-2000:], flush=True) + n_main = sum(1 for b in BUGS if b[3:] == ("Main.lean",)) + lines = ["# Planted bugs", "", + f"Written by `tests/plant_bugs.py`. Each bug is a change of one line, to `{CODE}` " + f"({len(BUGS) - n_main} bugs) or to `Main.lean` ({n_main}), tested with " + f"`tests/run_tests.py --count {args.count} --no-oracles`, which builds only the " + "executable and compares it with the model alone. Every bug must give failures, and " + "the unchanged code none. A bug that does not build, or a run that gives up or " + "dies, counts as missed. So does a run on other inputs than the unchanged code's: " + "every run must give the same `- Inputs SHA-256:` line, which here is " + f"{inputs or 'missing'}.", "", + "| Planted bug | Failures |", "|---|---|"] + lines += [f"| {what} | {'did not build or did not finish' if n is None else n}{note} |" + for what, n, note in rows] + lines += ["", f"Bugs caught: {sum(1 for what, n, note in rows[1:] if n and not note)} of {len(BUGS)}", ""] + RECORD.write_text("\n".join(lines), encoding="utf-8") + return 1 if missed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/run_tests.py b/tests/run_tests.py new file mode 100644 index 0000000..134c59a --- /dev/null +++ b/tests/run_tests.py @@ -0,0 +1,3221 @@ +#!/usr/bin/env python3 +"""Differential tests: the Lean reader of the SQLite file format against +a model of the format and against SQLite itself. + +The tool answers `varint`, `header`, `page`, `record` and `local` +requests, and prints a whole file with `sqlite dump FILE` (see the +header of Main.lean). Its answers are checked two ways. + +1. Against the model, a Python reader written from "Database File + Format" (https://www.sqlite.org/fileformat2.html), sections 1.3, 1.6 + and 2.1, that shares nothing with the Lean code: + - varints as 1.6 gives them: one to nine bytes, the first eight + giving their low seven bits and ending the varint when their high + bit is clear, a ninth giving all eight; + - the 100-byte header as the table of 1.3 lays it out, big-endian, + with the page size 1 read as 65536 (1.3.2); + - a b-tree page as 1.6 lays it out: the page header after the + database header on page 1, at the start of every other page; the + right-most pointer on interior pages only; a content start of 0 + read as 65536; the cell pointer array; and each cell as the cell + table of 1.6 gives it, with the part of its payload kept on the + page worked out from X, M and K, and the first overflow page when + the payload spills; + - records as 2.1 gives them: the header size, one serial type per + column, and the body. + Where 1.3 says a field "must" hold a value, or a page or record could + be read more loosely than 1.6 and 2.1 say, the model follows the + choices Spec/Defs.lean writes down for this reader, which are + stricter than SQLite in places: + - HeaderValid: the header string; a page size that is a power of two + from 512 to 65536; a read version of 1 or 2 (any write version); a + usable size of at least 480; payload fractions of exactly 64, 32 + and 32; the 20 bytes reserved for expansion all zero; a schema + format from 0 to 4; a text encoding of 1, 2 or 3, or of 0 when the + schema format is 0 (SQLite writes both as 0 while a database has + no schema); + - PageEncodes: a page type of 2, 5, 10 or 13; the cell pointer array + ending no later than the cell content area starts, and that area + starting no later than the usable size; every cell starting no + earlier than the content area and ending within the usable size; + - RecordEncodes: serial types 10 and 11 refused; a body of exactly + the columns, no byte more; at least one column; every serial type + ending within the header. + The `local` request is `localSize`, in natural numbers as + Spec/Defs.lean computes it, so a difference below 0 is 0 and a + remainder by 0 is the number itself. The model prints what `sqlite + dump` prints by the header of Main.lean, which gives the format, and + the model's own answers to the requests, with no other rule: + - `header` and the 18 fields, as `header` gives them; + - for each whole page of the file in turn, N from 1 to the file's + length divided by the page size: `page N` and the page as `page` + gives it, or `page N FAIL` when it is not a b-tree page the + model accepts; + - after a page, for each cell whose payload is all on its page and + not empty, in the order of the cell pointers: `record N P ROWID` + and the values as `record` gives them, or FAIL when the payload is + not a record. P is the cell's pointer, ROWID its rowid (`-` on an + index page). An empty payload holds no record and gets no line. + It exits with 0. When the file does not start with a header the + model accepts, it prints nothing on standard output and exits with + 1; on wrong arguments or a file it cannot read, it prints nothing on + standard output and exits with 2. The model must give the answers + Spec.lean states in `examples` and `rejects`. Every answer of the + tool must equal the model's. The record counts, for each kind of + request, why the model refuses what it refuses. + +2. Against SQLite: Python's sqlite3 module, linked to the SQLite library + SQLITE_PINNED, makes databases of many kinds (see below) and then + answers questions about them in SQL. The tool reads each file with + `sqlite dump`, and the harness checks: + - the header fields against the PRAGMAs page_size, page_count, + freelist_count, schema_version, user_version, application_id, + encoding, default_cache_size, journal_mode (versions 1, or 2 for + WAL), auto_vacuum (the largest root page and the incremental vacuum + flag), and sqlite_version(); + - every row of the dbstat virtual table: the harness walks each + b-tree from the root page sqlite_schema gives, by the child + pointers the tool reads, and follows each overflow chain from the + first overflow page the tool reads (the next page number is the + first 4 bytes of each overflow page, 1.7). From that it rebuilds + what dbstat gives for every page: the b-tree it belongs to, its + path, whether it is an interior, leaf or overflow page, its cells, + the payload bytes on it (the sum of the parts the tool keeps on the + page), its unused bytes, and its largest payload. The types the + tool reads must be table types in a table b-tree and index types in + an index or a WITHOUT ROWID table, and each overflow chain must end + where the payload size says. The key of each table interior cell + must be no less than the rowids to its left and less than those to + its right (1.6); + - that every page is accounted for, once: a page dbstat lists, a + freelist page reached from the first trunk page the tool reads + (1.5), or a pointer-map page (1.8); and that each pointer-map entry + gives the parent the tool's reading gives. The record counts the + overflow, freelist and pointer-map pages whose bytes the tool reads + as a b-tree page. That count is not a check: SQLite gives no b-tree + reading of those pages to compare with, and their dump lines are + compared with the model's like every other; + - every row of every rowid table, sqlite_schema among them, against + SELECT rowid, * in rowid order: the rowids the tool reads, and the + values of the records. A cell whose payload is all on its page is + read from the dump; for one that spills, the harness joins the part + on the page and the overflow pages, as many as the payload size + needs, and asks the tool's `record` request to read the whole + payload. So the tool's split of the payload must be where SQLite + put the bytes, or the record does not read back; + - every entry of every index and WITHOUT ROWID table, against SELECT + of the columns PRAGMA index_xinfo lists, as multisets. + Text is compared as bytes in the database's encoding, a float as its + 64 bits, a blob as its bytes. + + Each of these answers of SQLite is also read by the model: the + model's dump of the file, walked the same way, read with SQLite's own + rules (below). The model, so read, must give SQLite's answer, whether + or not the tool agrees; any answer it does not give is a failure. A + disagreement between the tool and SQLite is allowed only for a named + reason that the model confirms: the model, with SQLite's rules, gives + SQLite's answer, and needs at least one of those rules to do so. The + disagreement is counted under the rules it needs. The rules: + - an INTEGER PRIMARY KEY column is NULL in the record, and SQLite + gives the rowid for it (2.3); + - in a column of REAL affinity, a float with no fractional part is + kept as an integer, and SQLite gives it back as a float (2.3); + - user_version and application_id read their 4 bytes as a signed + integer; + - default_cache_size reads its 4 bytes as a signed integer and gives + its absolute value, or SQLite's built-in default, -2000, for 0; + - dbstat counts the unused bytes of a page from the content start as + stored, so a 0 counts as 0 and not as 65536 (1.6); + - PRAGMA encoding gives UTF-8 for a text encoding of 0. SQLite + writes 0 there, with a schema format of 0, while a database has no + schema, and the field then names no encoding; for a database not + yet created, PRAGMA encoding gives "the text encoding that will be + used to create the main database" (pragma.html), UTF-8 by default + for a connection opened with sqlite3_open_v2 (c3ref/open.html), as + the sqlite3 module opens it. The tool and the model read such a + header, as Spec/Defs.lean allows; this rule is about PRAGMA + encoding alone. + Any other disagreement is a failure. The record gives each named + difference with its count and the first case. If the installed + SQLite is not SQLITE_PINNED or has no dbstat, SQLite is left out as + an oracle, and the record says so on its `- Oracles left out:` line. + +Inputs come from a recorded seed: + +- databases made by SQLite: a fixed set (FIXED_DATABASES) and random + ones, with page sizes from 512 to 65536, reserved bytes at the end of + each page (set with SQLITE_FILECONTROL_RESERVE_BYTES through ctypes, + on the library the sqlite3 module uses), UTF-8 and UTF-16 text, + auto_vacuum and incremental vacuum (pointer-map pages), WAL, the + legacy file format (schema format 1, set with + SQLITE_DBCONFIG_LEGACY_FILE_FORMAT), secure_delete off, + user_version, application_id and default_cache_size; a database with + no schema, whose schema format and text encoding are 0; tables with and + without rowid, STRICT tables, INTEGER PRIMARY KEY, AUTOINCREMENT, a + table of 70 columns, indexes (UNIQUE, DESC, COLLATE, partial); + integers of every size and sign around each boundary, 0 and 1, + floats, NULL, text and blobs from empty to big enough for long + overflow chains; payload sizes on both sides of each place where the + part kept on the page changes; rows enough for interior pages; + explicit negative and huge rowids; deletes and updates that leave + freeblocks, fragments and free pages; dropped tables, VACUUM, + ANALYZE, views and triggers. SQLite is never left to choose at + random, so the same seed makes the same files: rows to delete from a + WITHOUT ROWID table are picked here, not with random(), and once a + table may hold the largest rowid, 2^63 - 1, every later row in it is + given its rowid here (without one, SQLite would pick it at random, + autoinc.html). Each database is made twice, and the two must be the + same bytes. Each is dumped, and so are copies of small ones with one + to three pages changed, in the ways listed below that keep the file's + length and its 100-byte header; each copy differs from its database; +- pages of those databases asked with `page N`: for databases of pages + up to 2048 bytes, page 0 and each of the first 8 pages, with the file + cut after the page; for files up to 128 KB, the last page and the one + after it; pages moved to page 2 of a two-page file, whole and one + byte short; and pages changed in one way, so that the file's bytes + differ: a bit flipped, a page header byte, the cell count, the + content start, a cell pointer, a payload size, a rowid, an overflow + page number, the right-most pointer, the page type, a byte of a cell, + the freeblock and fragment fields, the reserved byte or the page size + field of the header, or the file cut short; +- b-tree pages built from their parts, of every type and page size, + with payload sizes around X, M and K, and with defects: the content + start before the pointer array ends, past the usable size, 0, or past + the first cell; a cell pointer before the content area; a cell running + past the usable size; the cell count one more or one less; a payload + size one more; a byte changed; the page type changed. A defect that + cannot be made on a page (no cell to change, say) is not made, and the + page is counted as one with none. And pages at the edges of + PageEncodes, empty or with one cell, for three page sizes with and + without reserved bytes, whole and one byte short; +- every value of every byte of two real headers and of the header of + the database with no schema (so a text encoding of 0 with each schema + format, and each text encoding with a schema format of 0), each + schema format from 0 to 5 with each text encoding from 0 to 4 in a + real header and in that one, every value of the page size field, + every value of the versions, the reserved byte and the payload + fractions for each page size, headers from random fields (among them + a text encoding of 0 with each schema format from 0 to 4), a real + header cut short; +- every varint of one and two bytes, each length at its bounds, nine + bytes with every ninth byte, and random bytes; +- random records, with integers stored in their fewest bytes and in + more, floats, text, blobs, 0 and 1, header varints in more bytes than + needed; and changed: serial types 10, 11 and huge ones, the body one + byte longer or shorter, no body, the header size one off, smaller + than its varint or past the record, a serial type running past the + header, no column, a bit flipped; and random bytes; +- `local` over a grid of page types, usable sizes (0 to 40, around 480 + and 512, each page size less some reserved bytes, and huge ones) and + payload sizes around X, M and K, every payload size up to three pages + for two usable sizes, and random ones; +- the inputs of Spec.lean's `examples` and `rejects`; +- malformed requests, which the tool must answer with `error`, and odd + but well-formed ones; +- the command line: `sqlite dump` on a database, on a file with no + header, an empty file, a header alone, a file cut inside its header, + a file with part of a page at the end, a file that does not exist and + a folder; `sqlite serve` with nothing to read; and wrong arguments: + what each run prints on standard output and its exit status. + +Every seventh request ends in CR LF. The record gives a SHA-256 of the +files made and of the requests built from them, so that a rerun with +the same seed and count can be checked to have the same inputs. It +leaves out the payloads that spill, which are joined by the tool's +reading. + + python3 tests/run_tests.py [--seed N] [--count N] [--no-oracles] [--no-build] + +`--count` is the number of random varints and records; a fifth as many +random headers and built pages, a third as many changed pages, half as +many random `local` requests, and `--count` / 1000 random databases +after the fixed ones are made. `--no-oracles` leaves SQLite out as an +oracle and compares the tool with the model alone, as +tests/plant_bugs.py does; the databases are still made, as inputs. The +script builds the executable with `lake build sqlite` first, or, with +`--no-build`, takes the one in .lake/build/bin as it is, for when +another process owns the build; either way it runs a copy, so a build +during the run cannot change it. It writes tests/TEST-RESULTS.md and +exits non-zero if any check fails. +""" + +import argparse +import collections +import concurrent.futures +import ctypes +import hashlib +import os +import random +import shutil +import sqlite3 +import struct +import subprocess +import sys +import sysconfig +import tempfile +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BUILT_EXE = ROOT / ".lake" / "build" / "bin" / "sqlite" +TESTS = ROOT / "tests" +RESULTS = TESTS / "TEST-RESULTS.md" +TIMEOUT = 3600 +BUILD_TIMEOUT = 3600 +SQLITE_PINNED = "3.45.1" +# The requests are split among this many `sqlite serve` processes, and +# this many dumps run at once. +JOBS = max(1, min(4, os.cpu_count() or 1)) +# The copy of the executable the run uses (set in main). +EXE = None + +# --------------------------------------------------------------------- +# The model: "Database File Format", sections 1.3, 1.6 and 2.1, with the +# choices of Spec/Defs.lean. + +MAGIC = b"SQLite format 3\x00" +PAGE_SIZES = (512, 1024, 2048, 4096, 8192, 16384, 32768, 65536) +INDEX_INTERIOR, TABLE_INTERIOR, INDEX_LEAF, TABLE_LEAF = 2, 5, 10, 13 +PAGE_TYPES = (INDEX_INTERIOR, TABLE_INTERIOR, INDEX_LEAF, TABLE_LEAF) +INTERIOR = (INDEX_INTERIOR, TABLE_INTERIOR) + +# The fields of a header, in the order of the table in 1.3. +H_PAGE_SIZE, H_WRITE, H_READ, H_RESERVED, H_CHANGE, H_SIZE, H_TRUNK, H_FREE, H_COOKIE, \ + H_SCHEMA, H_CACHE, H_LARGEST, H_ENCODING, H_USER, H_INCREMENTAL, H_APPID, H_VALID, \ + H_VERSION = range(18) + + +def nsub(a, b): + """a - b in natural numbers, as Spec/Defs.lean computes it.""" + return a - b if a > b else 0 + + +def ndiv(a, b): + return a // b if b else 0 + + +def nmod(a, b): + return a % b if b else a + + +def be(bs): + """Bytes as a big-endian unsigned integer.""" + return int.from_bytes(bs, "big") + + +def read_varint(b, pos=0, end=None): + """The varint at `pos`, reading no byte at or past `end`: its value + and length, or None when the bytes end first (1.6). "A varint is + between 1 and 9 bytes in length. The varint consists of either zero + or more bytes which have the high-order bit set followed by a single + byte with the high-order bit clear, or nine bytes, whichever is + shorter. The lower seven bits of each of the first eight bytes and + all 8 bits of the ninth byte are used to reconstruct the 64-bit + twos-complement integer. Varints are big-endian." """ + end = len(b) if end is None else end + v = 0 + for i in range(9): + if pos + i >= end: + return None + x = b[pos + i] + if i == 8: + return (v << 8) | x, 9 + v = (v << 7) | (x & 0x7F) + if x < 0x80: + return v, i + 1 + raise AssertionError("unreachable") + + +WHY_H_SHORT = "fewer than 100 bytes (1.3)" +WHY_H_MAGIC = "not the header string (1.3.1)" +WHY_H_PAGE_SIZE = "a page size that is not a power of two from 512 to 65536 (1.3.2)" +WHY_H_READ = "a read version other than 1 and 2 (1.3.3)" +WHY_H_FRACTIONS = "payload fractions other than 64, 32 and 32 (1.3.5)" +WHY_H_USABLE = "a usable size below 480 (1.3.4)" +WHY_H_SCHEMA = "a schema format above 4 (1.3.10)" +WHY_H_ENCODING = "a text encoding other than 0, 1, 2 and 3 (1.3.13)" +WHY_H_ENCODING_0 = ("a text encoding of 0 with a schema format other than 0 (1.3.13; Spec/Defs.lean " + "HeaderValid allows 0 only with a schema format of 0)") +WHY_H_EXPANSION = "the 20 bytes reserved for expansion not all zero (1.3.17)" + + +def read_header(f): + """The 18 fields of the database header at the start of `f`, in the + order of the table in 1.3, and None; or None and why it is refused.""" + if len(f) < 100: + return None, WHY_H_SHORT + if f[:16] != MAGIC: + return None, WHY_H_MAGIC + # "Must be a power of two between 512 and 32768 inclusive, or the + # value 1 representing a page size of 65536." + stored = be(f[16:18]) + page_size = 65536 if stored == 1 else stored + if page_size not in PAGE_SIZES: + return None, WHY_H_PAGE_SIZE + write_version, read_version, reserved = f[18], f[19], f[20] + # "If a database file with a read version greater than 2 is + # encountered, then that database cannot be read or written." + if read_version not in (1, 2): + return None, WHY_H_READ + # "The maximum and minimum embedded payload fractions and the leaf + # payload fraction values must be 64, 32, and 32." + if f[21:24] != bytes([64, 32, 32]): + return None, WHY_H_FRACTIONS + # "the usable size is not allowed to be less than 480" + if page_size - reserved < 480: + return None, WHY_H_USABLE + words = [be(f[i:i + 4]) for i in range(24, 72, 4)] + schema_format, encoding = words[H_SCHEMA - 4], words[H_ENCODING - 4] + if schema_format > 4: + return None, WHY_H_SCHEMA + # "A value of 1 means UTF-8. A value of 2 means UTF-16le. A value of 3 + # means UTF-16be. No other values are allowed." HeaderValid allows 0 + # too, when the schema format is 0: SQLite writes both as 0 while a + # database has no schema, so no text is stored yet. + if encoding > 3: + return None, WHY_H_ENCODING + if encoding == 0 and schema_format != 0: + return None, WHY_H_ENCODING_0 + # "Reserved for expansion. Must be zero." + if any(f[72:92]): + return None, WHY_H_EXPANSION + return (page_size, write_version, read_version, reserved, *words, be(f[92:96]), + be(f[96:100])), None + + +def max_local(t, U): + """X: "U-35" on a table leaf page, "((U-12)*64/255)-23" on the + others.""" + if t == TABLE_LEAF: + return nsub(U, 35) + return nsub(ndiv(nsub(U, 12) * 64, 255), 23) + + +def min_local(U): + """M: "((U-12)*32/255)-23".""" + return nsub(ndiv(nsub(U, 12) * 32, 255), 23) + + +def local_size(t, U, P): + """How many bytes of a payload of P bytes stay on a page of type t + and usable size U (1.6): "If P<=X then all P bytes of payload are + stored directly on the btree page without overflow. If P>X and K<=X + then the first K bytes of P are stored on the btree page ... If P>X + and K>X then the first M bytes of P are stored on the btree page", + with K = M+((P-M)%(U-4)).""" + X = max_local(t, U) + if P <= X: + return P + M = min_local(U) + K = M + nmod(nsub(P, M), nsub(U, 4)) + return K if K <= X else M + + +def int64(n): + """A 64-bit two's complement integer from its bits.""" + return n - (1 << 64) if n >= 1 << 63 else n + + +def read_cell(t, U, page, p): + """The cell at offset p of a page of type t, reading no byte at or + past the usable size U: (left child, payload size, rowid, the part + of the payload on the page, first overflow page), and its length; + or None. The parts, in order (1.6): a table leaf cell has the + payload size, the rowid, the payload; a table interior cell the left + child and the rowid; an index leaf cell the payload size and the + payload; an index interior cell the left child, the payload size and + the payload; a payload that spills is followed by "A 4-byte + big-endian integer page number for the first page of the overflow + page list".""" + pos = p + left = size = rowid = overflow = None + kept = b"" + if t in INTERIOR: + if pos + 4 > U: + return None + left = be(page[pos:pos + 4]) + pos += 4 + if t != TABLE_INTERIOR: + r = read_varint(page, pos, U) + if r is None: + return None + size, k = r + pos += k + if t in (TABLE_LEAF, TABLE_INTERIOR): + r = read_varint(page, pos, U) + if r is None: + return None + rowid = int64(r[0]) + pos += r[1] + if t != TABLE_INTERIOR: + L = local_size(t, U, size) + if pos + L > U: + return None + kept = bytes(page[pos:pos + L]) + pos += L + if L < size: + if pos + 4 > U: + return None + overflow = be(page[pos:pos + 4]) + pos += 4 + return (left, size, rowid, kept, overflow), pos - p + + +WHY_P_HEADER = "the file's header is refused" +WHY_P_ZERO = "page 0: pages are numbered from 1 (1.2)" +WHY_P_PAST = "the page is not all in the file (1.2)" +WHY_P_TYPE = "a page type other than 2, 5, 10 and 13 (1.6)" +WHY_P_ARRAY = "the cell pointer array runs past the start of the cell content area (1.6)" +WHY_P_CONTENT = "the cell content area starts past the usable size (1.6)" +WHY_P_CELL_START = "a cell starts before the cell content area (1.6)" +WHY_P_CELL_END = "a cell runs past the usable size (1.6)" + + +def read_page(h, f, n): + """Page n of the file f with header h, as a b-tree page: (type, first + freeblock, cell count, content start, fragmented bytes, right-most + pointer, cell pointers, cells), and None; or None and why.""" + ps = h[H_PAGE_SIZE] + if n < 1: + return None, WHY_P_ZERO + start = (n - 1) * ps + if start + ps > len(f): + return None, WHY_P_PAST + page = f[start:start + ps] + # "The 100-byte database file header is found only on page 1" + off = 100 if n == 1 else 0 + U = ps - h[H_RESERVED] + t = page[off] + if t not in PAGE_TYPES: + return None, WHY_P_TYPE + hs = 12 if t in INTERIOR else 8 + first_freeblock = be(page[off + 1:off + 3]) + count = be(page[off + 3:off + 5]) + # "A zero value for this integer is interpreted as 65536." + content = be(page[off + 5:off + 7]) or 65536 + fragmented = page[off + 7] + right = be(page[off + 8:off + 12]) if t in INTERIOR else None + if off + hs + 2 * count > content: + return None, WHY_P_ARRAY + if content > U: + return None, WHY_P_CONTENT + pointers = [be(page[off + hs + 2 * i:off + hs + 2 * i + 2]) for i in range(count)] + cells = [] + for p in pointers: + if p < content: + return None, WHY_P_CELL_START + c = read_cell(t, U, page, p) + if c is None: + return None, WHY_P_CELL_END + cells.append(c[0]) + return (t, first_freeblock, count, content, fragmented, right, pointers, cells), None + + +# "Serial Type Codes Of The Record Format": the content size of types 0 +# to 9; 10 and 11 are "Reserved for internal use". +FIXED_SIZES = {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 6, 6: 8, 7: 8, 8: 0, 9: 0} + + +def content_size(t): + if t in FIXED_SIZES: + return FIXED_SIZES[t] + if t in (10, 11): + return None + return (t - 12) // 2 if t % 2 == 0 else (t - 13) // 2 + + +def hx(b): + """Bytes as `sqlite serve` writes them: hex, or `-` for none.""" + return bytes(b).hex() or "-" + + +def value_of(t, c): + """The value of a column of serial type t with content c, written as + `sqlite serve` writes it.""" + if t == 0: + return "null" + if t <= 6: + return f"int:{int.from_bytes(c, 'big', signed=True)}" + if t == 7: + return f"real:{be(c)}" + if t == 8: + return "int:0" + if t == 9: + return "int:1" + return ("blob:" if t % 2 == 0 else "text:") + hx(c) + + +WHY_R_SIZE_ENDS = "the header size varint does not end (2.1)" +WHY_R_SIZE_SMALL = "a header size smaller than its own varint (2.1)" +WHY_R_SIZE_PAST = "a header size past the end of the record (2.1)" +WHY_R_TYPE_PAST = "a serial type runs past the end of the header (2.1)" +WHY_R_NO_COLUMN = "no column (2.1: \"one or more additional varints\")" +WHY_R_RESERVED = "serial type 10 or 11, \"Reserved for internal use\" (2.1)" +WHY_R_SHORT = "the body ends before the last column (2.1)" +WHY_R_LONG = "bytes after the last column (Spec/Defs.lean RecordEncodes)" + + +def read_record(p): + """The values of the record p (2.1), and None; or None and why. "The + header begins with a single varint which determines the total number + of bytes in the header. The varint value is the size of the header + in bytes including the size varint itself. Following the size varint + are one or more additional varints, one per column." The values of + the columns follow the header, one after another.""" + r = read_varint(p) + if r is None: + return None, WHY_R_SIZE_ENDS + hsize, n = r + if hsize < n: + return None, WHY_R_SIZE_SMALL + if hsize > len(p): + return None, WHY_R_SIZE_PAST + types, pos = [], n + while pos < hsize: + r = read_varint(p, pos, hsize) + if r is None: + return None, WHY_R_TYPE_PAST + types.append(r[0]) + pos += r[1] + if not types: + return None, WHY_R_NO_COLUMN + values, body = [], hsize + for t in types: + size = content_size(t) + if size is None: + return None, WHY_R_RESERVED + if body + size > len(p): + return None, WHY_R_SHORT + values.append(value_of(t, p[body:body + size])) + body += size + if body != len(p): + return None, WHY_R_LONG + return values, None + + +def opt(x): + return "-" if x is None else str(x) + + +def show_header(h): + return " ".join(str(x) for x in h) + + +def show_cell(p, c): + left, size, rowid, kept, overflow = c + return f"{p}/{opt(left)}/{opt(size)}/{opt(rowid)}/{hx(kept)}/{opt(overflow)}" + + +def show_page(pg): + t, first_freeblock, count, content, fragmented, right, pointers, cells = pg + return (f"{t} {first_freeblock} {count} {content} {fragmented} {opt(right)} |" + + "".join(" " + show_cell(p, c) for p, c in zip(pointers, cells))) + + +WHY_V_ENDS = "the varint does not end (1.6)" + + +def model(op, args): + """The model's answer to a request, and why it is FAIL (or None).""" + if op == "varint": + r = read_varint(args[0]) + if r is None: + return "FAIL", WHY_V_ENDS + return f"OK {r[0]} {r[1]} {hx(args[0][r[1]:])}", None + if op == "header": + h, why = read_header(args[0]) + return ("FAIL", why) if h is None else ("OK " + show_header(h), None) + if op == "page": + n, f = args + h, why = read_header(f) + if h is None: + return "FAIL", WHY_P_HEADER + pg, why = read_page(h, f, n) + return ("FAIL", why) if pg is None else ("OK " + show_page(pg), None) + if op == "record": + vs, why = read_record(args[0]) + return ("FAIL", why) if vs is None else ("OK " + " ".join(vs), None) + if op == "local": + t, U, P = args + return f"OK {local_size(t, U, P)}", None + raise ValueError(op) + + +def request(op, args): + if op == "page": + return f"page {args[0]} {hx(args[1])}" + if op == "local": + return f"local {args[0]} {args[1]} {args[2]}" + return f"{op} {hx(args[0])}" + + +def model_dump(f): + """What `sqlite dump` prints for the file f, as lines, and its exit + status, by the header of Main.lean: the header, then each whole page + in turn, read or FAIL, and after a page the record of each cell + whose payload is all on its page and not empty; nothing, and exit + status 1, when the header is refused.""" + h, _ = read_header(f) + if h is None: + return [], 1 + out = ["header " + show_header(h)] + ps = h[H_PAGE_SIZE] + for n in range(1, len(f) // ps + 1): + pg, _ = read_page(h, f, n) + if pg is None: + out.append(f"page {n} FAIL") + continue + out.append(f"page {n} {show_page(pg)}") + for p, c in zip(pg[6], pg[7]): + # All on its page (no overflow page) and not empty. + if c[4] is None and len(c[3]) > 0: + vs, _ = read_record(c[3]) + out.append(f"record {n} {p} {opt(c[2])} " + ("FAIL" if vs is None else " ".join(vs))) + return out, 0 + + +HEX_DIGITS = frozenset("0123456789abcdef") +DIGITS = frozenset("0123456789") + + +def hex_field(f): + if f == "-": + return b"" + if not f or len(f) % 2 or any(c not in HEX_DIGITS for c in f): + return None + return bytes.fromhex(f) + + +def decimal_field(f): + if not f or any(c not in DIGITS for c in f): + return None + return int(f) + + +def parse_request(line): + """The operation and arguments of a request line, or None when it is + malformed.""" + fs = line.split(" ") + op = fs[0] + if op in ("varint", "header", "record") and len(fs) == 2: + b = hex_field(fs[1]) + return None if b is None else (op, [b]) + if op == "page" and len(fs) == 3: + n, b = decimal_field(fs[1]), hex_field(fs[2]) + return None if n is None or b is None else (op, [n, b]) + if op == "local" and len(fs) == 4: + t, U, P = (decimal_field(x) for x in fs[1:]) + if t not in PAGE_TYPES or U is None or P is None: + return None + return (op, [t, U, P]) + return None + + +def model_line(raw): + """What the tool must answer to a request line, malformed or not.""" + line = raw[:-1] if raw.endswith("\n") else raw + if raw.endswith("\n") and line.endswith("\r"): + line = line[:-1] + req = parse_request(line) + return "error" if req is None else model(*req)[0] + + +# --------------------------------------------------------------------- +# Spec.lean's `examples` and `rejects`, with the answers they state. + +# Spec/Examples.lean `sqliteHeaderBytes`: the first 100 bytes of a +# database SQLite 3.45.1 wrote. +SPEC_HEADER = bytes([ + 0x53, 0x51, 0x4C, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6F, 0x72, 0x6D, 0x61, 0x74, 0x20, 0x33, 0x00, + 0x02, 0x00, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x2E, 0x76, 0x89]) +# `onePageFile`: that header, then a table leaf page with one cell at 507. +ONE_PAGE_FILE = (SPEC_HEADER + bytes([0x0D, 0, 0, 0, 1, 1, 0xFB, 0, 1, 0xFB]) + bytes(397) + + bytes([3, 1, 2, 1, 5])) + + +def spec_header_with(**changes): + """`headerBytes { sqliteHeader with ... }`.""" + b = bytearray(SPEC_HEADER) + for k, v in changes.items(): + if k == "pageSize": + b[16:18] = (1 if v == 65536 else v).to_bytes(2, "big") + elif k == "readVersion": + b[19] = v + elif k == "reserved": + b[20] = v + elif k == "textEncoding": + b[56:60] = v.to_bytes(4, "big") + return bytes(b) + + +SPEC_CASES = [ + # examples + ("varint", [b"\x00"], "OK 0 1 -"), + ("varint", [b"\x7f\x55"], "OK 127 1 55"), + ("varint", [b"\x81\x00"], "OK 128 2 -"), + ("varint", [b"\xff" * 9], f"OK {2 ** 64 - 1} 9 -"), + ("local", [TABLE_LEAF, 4096, 10000], "OK 1816"), + ("record", [bytes([3, 1, 0x13, 7, 0x61, 0x62, 0x63])], "OK int:7 text:616263"), + ("header", [SPEC_HEADER], "OK 512 1 1 0 2 3 0 0 1 4 0 0 1 0 0 0 2 3045001"), + ("page", [1, ONE_PAGE_FILE], "OK 13 0 1 507 0 - | 507/-/3/1/020105/-"), + ("record", [bytes([2, 1, 5])], "OK int:5"), + # rejects + ("varint", [b"\x81\x80"], "FAIL"), + ("header", [bytes(16) + SPEC_HEADER[16:]], "FAIL"), + ("header", [spec_header_with(pageSize=768)], "FAIL"), + ("header", [spec_header_with(readVersion=3)], "FAIL"), + ("header", [spec_header_with(reserved=33)], "FAIL"), + ("header", [spec_header_with(textEncoding=0)], "FAIL"), + ("header", [SPEC_HEADER[:21] + bytes([63]) + SPEC_HEADER[22:]], "FAIL"), + ("page", [2, ONE_PAGE_FILE], "FAIL"), + ("page", [0, ONE_PAGE_FILE], "FAIL"), + ("record", [bytes([2, 0x0A])], "FAIL"), + ("record", [bytes([2, 1, 7, 0])], "FAIL"), + ("record", [bytes([5, 1])], "FAIL"), + ("record", [bytes([1])], "FAIL"), +] + +# --------------------------------------------------------------------- +# Encoders, for building inputs. + + +def varint_len(v): + for k in range(1, 9): + if v < 1 << (7 * k): + return k + return 9 + + +def enc_varint(v, length=None): + """v (below 2^64) as a varint of `length` bytes, the fewest when + None; a varint of fewer than 9 bytes holds 7 bits a byte.""" + length = varint_len(v) if length is None else length + if length == 9: + a = v >> 8 + return bytes(((a >> (7 * (7 - i))) & 0x7F) | 0x80 for i in range(8)) + bytes([v & 0xFF]) + assert v < 1 << (7 * length) + return bytes(((v >> (7 * (length - 1 - i))) & 0x7F) | (0x80 if i < length - 1 else 0) + for i in range(length)) + + +def some_varint(rng, v): + """v as a varint, mostly in the fewest bytes, sometimes in more.""" + k = varint_len(v) + if k < 9 and rng.random() < 0.15: + k = rng.randint(k, 9) + return enc_varint(v, k) + + +INT_EDGES = sorted({s * (2 ** k + d) for k in (0, 7, 8, 15, 16, 23, 24, 31, 32, 47, 48, 53, 63) + for d in (-1, 0, 1) for s in (1, -1) + if -2 ** 63 <= s * (2 ** k + d) < 2 ** 63} | {0, 1, 2, -2}) +INT_WIDTH = {1: 1, 2: 2, 3: 3, 4: 4, 5: 6, 6: 8} + + +def random_int(rng): + if rng.random() < 0.4: + return rng.choice(INT_EDGES) + bits = rng.randint(0, 63) + v = rng.getrandbits(bits) if bits else 0 + return -v - 1 if rng.random() < 0.5 else v + + +def int_type(v, rng=None): + """The serial type for v in the fewest bytes, or, with rng, + sometimes a wider one.""" + for t in range(1, 7): + if -(1 << (8 * INT_WIDTH[t] - 1)) <= v < 1 << (8 * INT_WIDTH[t] - 1): + if rng is not None and rng.random() < 0.15: + return rng.randint(t, 6) + return t + raise ValueError(v) + + +def blob_size(rng): + r = rng.random() + if r < 0.6: + return rng.randint(0, 20) + if r < 0.94: + return rng.randint(21, 150) + if r < 0.999: + return rng.randint(151, 3000) + return rng.randint(3001, 70000) + + +def random_column(rng): + """A serial type and content.""" + r = rng.random() + if r < 0.08: + return 0, b"" + if r < 0.45: + v = random_int(rng) + t = int_type(v, rng) + return t, v.to_bytes(INT_WIDTH[t], "big", signed=True) + if r < 0.52: + return rng.choice((8, 9)), b"" + if r < 0.64: + return 7, rng.choice([struct.pack(">d", x) for x in (0.0, -0.0, 1.0, 1.5, -2.25, 1e300, + 5e-324, float("inf"))] + + [rng.randbytes(8)] * 4) + n = blob_size(rng) + return (12 if r < 0.82 else 13) + 2 * n, rng.randbytes(n) + + +def record_bytes(rng, cols, header_len=None): + """A record of the columns (serial type, content).""" + types = b"".join(some_varint(rng, t) for t, _ in cols) + body = b"".join(c for _, c in cols) + k = 1 + while varint_len(k + len(types)) > k: + k += 1 + if header_len is not None: + k = max(k, header_len) + elif k < 9 and rng.random() < 0.08: + k = rng.randint(k, 9) + return enc_varint(k + len(types), k) + types + body + + +# --------------------------------------------------------------------- +# Inputs that are not databases. Each item is (label, op, args). + + +def varint_sets(rng): + for x in range(256): + yield ("every varint of one byte", "varint", [bytes([x])]) + for x in range(65536): + yield ("every two bytes", "varint", [x.to_bytes(2, "big")]) + for k in range(1, 10): + for v in (0, 1, 127, 128, (1 << (7 * k)) - 1, 1 << (7 * k), 2 ** 63, 2 ** 64 - 1): + if v < 2 ** 64 and varint_len(v) <= k: + yield ("each length, at its bounds", "varint", [enc_varint(v, k)]) + yield ("each length, at its bounds, with bytes after", "varint", + [enc_varint(v, k) + rng.randbytes(rng.randint(1, 4))]) + yield ("each length, at its bounds, cut short", "varint", [enc_varint(v, k)[:-1]]) + for last in range(256): + yield ("nine bytes, every ninth byte", "varint", [rng.randbytes(8) + bytes([last])]) + yield ("nine bytes, every ninth byte", "varint", + [bytes(x | 0x80 for x in rng.randbytes(8)) + bytes([last])]) + + +def random_varint(rng): + k = rng.randint(0, 12) + b = bytearray(rng.randbytes(k)) + for i in range(min(k, rng.randint(0, 10))): + b[i] |= 0x80 + return bytes(b) + + +def make_header(rng, page_size, reserved, **fields): + """A header this reader accepts, with random fields.""" + words = { + H_CHANGE: rng.getrandbits(32), H_SIZE: rng.getrandbits(rng.choice((4, 16, 32))), + H_TRUNK: rng.getrandbits(rng.choice((0, 8, 32))), H_FREE: rng.getrandbits(rng.choice((0, 8, 32))), + H_COOKIE: rng.getrandbits(32), H_SCHEMA: rng.randint(0, 4), + H_CACHE: rng.getrandbits(32), H_LARGEST: rng.getrandbits(rng.choice((0, 8, 32))), + H_ENCODING: rng.randint(1, 3), H_USER: rng.getrandbits(32), + H_INCREMENTAL: rng.getrandbits(rng.choice((0, 1, 32))), H_APPID: rng.getrandbits(32), + } + # A text encoding of 0, as SQLite writes it while a database has no + # schema, goes only with a schema format of 0. + if words[H_SCHEMA] == 0 and rng.random() < 0.5: + words[H_ENCODING] = 0 + words.update(fields) + b = bytearray(MAGIC) + b += (1 if page_size == 65536 else page_size).to_bytes(2, "big") + b += bytes([fields.get(H_WRITE, rng.choice((1, 2, 0, 3, 255))), fields.get(H_READ, rng.choice((1, 2))), + reserved, 64, 32, 32]) + for i in range(H_CHANGE, H_APPID + 1): + b += words[i].to_bytes(4, "big") + b += bytes(20) + b += rng.getrandbits(32).to_bytes(4, "big") + rng.getrandbits(32).to_bytes(4, "big") + return bytes(b) + + +def random_reserved(rng, page_size): + if rng.random() < 0.5: + return 0 + return rng.randint(1, min(255, page_size - 480)) + + +def random_header(rng): + """A header from random fields, each mostly one this reader accepts.""" + ps = rng.choice(PAGE_SIZES) + b = bytearray(make_header(rng, ps, random_reserved(rng, ps))) + for _ in range(rng.choice((0, 1, 1, 2, 3))): + what = rng.randrange(10) + if what == 0: + b[rng.randrange(16)] = rng.randrange(256) + elif what == 1: + b[16:18] = rng.choice((0, 1, 2, 256, 511, 513, 768, 32768, 65535, rng.getrandbits(16), + rng.choice(PAGE_SIZES[:-1]))).to_bytes(2, "big") + elif what == 2: + b[18 + rng.randrange(2)] = rng.choice((0, 1, 2, 3, 255, rng.randrange(256))) + elif what == 3: + b[20] = rng.randrange(256) + elif what == 4: + b[21 + rng.randrange(3)] = rng.choice((0, 31, 32, 33, 63, 64, 65, rng.randrange(256))) + elif what == 5: + b[44:48] = rng.choice((0, 4, 5, 2 ** 32 - 1, rng.getrandbits(32))).to_bytes(4, "big") + elif what == 6: + b[56:60] = rng.choice((0, 1, 3, 4, 2 ** 32 - 1, rng.getrandbits(32))).to_bytes(4, "big") + elif what == 7: + b[72 + rng.randrange(20)] = rng.randrange(1, 256) + elif what == 8: + # A text encoding of 0, with a schema format from 0 to 4. + b[44:48] = rng.randint(0, 4).to_bytes(4, "big") + b[56:60] = bytes(4) + else: + b[rng.randrange(100)] = rng.randrange(256) + return bytes(b) + rng.randbytes(rng.choice((0, 0, 1, 50))) + + +def random_record(rng): + n = rng.choice((1, 1, 2, 3, 4, 6, 10)) if rng.random() < 0.98 else rng.randint(11, 80) + cols = [random_column(rng) for _ in range(n)] + return cols, record_bytes(rng, cols) + + +def changed_record(rng, cols, rec): + """The record changed in one way; a label and the bytes.""" + what = rng.randrange(12) + if what == 0: + i = rng.randrange(len(cols)) + cols = cols[:i] + [(rng.choice((10, 11)), cols[i][1])] + cols[i + 1:] + return "serial type 10 or 11", record_bytes(rng, cols) + if what == 1: + i = rng.randrange(len(cols)) + t = rng.choice((2 ** 64 - 1, 2 ** 64 - 2, 2 ** 63, rng.getrandbits(64), 2 * len(rec) + 12, + 2 * len(rec) + 13)) + cols = cols[:i] + [(t, cols[i][1])] + cols[i + 1:] + return "a huge serial type", record_bytes(rng, cols) + if what == 2: + return "the body one byte longer", rec + rng.randbytes(1) + if what == 3: + return "the body one byte shorter", rec[:-1] + if what == 4: + return "no body", record_bytes(rng, [(t, b"") for t, _ in cols]) + r = read_varint(rec) + hsize, k = r + rest = rec[k:] + if what == 5: + return "the header size one more", enc_varint(hsize + 1, k if varint_len(hsize + 1) <= k else None) + rest + if what == 6: + return "the header size one less", enc_varint(hsize - 1, k) + rest + if what == 7: + small = rng.randrange(0, 2) + return "a header size smaller than its varint", enc_varint(small, rng.randint(2, 9)) + rest + if what == 8: + return "a header size past the record", enc_varint(len(rec) + rng.randint(1, 300)) + rest + if what == 9: + return "a serial type running past the header", enc_varint(hsize, k) + rest[:hsize - k - 1] + bytes( + [rec[hsize - 1] | 0x80]) + rest[hsize - k:] + if what == 10: + return "no column", bytes([1]) + rng.randbytes(rng.choice((0, 0, 1, 3))) + b = bytearray(rec) + b[rng.randrange(len(b))] ^= 1 << rng.randrange(8) + return "a bit flipped", bytes(b) + + +def local_sets(rng): + us = list(range(0, 41)) + list(range(470, 530)) + us += [ps - r for ps in PAGE_SIZES for r in (0, 1, 2, 3, 8, 12, 17, 32, 33, 64, 100, 128, 200, 255) + if ps - r >= 0] + us += [65537, 2 ** 20, 2 ** 32 + 5, 10 ** 20] + for U in sorted(set(us)): + for t in PAGE_TYPES: + X, M = max_local(t, U), min_local(U) + ps = {0, 1, 2, U, 2 * U, 10 ** 25, 2 ** 64 - 1} + for d in range(-3, 4): + ps |= {X + d, M + d} + for j in (1, 2, 3): + ps |= {X + j * nsub(U, 4) + d, M + j * nsub(U, 4) + d, + X + j * nsub(U, 4) + (X - M) + d} + for _ in range(4): + ps.add(rng.randrange(4 * U + 10)) + for P in sorted(p for p in ps if p >= 0): + yield ("a grid of usable and payload sizes", "local", [t, U, P]) + for U in (480, 512): + for t in PAGE_TYPES: + for P in range(0, 3 * U + 1): + yield ("every payload size up to three pages", "local", [t, U, P]) + + +def random_local(rng): + U = rng.choice((rng.randrange(480, 65537), rng.randrange(0, 600), rng.getrandbits(rng.randint(1, 40)))) + P = rng.choice((rng.randrange(0, 4 * U + 10), rng.getrandbits(rng.randint(1, 64)))) + return [rng.choice(PAGE_TYPES), U, P] + + +# --------------------------------------------------------------------- +# Built b-tree pages. + + +def random_key(rng): + if rng.random() < 0.3: + return rng.choice((0, 1, 2 ** 63 - 1, 2 ** 63, 2 ** 64 - 1, 2 ** 63 + 1, 127, 128)) + return rng.getrandbits(rng.choice((7, 14, 32, 63, 64))) + + +def payload_size(rng, t, U): + """A payload size, often near X, M or K.""" + X, M, W = max_local(t, U), min_local(U), U - 4 + choices = [rng.randint(0, 40), X - rng.randint(0, 3), X + rng.randint(1, 3), M + rng.randint(-2, 2), + X + rng.randint(1, 3) * W + rng.randint(-2, 2), + M + rng.randint(1, 3) * W + rng.randint(-2, 2), + X + rng.randint(1, 3) * W + (X - M) + rng.randint(-2, 2), + rng.randint(0, 4 * U), rng.getrandbits(rng.choice((16, 32, 64)))] + return min(max(0, rng.choice(choices)), 2 ** 64 - 1) + + +def random_cell(rng, t, U): + """A cell of type t for usable size U, as bytes, and the bytes of its + payload kept on the page, a record when one fits.""" + if t == TABLE_INTERIOR: + return rng.getrandbits(32).to_bytes(4, "big") + some_varint(rng, random_key(rng)) + P = payload_size(rng, t, U) + L = local_size(t, U, P) + kept = rng.randbytes(L) + if L == P and rng.random() < 0.5: + cols, rec = random_record(rng) + if len(rec) <= max_local(t, U): + P = L = len(rec) + kept = rec + out = rng.getrandbits(32).to_bytes(4, "big") if t == INDEX_INTERIOR else b"" + out += some_varint(rng, P) + if t == TABLE_LEAF: + out += some_varint(rng, random_key(rng)) + out += kept + if L < P: + out += rng.getrandbits(32).to_bytes(4, "big") + return out + + +def page_file(rng, ps, reserved, n, t, content, cells, count=None): + """A file whose page n (1 or 2) has type t, the content start + `content` and the cells [(offset, bytes)], pointers in order.""" + off = 100 if n == 1 else 0 + hs = 12 if t in INTERIOR else 8 + page = bytearray(ps) + for p, c in cells: + page[p:p + len(c)] = c[:ps - p] + page[off] = t + page[off + 3:off + 5] = (len(cells) if count is None else count).to_bytes(2, "big") + page[off + 5:off + 7] = (content % 65536).to_bytes(2, "big") + if t in INTERIOR: + page[off + 8:off + 12] = rng.getrandbits(32).to_bytes(4, "big") + for i, (p, _) in enumerate(cells): + page[off + hs + 2 * i:off + hs + 2 * i + 2] = p.to_bytes(2, "big") + header = make_header(rng, ps, reserved) + if n == 1: + page[:100] = header + return bytes(page) + return header + bytes(ps - 100) + bytes(page) + + +def edge_pages(rng): + """Pages at the edges of PageEncodes: empty pages whose content + start is just inside or outside the usable size, or 0; one cell that + ends just inside or outside the usable size, or starts just before + the content area; a content start just before, at and after the end + of the pointer array.""" + for ps, reserved in ((512, 0), (512, 5), (4096, 0), (4096, 5), (65536, 0), (65536, 5)): + U = ps - reserved + for n in ((1,) if ps == 65536 else (1, 2)): + off = 100 if n == 1 else 0 + for t in PAGE_TYPES: + hs = 12 if t in INTERIOR else 8 + for content in sorted({U - 1, U, U + 1, ps, 65536}): + if content <= 65536: + yield ("an edge of a page, empty, content start U-1 to 65536", "page", + [n, page_file(rng, ps, reserved, n, t, content, [])]) + c = random_cell(rng, t, U) + for end in (U - 1, U, U + 1): + p = end - len(c) + if p >= off + hs + 2 and end <= ps: + yield ("an edge of a page, a cell ending at U-1, U or U+1", "page", + [n, page_file(rng, ps, reserved, n, t, p, [(p, c)])]) + yield ("an edge of a page, a cell starting one byte before the content area", + "page", [n, page_file(rng, ps, reserved, n, t, p + 1, [(p, c)])]) + p = U - len(c) + floor = off + hs + 2 + for content in (floor - 1, floor, floor + 1): + if p >= floor + 1: + yield ("an edge of a page, a content start around the end of the pointer array", + "page", [n, page_file(rng, ps, reserved, n, t, content, [(p, c)])]) + yield ("an edge of a page, a content start around the end of the pointer array", + "page", [n, page_file(rng, ps, reserved, n, t, content + 2, [], count=1)]) + + +PAGE_DEFECTS = ["none"] * 6 + [ + "the content start past the first cell", "the content start before the pointer array ends", + "the content start past the usable size", "the content start 0", "a cell pointer before the content area", + "a cell running past the usable size", "the cell count one more", "the cell count one less", + "a payload size one more", "a byte changed", "the page type changed"] + + +def built_page(rng, ps, reserved, n): + """A file whose page n (1 or 2) is a b-tree page built from random + parts, and the defect it was given. A defect that cannot be made, or + would change nothing, is not made, and the defect is "none".""" + U = ps - reserved + off = 100 if n == 1 else 0 + t = rng.choice(PAGE_TYPES) if rng.random() < 0.97 else rng.randrange(256) + tt = t if t in PAGE_TYPES else TABLE_LEAF + hs = 12 if tt in INTERIOR else 8 + want = rng.choice((0, 0, 1, 2, 3, 5, 10, 30, 100)) + cells, pos = [], U + for _ in range(want): + c = random_cell(rng, tt, U) + gap = rng.choice((0, 0, 0, 1, 3, 4, 10)) + if pos - gap - len(c) < off + hs + 2 * (len(cells) + 1): + break + pos -= gap + len(c) + cells.append((pos, c)) + floor = off + hs + 2 * len(cells) + content = pos if rng.random() < 0.8 else rng.randint(floor, pos) + page = bytearray(rng.randbytes(ps) if rng.random() < 0.5 else bytes(ps)) + pointers = [p for p, _ in cells] + if rng.random() < 0.3: + rng.shuffle(pointers) + count = len(pointers) + # The cells first, so that a defect that moves one is not written over. + for p, c in cells: + page[p:p + len(c)] = c + # The payload size varints that can hold one more in their bytes. + grow = [] + if tt != TABLE_INTERIOR: + for p, _ in cells: + q = p + (4 if tt == INDEX_INTERIOR else 0) + r = read_varint(page, q) + if r and r[0] + 1 < 2 ** 64 and varint_len(r[0] + 1) <= r[1]: + grow.append((q, r)) + defect = rng.choice(PAGE_DEFECTS) + if defect == "the content start past the first cell" and cells: + content = min(pointers) + 1 + elif defect == "the content start before the pointer array ends": + content = max(1, floor - 1) + elif defect == "the content start past the usable size": + content = U + rng.randint(1, 3) + elif defect == "the content start 0" and content != 65536: + content = 65536 + elif defect == "a cell pointer before the content area" and cells: + pointers[rng.randrange(count)] = max(0, content - rng.randint(1, 4)) + elif defect == "a cell running past the usable size" and cells: + i = rng.randrange(count) + c = dict((p, c) for p, c in cells)[pointers[i]] + p = U - len(c) + rng.randint(1, 4) + page[p:p + len(c)] = c[:max(0, ps - p)] + pointers[i] = p + elif defect == "the cell count one more": + count += 1 + elif defect == "the cell count one less" and count: + count -= 1 + elif defect == "a payload size one more" and grow: + q, r = rng.choice(grow) + page[q:q + r[1]] = enc_varint(r[0] + 1, r[1]) + elif defect not in ("a byte changed", "the page type changed"): + defect = "none" + page[off] = t + page[off + 1:off + 3] = rng.choice((0, rng.getrandbits(16))).to_bytes(2, "big") + page[off + 3:off + 5] = min(count, 65535).to_bytes(2, "big") + page[off + 5:off + 7] = (content % 65536 if content <= 65536 else content % 65536).to_bytes(2, "big") + page[off + 7] = rng.choice((0, rng.randrange(256))) + if tt in INTERIOR: + page[off + 8:off + 12] = rng.getrandbits(32).to_bytes(4, "big") + for i, p in enumerate(pointers[:count] + [rng.getrandbits(16)] * (count - len(pointers))): + page[off + hs + 2 * i:off + hs + 2 * i + 2] = (p % 65536).to_bytes(2, "big") + if defect == "a byte changed": + i = rng.randrange(off, ps) + page[i] = (page[i] + rng.randrange(1, 256)) % 256 + elif defect == "the page type changed": + page[off] = {2: 10, 10: 2, 5: 13, 13: 5}.get(t, rng.choice(PAGE_TYPES)) + header = make_header(rng, ps, reserved) + if n == 1: + page[:100] = header + return bytes(page), defect + return header + bytes(ps - 100) + bytes(page), defect + + +# --------------------------------------------------------------------- +# Pages of real databases, changed. + +INTERESTING_16 = (0, 1, 2, 7, 8, 12, 99, 100, 107, 108, 112, 255, 256, 479, 480, 511, 512, 513, + 1023, 1024, 4095, 4096, 32767, 32768, 65534, 65535) + + +# The changes of changed_page that change the file's length or its +# 100-byte header. +RESHAPING = ("the reserved byte of the header changed", "the page size field changed", "the file cut short") + + +def changed_page(rng, h, f, n, keep_shape=False): + """f, a file with header h, with page n changed in one way, so that + the bytes differ from f; the label and the bytes. With `keep_shape`, + only the changes that keep the file's length and its 100-byte + header. A change that leaves the bytes as they were is made again, + from f, until they differ.""" + off = (n - 1) * h[H_PAGE_SIZE] + (100 if n == 1 else 0) + pg, _ = read_page(h, f, n) + t = f[off] + count = be(f[off + 3:off + 5]) + kinds = ["a bit flipped", "a page header byte changed", "the cell count changed", + "the content start changed", "the page type changed", "the reserved byte of the header changed", + "the page size field changed", "the file cut short", "the freeblock and fragment fields changed"] + if count: + kinds.append("a cell pointer changed") + if pg is not None and pg[2]: + kinds.append("a byte of a cell changed") + if t != TABLE_INTERIOR: + kinds += ["a payload size changed"] * 2 + if t in (TABLE_LEAF, TABLE_INTERIOR): + kinds.append("a rowid changed") + if any(c[4] is not None for c in pg[7]): + kinds.append("an overflow page number changed") + if pg is not None and t in INTERIOR: + kinds.append("the right-most pointer changed") + if keep_shape: + kinds = [k for k in kinds if k not in RESHAPING] + what = rng.choice(kinds) + for _ in range(1000): + b = bytearray(f) + change_page(rng, b, what, h, f, n, pg) + if keep_shape and n == 1 and b[:100] != f[:100]: + continue # a bit flipped in the header + if b != f: + return what, bytes(b) + raise AssertionError(f"{what}: page {n} unchanged after 1000 tries") + + +def change_page(rng, b, what, h, f, n, pg): + """Make the change `what` to page n of b, a copy of f.""" + ps, U = h[H_PAGE_SIZE], h[H_PAGE_SIZE] - h[H_RESERVED] + base = (n - 1) * ps + off = base + (100 if n == 1 else 0) + t = f[off] + hs = 12 if t in INTERIOR else 8 + count = be(f[off + 3:off + 5]) + if what == "a page header byte changed": + b[off + rng.randrange(hs)] = rng.choice((0, 1, 2, 5, 10, 13, 255, rng.randrange(256))) + elif what == "the cell count changed": + b[off + 3:off + 5] = max(0, min(65535, count + rng.choice((-2, -1, 1, 2, 100, 65535)))).to_bytes(2, "big") + elif what == "the content start changed": + cs = be(b[off + 5:off + 7]) or 65536 + floor = (off - base) + hs + 2 * count + new = rng.choice((0, 1, floor - 1, floor, floor + 1, cs - 1, cs + 1, U - 1, U, U + 1, + rng.choice(INTERESTING_16))) + b[off + 5:off + 7] = (max(0, new) % 65536).to_bytes(2, "big") + elif what == "a cell pointer changed": + i = rng.randrange(min(count, (ps - (off - base) - hs) // 2)) + cs = be(b[off + 5:off + 7]) or 65536 + new = rng.choice((cs - 1, cs, U - 1, U - 4, U, ps - 1, rng.choice(INTERESTING_16), rng.randrange(ps))) + b[off + hs + 2 * i:off + hs + 2 * i + 2] = (max(0, new) % 65536).to_bytes(2, "big") + elif what == "a payload size changed": + q = base + rng.choice(pg[6]) + (4 if t == INDEX_INTERIOR else 0) + r = read_varint(b, q) + P = payload_size(rng, t, U) + if varint_len(P) > r[1]: + P = rng.getrandbits(7 * r[1]) if r[1] < 9 else rng.getrandbits(64) + b[q:q + r[1]] = enc_varint(P, r[1]) + elif what == "a rowid changed": + i = rng.randrange(pg[2]) + q = base + pg[6][i] + 4 + if t == TABLE_LEAF: + q = base + pg[6][i] + read_varint(b, base + pg[6][i])[1] + r = read_varint(b, q) + v = rng.getrandbits(64) if r[1] == 9 else rng.getrandbits(7 * r[1]) + b[q:q + r[1]] = enc_varint(v, r[1]) + elif what == "the page type changed": + b[off] = rng.choice([x for x in PAGE_TYPES + (0, 1, 255) if x != t]) + elif what == "a byte of a cell changed": + p = base + rng.choice(pg[6]) + rng.randrange(6) + if p < base + ps: + b[p] = rng.randrange(256) + elif what == "an overflow page number changed": + i = rng.choice([k for k, c in enumerate(pg[7]) if c[4] is not None]) + _, size = read_cell(t, U, f[base:base + ps], pg[6][i]) + p = base + pg[6][i] + size - 4 + b[p:p + 4] = rng.getrandbits(32).to_bytes(4, "big") + elif what == "the reserved byte of the header changed": + b[20] = min(255, rng.choice((0, 1, h[H_RESERVED] + 1, h[H_RESERVED] + 4, ps - 480, ps - 479, + rng.randrange(256)))) + elif what == "the page size field changed": + new = rng.choice(PAGE_SIZES) + b[16:18] = (1 if new == 65536 else new).to_bytes(2, "big") + elif what == "the file cut short": + del b[len(b) - rng.randint(1, 8):] + elif what == "the right-most pointer changed": + b[off + 8:off + 12] = rng.getrandbits(32).to_bytes(4, "big") + elif what == "the freeblock and fragment fields changed": + b[off + 1:off + 3] = rng.choice((0, rng.getrandbits(16))).to_bytes(2, "big") + b[off + 7] = rng.randrange(256) + else: + b[base + rng.randrange(ps)] ^= 1 << rng.randrange(8) + + +# --------------------------------------------------------------------- +# Databases made by SQLite. + +TEXT_POOL = "abcdefghij ABC 0123456789 _-.,;\x00\n\té€üßøñ日本語中文한국😀🎉
" +DECLS = ["", "", "INTEGER", "INT", "REAL", "DOUBLE", "FLOAT", "TEXT", "VARCHAR(20)", "BLOB", "NUMERIC", + "BOOLEAN", "DATE"] +STRICT_DECLS = ["INT", "INTEGER", "REAL", "TEXT", "BLOB", "ANY"] +FLOAT_EDGES = [0.0, -0.0, 1.0, -1.0, 0.5, 3.0, -7.0, 1e15, 2.0 ** 53, 2.0 ** 63, -(2.0 ** 63), 1e300, -1e-300, + 5e-324, float("inf"), float("-inf"), float("nan"), 123456789.0, 0.1] + + +def affinity(decl): + """The affinity of a declared type (datatype3.html 3.1).""" + d = decl.upper() + if "INT" in d: + return "INTEGER" + if "CHAR" in d or "CLOB" in d or "TEXT" in d: + return "TEXT" + if "BLOB" in d or not d: + return "BLOB" + if "REAL" in d or "FLOA" in d or "DOUB" in d: + return "REAL" + return "NUMERIC" + + +def value_bytes(rng, big, share): + """The size of a text or blob: mostly small, and, when `big` is + positive, a share of them up to `big` bytes; at most -big bytes when + it is negative.""" + if big < 0: + return rng.randint(0, -big) + if big and rng.random() < share: + return rng.randint(1000, big) + r = rng.random() + if r < 0.6: + return rng.randint(0, 24) + if r < 0.92: + return rng.randint(25, 400) + return rng.randint(401, 3000) + + +def sql_value(rng, kind, s): + if kind == "null": + return None + if kind == "int": + return random_int(rng) + if kind == "bool": + return rng.randint(0, 1) + if kind == "real": + if rng.random() < 0.4: + return rng.choice(FLOAT_EDGES) + x = struct.unpack(">d", rng.randbytes(8))[0] + return 0.0 if x != x else x + n = value_bytes(rng, s.big or 0, s.big_share or 0.04) + if kind == "text": + if n > 2000: + return (rng.choice(TEXT_POOL) * n)[:n] + return "".join(rng.choice(TEXT_POOL) for _ in range(n)) + return rng.randbytes(n) + + +KINDS = ("null", "int", "bool", "real", "text", "blob") +STRICT_KINDS = {"INT": ("null", "int", "bool"), "INTEGER": ("null", "int", "bool"), + "REAL": ("null", "real", "int"), "TEXT": ("null", "text"), "BLOB": ("null", "blob"), + "ANY": KINDS} + + +class Settings(dict): + """Settings of a database; a setting not given is None.""" + __getattr__ = dict.get + __setattr__ = dict.__setitem__ + + +def table(cols, kind="rowid", pk=(), indexes=(), rows=None, strict=False): + """A table of a fixed database: columns (declared type, value kinds), + its kind (rowid, alias, pk or without), primary key columns, and + indexes (lists of column numbers).""" + return dict(cols=cols, kind=kind, pk=list(pk), indexes=[list(x) for x in indexes], rows=rows, + strict=strict) + + +ANY = ("null", "int", "bool", "real", "text", "blob") + +# The fixed databases: what each is for, and its settings. `rows` is an +# upper bound: rows go in until the file reaches its budget (see +# `budget`). +FIXED_DATABASES = [ + ("4096-byte pages, as SQLite makes them by default", + Settings(page_size=4096, tables=3, rows=400, indexes=2)), + ("65536-byte pages: an empty table, an empty index and an empty WITHOUT ROWID table, whose " + "content start 0 means 65536; long overflow chains", + Settings(page_size=65536, big=150000, big_share=0.25, schema=[ + table([("", ("int", "text")), ("BLOB", ("blob",))], indexes=[[0]], rows=0), + table([("TEXT", ("text",)), ("", ANY)], kind="without", pk=[0], rows=0), + table([("", ("blob", "text")), ("REAL", ("real", "int"))], indexes=[[1]], rows=60)])), + ("512-byte pages: deep table and index trees, negative and huge rowids", + Settings(page_size=512, rows=3000, rowids="wide", big=-30, schema=[ + table([("INTEGER", ("int",)), ("TEXT", ("text", "null"))], indexes=[[0], [1, 0]]), + table([("", ANY), ("", ANY), ("", ANY)], kind="without", pk=[1, 0], indexes=[[2]])])), + ("512-byte pages: payload sizes on both sides of X, M and K, in a table and an index", + Settings(page_size=512, sweep=True)), + ("1024-byte pages, 33 bytes reserved (a usable size of 991): payload sizes around X, M and K", + Settings(page_size=1024, reserve=33, sweep=True)), + ("2048-byte pages, 255 bytes reserved, big values over overflow chains", + Settings(page_size=2048, reserve=255, tables=2, rows=200, big=30000, big_share=0.2, indexes=1)), + ("65536-byte pages, 200 bytes reserved, long overflow chains", + Settings(page_size=65536, reserve=200, tables=1, rows=40, big=300000, big_share=0.3)), + ("UTF-16le text, WITHOUT ROWID tables", + Settings(page_size=1024, encoding="UTF-16le", tables=3, rows=300, without=1.0, indexes=2)), + ("UTF-16be text, auto_vacuum full, deletes", + Settings(page_size=1024, encoding="UTF-16be", auto_vacuum=1, tables=3, rows=300, deletes=0.5, + indexes=1)), + ("incremental vacuum, WAL, deletes that leave free pages", + Settings(page_size=1024, auto_vacuum=2, wal=True, tables=3, rows=400, deletes=0.6, drop=True, + incremental=True, indexes=1)), + ("secure_delete off: free pages keep the b-tree pages they were", + Settings(page_size=512, secure_delete="OFF", tables=3, rows=400, deletes=0.6, updates=0.3, drop=True, + indexes=1)), + ("the legacy file format (schema format 1): 0 and 1 as serial type 1", + Settings(page_size=1024, legacy=True, tables=2, rows=300, indexes=1, schema=[ + table([("", ("bool", "int")), ("", ("bool", "null"))], indexes=[[1]]), + table([("BOOLEAN", ("bool",)), ("TEXT", ("text",))], kind="without", pk=[1])])), + ("no table, only user_version, application_id and default_cache_size", + Settings(page_size=4096, tables=0)), + ("a table of 70 columns: record headers of more than 127 bytes", + Settings(page_size=4096, wide=True, tables=1, rows=60)), + ("8192-byte pages: index keys that spill, on interior and leaf index pages", + Settings(page_size=8192, big=20000, big_share=0.4, schema=[ + table([("TEXT", ("text",)), ("", ("int",))], indexes=[[0]], rows=300), + table([("BLOB", ("blob",)), ("", ANY)], kind="without", pk=[0], rows=200)])), + ("STRICT tables with REAL columns holding whole numbers", + Settings(page_size=2048, strict=1.0, tables=3, rows=300, indexes=1, schema=[ + table([("REAL", ("int", "real")), ("INTEGER", ("int",)), ("ANY", ANY)], indexes=[[0]], + strict=True), + table([("REAL", ("real", "int")), ("TEXT", ("text",))], kind="without", pk=[0], strict=True), + table([("REAL", ("int",)), ("", ("real",))], indexes=[[0, 1]])])), + ("AUTOINCREMENT, ANALYZE, views and triggers", + Settings(page_size=1024, autoincrement=True, analyze=True, views=True, tables=3, rows=200, + indexes=2)), + ("512-byte pages, 40 tables: sqlite_schema spans pages, so page 1 is an interior page", + Settings(page_size=512, tables=40, rows=3, indexes=1)), + ("32768-byte pages, 1 byte reserved, VACUUM after deletes and updates", + Settings(page_size=32768, reserve=1, tables=2, rows=800, deletes=0.4, updates=0.4, vacuum=True, + indexes=1)), + ("16384-byte pages, UTF-16be, auto_vacuum full, 7 bytes reserved, big text", + Settings(page_size=16384, reserve=7, encoding="UTF-16be", auto_vacuum=1, tables=2, rows=300, + big=40000, big_share=0.1, indexes=1, deletes=0.3)), +] + + +def random_settings(rng): + ps = rng.choice(PAGE_SIZES[:5] * 3 + PAGE_SIZES[5:]) + return Settings( + page_size=ps, + reserve=random_reserved(rng, ps) if rng.random() < 0.3 else 0, + encoding=rng.choice(("UTF-8",) * 4 + ("UTF-16le", "UTF-16be")), + auto_vacuum=rng.choice((0, 0, 1, 2)), wal=rng.random() < 0.2, legacy=rng.random() < 0.1, + secure_delete=rng.choice(("ON", "OFF", "FAST")), + tables=rng.randint(1, 4), rows=rng.choice((5, 50, 200, 600)), + big=rng.choice((0, 0, 5000, 50000)), big_share=rng.choice((0.02, 0.1)), + indexes=rng.randint(0, 3), without=rng.choice((0, 0, 0.5)), + strict=rng.choice((0, 0, 0.3)), deletes=rng.choice((0, 0, 0.3, 0.7)), + updates=rng.choice((0, 0, 0.3)), drop=rng.random() < 0.2, vacuum=rng.random() < 0.15, + incremental=rng.random() < 0.3, rowids=rng.choice(("auto", "auto", "wide")), + autoincrement=rng.random() < 0.1, analyze=rng.random() < 0.1, views=rng.random() < 0.1, + empty=rng.random() < 0.15, wide=rng.random() < 0.05, + user_version=rng.choice((None, rng.randint(-2 ** 31, 2 ** 31 - 1))), + application_id=rng.choice((None, rng.randint(-2 ** 31, 2 ** 31 - 1))), + cache_size=rng.choice((None, None, rng.randint(1, 2 ** 31 - 1)))) + + +def budget(ps): + """The most bytes a database of page size ps gets: `sqlite dump` + takes time in proportion to the pages times the file size, so the + budget grows as the square root of the page size (512 KB for 512-byte + pages, 5.8 MB for 65536-byte pages).""" + return int(512 * 1024 * (ps / 512) ** 0.5) + + +class Lib: + """The SQLite library Python's sqlite3 module uses, through ctypes, + for the two settings SQL does not reach: the reserved bytes and the + legacy file format.""" + RESERVE_BYTES = 38 # SQLITE_FILECONTROL_RESERVE_BYTES + LEGACY_FILE_FORMAT = 1016 # SQLITE_DBCONFIG_LEGACY_FILE_FORMAT + + def __init__(self, folder): + if (sys.implementation.name != "cpython" or sysconfig.get_config_var("Py_GIL_DISABLED") + or ctypes.sizeof(ctypes.c_void_p) != 8): + raise OSError("not a 64-bit CPython with the GIL") + self.lib = ctypes.CDLL("libsqlite3.so.0") + self.lib.sqlite3_libversion.restype = ctypes.c_char_p + version = self.lib.sqlite3_libversion().decode() + if version != sqlite3.sqlite_version: + raise OSError(f"libsqlite3.so.0 is {version}, the sqlite3 module {sqlite3.sqlite_version}") + self.lib.sqlite3_db_filename.restype = ctypes.c_char_p + self.lib.sqlite3_db_filename.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + self.lib.sqlite3_file_control.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, + ctypes.c_void_p] + self.lib.sqlite3_db_config.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int, + ctypes.c_void_p] + probe = Path(folder) / "probe.db" + con = sqlite3.connect(probe) + try: + self.handle(con, probe) + finally: + con.close() + + def handle(self, con, path): + # The sqlite3* is the first field after the object header. + h = ctypes.c_void_p.from_address(id(con) + 16).value + name = self.lib.sqlite3_db_filename(h, b"main") if h else None + if not name or os.path.realpath(name.decode()) != os.path.realpath(path): + raise OSError("the connection's sqlite3 handle was not found") + return h + + def configure(self, con, path, reserve, legacy): + h = self.handle(con, path) + if legacy and self.lib.sqlite3_db_config(h, self.LEGACY_FILE_FORMAT, 1, None) != 0: + raise OSError("SQLITE_DBCONFIG_LEGACY_FILE_FORMAT failed") + if reserve: + v = ctypes.c_int(reserve) + if self.lib.sqlite3_file_control(h, b"main", self.RESERVE_BYTES, ctypes.byref(v)) != 0: + raise OSError("SQLITE_FILECONTROL_RESERVE_BYTES failed") + + +def q(name): + return '"' + name.replace('"', '""') + '"' + + +def make_database(path, s, rng, lib): + """Make a database at `path` with settings s; what the oracle needs + to know about it that SQLite does not say (the partial indexes' + conditions).""" + con = sqlite3.connect(path, isolation_level=None) + if lib is not None and (s.reserve or s.legacy): + lib.configure(con, path, s.reserve, s.legacy) + ex = con.execute + ex(f"PRAGMA page_size = {s.page_size}") + ex(f"PRAGMA encoding = '{s.encoding or 'UTF-8'}'") + ex(f"PRAGMA auto_vacuum = {s.auto_vacuum or 0}") + ex(f"PRAGMA secure_delete = {s.secure_delete or 'ON'}") + if s.wal: + ex("PRAGMA journal_mode = WAL") + if s.user_version is not None: + ex(f"PRAGMA user_version = {s.user_version}") + if s.application_id is not None: + ex(f"PRAGMA application_id = {s.application_id}") + if s.cache_size is not None: + ex(f"PRAGMA default_cache_size = {s.cache_size}") + if not s.tables and not s.schema and not s.sweep: + ex("PRAGMA user_version = 7") + ex("PRAGMA application_id = -123456789") + ex("PRAGMA default_cache_size = 1234") + partial = {} + tables = [] + ex("BEGIN") + if s.sweep: + sweep_tables(con, rng, s) + specs = s.schema or [None] * (s.tables or 0) + for ti, spec in enumerate(specs): + tables.append(random_table(con, rng, s, ti, partial, spec)) + room = budget(s.page_size) // max(1, len(tables)) + for ti, t in enumerate(tables): + n = t["rows"] + if n is None: + n = 0 if (s.empty and ti == 0) else max(1, int((s.rows or 50) * rng.uniform(0.5, 1.5))) + insert_rows(con, rng, s, t, n, room * (ti + 1)) + ex("COMMIT") + if s.deletes or s.updates: + ex("BEGIN") + for t in tables: + change_rows(con, rng, s, t) + ex("COMMIT") + if s.drop and len(tables) > 1: + ex(f"DROP TABLE {q(tables[-1]['name'])}") + tables.pop() + if s.views and tables: + ex(f"CREATE VIEW v0 AS SELECT * FROM {q(tables[0]['name'])}") + ex(f"CREATE TRIGGER g0 AFTER INSERT ON {q(tables[0]['name'])} BEGIN SELECT 1; END") + if s.analyze: + ex("ANALYZE") + if s.incremental and s.auto_vacuum == 2: + ex(f"PRAGMA incremental_vacuum({rng.randint(1, 5)})") + if s.vacuum: + ex("VACUUM") + con.close() + return {"partial": partial} + + +def sweep_tables(con, rng, s): + """A table and an index whose payloads take every size near the + places where the part kept on the page changes (1.6): X and X+1, and + each time K wraps round, M+j*(U-4) and X+j*(U-4), for j up to 3.""" + U = s.page_size - (s.reserve or 0) + con.execute("CREATE TABLE sweep_t (b)") + con.execute("CREATE TABLE sweep_i (b)") + con.execute("CREATE INDEX sweep_ix ON sweep_i (b)") + for t, name in ((TABLE_LEAF, "sweep_t"), (INDEX_LEAF, "sweep_i")): + X, M = max_local(t, U), min_local(U) + marks = [X] + [M + j * (U - 4) for j in (1, 2, 3)] + [X + j * (U - 4) for j in (1, 2, 3)] + sizes = sorted({max(0, m + d) for m in marks for d in range(-9, 8)}) + con.executemany(f"INSERT INTO {name} VALUES (?)", [(rng.randbytes(n),) for n in sizes]) + + +def random_table(con, rng, s, ti, partial, spec): + """Create a table, random where `spec` does not say, and its indexes. + What the generator needs to know about it.""" + name = f"t{ti}" + spec = spec or {} + strict = spec.get("strict", rng.random() < (s.strict or 0)) + if "cols" in spec: + decls = [d for d, _ in spec["cols"]] + kinds = [k for _, k in spec["cols"]] + else: + ncols = 70 if s.wide else rng.choice((1, 2, 3, 4, 5, 8)) + decls = [rng.choice(STRICT_DECLS if strict else DECLS) for _ in range(ncols)] + kinds = [tuple(rng.choice(STRICT_KINDS[d] if strict else KINDS) for _ in range(rng.randint(1, 4))) + for d in decls] + cols = [f"c{i}" for i in range(len(decls))] + if "kind" in spec: + kind = spec["kind"] + elif rng.random() < (s.without or 0): + kind = "without" + else: + kind = rng.choice(("rowid", "rowid", "alias", "pk")) + auto = s.autoincrement and ti == 0 + if auto: + kind = "alias" + defs = [f"{c} {d}".strip() for c, d in zip(cols, decls)] + pk = [] + if kind == "alias": + defs[0] = "c0 INTEGER PRIMARY KEY" + (" AUTOINCREMENT" if auto else "") + # AUTOINCREMENT fails with SQLITE_FULL once the largest rowid is used. + kinds[0] = ("null",) if auto else ("int", "null") + elif kind in ("pk", "without"): + pk = [cols[i] for i in spec["pk"]] if spec.get("pk") else \ + rng.sample(cols, min(len(cols), rng.choice((1, 1, 2)))) + for c in pk: + i = cols.index(c) + kinds[i] = tuple(k for k in kinds[i] if k != "null") or ("int",) + if kind == "pk" and len(pk) == 1 and decls[i].upper() == "INTEGER": + # PRIMARY KEY (c) of an INTEGER column would make c the rowid. + decls[i] = "INT" + defs[i] = f"{c} INT" + defs.append("PRIMARY KEY (" + ", ".join(pk) + ")") + if not spec and rng.random() < 0.2 and len(cols) > 1: + defs.append(f"UNIQUE ({rng.choice(cols)})") + options = (["WITHOUT ROWID"] if kind == "without" else []) + (["STRICT"] if strict else []) + con.execute(f"CREATE TABLE {name} (" + ", ".join(defs) + ") " + ", ".join(options)) + if "indexes" in spec: + index_cols = [[cols[i] for i in x] for x in spec["indexes"]] + else: + index_cols = [rng.sample(cols, min(len(cols), rng.randint(1, 3))) for _ in range(s.indexes or 0)] + for xi, on in enumerate(index_cols): + parts = [] + for c in on: + part = c + if rng.random() < 0.2: + part += " COLLATE NOCASE" + if rng.random() < 0.3: + part += " DESC" + parts.append(part) + unique = "UNIQUE " if not spec and rng.random() < 0.15 else "" + iname = f"i{ti}_{xi}" + where = "" + if rng.random() < 0.15: + cond = f"{on[0]} IS NOT NULL" if rng.random() < 0.5 else f"typeof({on[0]}) = 'integer'" + where = " WHERE " + cond + partial[iname] = cond + con.execute(f"CREATE {unique}INDEX {iname} ON {name} (" + ", ".join(parts) + ")" + where) + return dict(name=name, cols=cols, kinds=kinds, kind=kind, pk=pk, rows=spec.get("rows")) + + +# The largest rowid. Once a table holds it, SQLite picks the rowid of a +# row inserted without one at random (autoinc.html). +TOP_ROWID = 2 ** 63 - 1 + + +def insert_rows(con, rng, s, t, n, limit): + """Insert up to n rows, in batches, until the file holds `limit` + bytes. Once a row of the table may hold the largest rowid, every + later row is given its rowid here, so that SQLite does not pick one + at random and the same seed makes the same file.""" + name, cols, kinds, kind = t["name"], t["cols"], t["kinds"], t["kind"] + explicit = kind != "without" and (s.rowids == "wide" or rng.random() < 0.3) and not ( + s.autoincrement and name == "t0") + target = ["rowid"] + cols if explicit and kind != "alias" else cols + sql = f"INSERT OR IGNORE INTO {name} (" + ", ".join(target) + ") VALUES (" + ", ".join( + "?" * len(target)) + ")" + # Whether the first value of a row is its rowid (or None, for SQLite + # to choose); without it, SQLite chooses every rowid, from 1 up. + rowid_first = kind == "alias" or explicit + top = False + done = 0 + while done < n: + rows = [] + for _ in range(min(25, n - done)): + vals = [sql_value(rng, rng.choice(k), s) for k in kinds] + if kind == "alias" and explicit: + vals[0] = random_rowid(rng, s) + if explicit and kind != "alias": + vals = [random_rowid(rng, s)] + vals + if rowid_first: + if vals[0] is None and top: + vals[0] = any_rowid(rng) + top = top or vals[0] == TOP_ROWID + rows.append(vals) + con.executemany(sql, rows) + done += len(rows) + if con.execute("PRAGMA page_count").fetchone()[0] * s.page_size >= limit: + break + + +def any_rowid(rng): + return rng.choice((rng.randint(-2 ** 63, 2 ** 63 - 1), rng.randint(-1000, 1000), TOP_ROWID, + -2 ** 63, -1, 0, rng.randint(-2 ** 31, 2 ** 31))) + + +def random_rowid(rng, s): + if s.rowids == "wide" or rng.random() < 0.3: + return any_rowid(rng) + return None + + +def change_rows(con, rng, s, t): + """Delete and update rows. The rows are picked here, not by SQLite's + random(), so that the same seed makes the same file.""" + name, cols, kinds, kind = t["name"], t["cols"], t["kinds"], t["kind"] + if kind == "without": + if s.deletes: + keys = con.execute(f"SELECT {', '.join(t['pk'])} FROM {name} ORDER BY " + f"{', '.join(t['pk'])}").fetchall() + gone = [k for k in keys if rng.random() < s.deletes] + con.executemany(f"DELETE FROM {name} WHERE " + " AND ".join(f"{c} = ?" for c in t["pk"]), gone) + return + ids = [r[0] for r in con.execute(f"SELECT rowid FROM {name} ORDER BY rowid")] + gone = [i for i in ids if rng.random() < (s.deletes or 0)] + con.executemany(f"DELETE FROM {name} WHERE rowid = ?", [(i,) for i in gone]) + gone = set(gone) + for i in ids: + if i not in gone and rng.random() < (s.updates or 0): + c = rng.randrange(len(cols)) + if kind == "alias" and c == 0: + continue + con.execute(f"UPDATE OR IGNORE {name} SET {cols[c]} = ? WHERE rowid = ?", + (sql_value(rng, rng.choice(kinds[c]), s), i)) + + +def all_databases(seed, count, lib): + """The settings of every database to make: (label, settings, rng + seed).""" + rng = random.Random(seed * 7 + 3) + out = [] + for label, s in FIXED_DATABASES: + s = Settings(s) + if lib is None and (s.reserve or s.legacy): + s.reserve, s.legacy = 0, False + out.append((label, s, rng.getrandbits(64))) + for _ in range(count // 1000): + s = random_settings(rng) + if lib is None: + s.reserve, s.legacy = 0, False + out.append((f"random settings, {s.page_size}-byte pages", s, rng.getrandbits(64))) + return out + + +# --------------------------------------------------------------------- +# What SQLite says about a database. + +ENCODINGS = {"UTF-8": "utf-8", "UTF-16le": "utf-16-le", "UTF-16be": "utf-16-be"} + + +def render(v, enc): + """A value from SQLite as `sqlite serve` writes a record value.""" + if v is None: + return "null" + if isinstance(v, int): + return f"int:{v}" + if isinstance(v, float): + return f"real:{struct.unpack('>Q', struct.pack('>d', v))[0]}" + if isinstance(v, str): + return "text:" + hx(v.encode(enc, "surrogatepass")) + return "blob:" + hx(v) + + +def ask_sqlite(path, meta): + """What SQLite says about the database at `path`.""" + con = sqlite3.connect(path) + con.text_factory = str + try: + def pragma(name): + return con.execute(f"PRAGMA {name}").fetchone()[0] + + facts = {k: pragma(k) for k in ("page_size", "page_count", "freelist_count", "schema_version", + "user_version", "application_id", "encoding", "auto_vacuum", + "journal_mode", "default_cache_size")} + facts["sqlite_version"] = con.execute("SELECT sqlite_version()").fetchone()[0] + enc = ENCODINGS[facts["encoding"]] + facts["dbstat"] = {r[2]: (r[0], r[1], r[3], r[4], r[5], r[6], r[7]) for r in con.execute( + "SELECT name, path, pageno, pagetype, ncell, payload, unused, mx_payload FROM dbstat")} + schema = con.execute("SELECT type, name, tbl_name, rootpage, sql FROM sqlite_schema").fetchall() + facts["roots"] = {1: "sqlite_schema"} + wr = {r[1]: r[4] for r in con.execute("PRAGMA table_list") if r[0] == "main"} + trees = [table_tree(con, "sqlite_schema", 1, enc)] + for typ, name, tbl, root, sql in schema: + if not root: + continue + facts["roots"][root] = name + if typ == "table" and not wr.get(name): + trees.append(table_tree(con, name, root, enc)) + elif typ == "table": + pk = [r[1] for r in con.execute(f"PRAGMA index_list({q(name)})") if r[3] == "pk"][0] + trees.append(index_tree(con, name, pk, name, root, None, enc)) + else: + trees.append(index_tree(con, name, name, tbl, root, meta["partial"].get(name), enc)) + facts["trees"] = trees + facts["max_root"] = max([1] + [r[3] or 0 for r in schema]) + return facts + finally: + con.close() + + +def table_tree(con, name, root, enc): + info = con.execute(f"PRAGMA table_info({q(name)})").fetchall() + pks = [r for r in info if r[5]] + alias = None + if len(pks) == 1 and pks[0][2].upper() == "INTEGER": + alias = pks[0][0] + cols = [(r[1], affinity(r[2]), r[0] == alias) for r in info] + rows = [(r[0], tuple(render(v, enc) for v in r[1:])) + for r in con.execute(f"SELECT rowid, * FROM {q(name)} ORDER BY rowid")] + return {"name": name, "root": root, "kind": "table", "cols": cols, "rows": rows} + + +def index_tree(con, name, index, table, root, cond, enc): + info = {r[0]: r for r in con.execute(f"PRAGMA table_info({q(table)})")} + xinfo = con.execute(f"PRAGMA index_xinfo({q(index)})").fetchall() + cols, exprs = [], [] + for r in xinfo: + if r[1] == -1: + cols.append(("rowid", "INTEGER", False)) + exprs.append("rowid") + else: + cols.append((info[r[1]][1], affinity(info[r[1]][2]), False)) + exprs.append(q(info[r[1]][1])) + sql = f"SELECT {', '.join(exprs)} FROM {q(table)}" + (f" WHERE {cond}" if cond else "") + rows = [tuple(render(v, enc) for v in r) for r in con.execute(sql)] + return {"name": name, "root": root, "kind": "index", "cols": cols, "rows": rows} + + +# --------------------------------------------------------------------- +# Reading the tool's output back. + + +def parse_cell(text): + p, left, size, rowid, kept, overflow = text.split("/") + + def num(x): + return None if x == "-" else int(x) + + return num(p), (num(left), num(size), num(rowid), hex_field(kept), num(overflow)) + + +def parse_page(text): + """A page as `showPage` writes it, back in the model's form.""" + head, _, cells = text.partition(" |") + t, ffb, count, content, frag, right = head.split(" ") + parsed = [parse_cell(c) for c in cells.split(" ") if c] + return (int(t), int(ffb), int(count), int(content), int(frag), None if right == "-" else int(right), + [p for p, _ in parsed], [c for _, c in parsed]) + + +def parse_dump(lines): + """The tool's dump: the header fields, the pages (None when FAIL), + and the records of cells (None when FAIL).""" + header, pages, records = None, {}, {} + for line in lines: + parts = line.split(" ") + if parts[0] == "header": + header = tuple(int(x) for x in parts[1:]) + elif parts[0] == "page": + n = int(parts[1]) + pages[n] = None if parts[2:] == ["FAIL"] else parse_page(line.split(" ", 2)[2]) + elif parts[0] == "record": + records[(int(parts[1]), int(parts[2]))] = None if parts[4:] == ["FAIL"] else parts[4:] + return header, pages, records + + +# --------------------------------------------------------------------- +# The comparison with SQLite. +# +# Each answer SQLite gives is put beside two readings of the file: the +# tool's dump, and the model's dump read with SQLite's own rules. The +# model, so read, must give SQLite's answer, whether or not the tool +# agrees. Where the tool and SQLite differ, the difference is allowed +# only when the model with SQLite's rules gives SQLite's answer and needs +# at least one of those rules to do so; it is counted under them. + +NAMED_ALIAS = "an INTEGER PRIMARY KEY column is NULL in the record; SQLite gives the rowid (2.3)" +NAMED_REAL = ("a whole-number float in a column of REAL affinity is kept as an integer; SQLite gives it " + "back as a float (2.3)") +NAMED_SIGNED = "PRAGMA {} reads the 4-byte field as a signed integer" +NAMED_CACHE = "PRAGMA default_cache_size gives SQLite's built-in default, -2000, for a field of 0" +NAMED_CACHE_SIGNED = ("PRAGMA default_cache_size reads the 4-byte field as a signed integer and gives its " + "absolute value") +NAMED_UNUSED = ("dbstat counts unused bytes from the content start as stored, so 0 counts as 0, not " + "65536 (1.6)") +NAMED_ENCODING_0 = ("PRAGMA encoding gives UTF-8, the default for a database not yet created, for a text " + "encoding of 0, which names none: SQLite writes 0, with a schema format of 0, while a " + "database has no schema") +ROLES = {"overflow": "overflow pages", "trunk": "freelist trunk pages", "leaf": "freelist leaf pages", + "ptrmap": "pointer-map pages"} + + +class Check: + """Counts of what was compared with SQLite: the tool's answers that + agree, the named differences, the model's answers (read with + SQLite's rules) that agree, and the failures.""" + + def __init__(self): + self.asked = collections.Counter() + self.agree = collections.Counter() + self.named = {} + self.model_asked = collections.Counter() + self.model_agree = collections.Counter() + # Overflow, freelist and pointer-map pages, and how many of them + # the tool reads as b-tree pages from their bytes: a count only. + self.other = collections.Counter() + self.other_read = collections.Counter() + self.failures = [] + + def ok(self, what): + self.asked[what] += 1 + self.agree[what] += 1 + + def name(self, what, reason, example): + self.asked[what] += 1 + n, ex = self.named.get(reason, (0, None)) + self.named[reason] = (n + 1, ex or example) + + def fail(self, what, where, want, got): + self.asked[what] += 1 + self.failures.append((where, short(str(want)), short(str(got)))) + + def model(self, what, where, theirs, mine): + """SQLite's answer against the model's, read with SQLite's + rules.""" + self.model_asked[what] += 1 + if mine == theirs: + self.model_agree[what] += 1 + else: + self.failures.append((f"{where}, as the model reads it with SQLite's rules", short(str(theirs)), + short(str(mine)))) + + def compare(self, what, where, theirs, tool, mine, rules, example): + """SQLite's answer `theirs` against the model's with SQLite's + rules, `mine`, which needed the rules `rules`; and against the + tool's.""" + self.model(what, where, theirs, mine) + if tool == theirs: + self.ok(what) + elif mine != theirs: + self.asked[what] += 1 # a failure already, the model's + elif rules: + self.name(what, "; ".join(sorted(rules)), example) + else: + self.fail(what, where, theirs, tool) + + +def float_bits_of_int(z): + return f"real:{struct.unpack('>Q', struct.pack('>d', float(z)))[0]}" + + +def sqlite_reads(value, col, rowid): + """What SQLite gives for a value of a record, read with its own rules + for the column `col` (name, affinity, rowid alias); and the rule + used, or None.""" + name, aff, alias = col + if alias and value == "null": + return f"int:{rowid}", NAMED_ALIAS + if aff == "REAL" and value.startswith("int:"): + return float_bits_of_int(int(value[4:])), NAMED_REAL + return value, None + + +def sqlite_reads_row(values, cols, rowid): + """A row or index entry, read with SQLite's rules: the values, and, + for each rule used, the first column it was used for.""" + if values is None or len(values) != len(cols): + return values, {} + out, rules = [], {} + for k, (v, col) in enumerate(zip(values, cols)): + x, rule = sqlite_reads(v, col, rowid) + out.append(x) + if rule: + rules.setdefault(rule, k) + return tuple(out), rules + + +def signed32(v): + return v - 2 ** 32 if v >= 2 ** 31 else v + + +TEXT_ENCODINGS = {1: "UTF-8", 2: "UTF-16le", 3: "UTF-16be"} + + +def header_says(h, sqlite_rules=False): + """What the header h gives for each PRAGMA the tests ask SQLite: the + value and the rule used, or None. Without `sqlite_rules`, each field + as it is read (1.3); with them, as SQLite reads it: user_version and + application_id as signed integers; default_cache_size as the + absolute value of a signed integer, or SQLite's built-in default, + -2000, for 0; a text encoding of 0, which names no encoding, as + UTF-8, the encoding of a database not yet created by default.""" + out = { + "page_size": h[H_PAGE_SIZE], + "page_count": h[H_SIZE], + "freelist_count": h[H_FREE], + "schema_version": h[H_COOKIE], + "encoding": TEXT_ENCODINGS.get(h[H_ENCODING], f"none (text encoding {h[H_ENCODING]})"), + # 1.3.3: versions 1 for a rollback journal, 2 for WAL. + "journal_mode": (h[H_WRITE], h[H_READ]), + # 1.3.8: the largest root page in auto-vacuum mode, else 0; and + # whether that mode is incremental. + "auto_vacuum: largest root page": h[H_LARGEST], + "auto_vacuum: incremental vacuum": h[H_INCREMENTAL], + "sqlite_version()": h[H_VERSION], + "user_version": h[H_USER], + "application_id": h[H_APPID], + "default_cache_size": h[H_CACHE], + } + out = {k: (v, None) for k, v in out.items()} + if sqlite_rules: + for name, i in (("user_version", H_USER), ("application_id", H_APPID)): + if h[i] >= 2 ** 31: + out[name] = (signed32(h[i]), NAMED_SIGNED.format(name)) + cache = signed32(h[H_CACHE]) + if cache == 0: + out["default_cache_size"] = (-2000, NAMED_CACHE) + elif cache < 0: + out["default_cache_size"] = (-cache, NAMED_CACHE_SIGNED) + if h[H_ENCODING] == 0: + out["encoding"] = ("UTF-8", NAMED_ENCODING_0) + return out + + +def sqlite_says(F): + """SQLite's answers to the same PRAGMAs, in the same terms.""" + return { + "page_size": F["page_size"], + "page_count": F["page_count"], + "freelist_count": F["freelist_count"], + "schema_version": F["schema_version"], + "encoding": F["encoding"], + "journal_mode": (2, 2) if F["journal_mode"] == "wal" else (1, 1), + "auto_vacuum: largest root page": F["max_root"] if F["auto_vacuum"] else 0, + "auto_vacuum: incremental vacuum": 1 if F["auto_vacuum"] == 2 else 0, + "sqlite_version()": sum(int(x) * m for x, m in zip(F["sqlite_version"].split("."), (1000000, 1000, 1))), + "user_version": F["user_version"], + "application_id": F["application_id"], + "default_cache_size": F["default_cache_size"], + } + + +class Reading: + """A reading of a database file, the tool's or the model's: its + header, its pages and the records of its cells, as a dump gives + them, and what the walk of its b-trees finds in them.""" + + def __init__(self, who, h, pages, records): + self.who, self.h, self.pages, self.records = who, h, pages, records + self.rows = {} # page -> the dbstat row it gives, the content start as read + self.rows_stored = {} # the same, the content start as stored (SQLite's rule) + self.parent = {} # page -> (pointer-map type, parent page) + self.entries = [] # per tree: [(rowid, where its record is)] + self.keys = [] # per table: (tree, its interior keys, the rowids of its leaves) + self.spills = [] # payloads that spill, joined from their overflow pages + self.roles = {} # page -> "trunk", "leaf" or "ptrmap" + self.roles_problem = None + + +class Database: + """One database made by SQLite: its file, what SQLite says about it, + and the comparison of the tool's reading and the model's with it.""" + + def __init__(self, index, label, settings, path, data, facts): + self.index, self.label, self.settings, self.path = index, label, settings, path + self.data, self.facts = data, facts + self.tool = self.model = None + self.read = False + + def where(self, more=""): + return f"database {self.index} ({self.label})" + (f", {more}" if more else "") + + def read_back(self, got, model_lines, check): + """Read the tool's dump `got` and the model's, `model_lines`, walk + the b-trees of each, and compare the header and the pages with + SQLite. The rows wait for the payloads that spill + (compare_entries).""" + h, pages, records = parse_dump(model_lines) + if h is not None: + self.model = self.walk(Reading("the model", h, pages, records), check) + h, pages, records = parse_dump(got) + if h is not None: + self.tool = self.walk(Reading("the tool", h, pages, records), check) + self.compare_keys(self.tool, check) + self.compare_header(check) + self.compare_pages(check) + self.read = True + + def compare_header(self, check): + F, tool, model = self.facts, self.tool, self.model + what = "header fields" + theirs = sqlite_says(F) + mine = header_says(model.h, sqlite_rules=True) if model else {k: (None, None) for k in theirs} + if tool is None: + # The tool reads no header of a file SQLite made: a failure, + # and the model's reading is still checked. + for name, value in theirs.items(): + check.model(what, self.where(f"PRAGMA {name}"), value, mine[name][0]) + check.fail(what, self.where(), "a header the tool reads", "none") + return + got = header_says(tool.h) + for name, value in theirs.items(): + m, rule = mine[name] + check.compare(what, self.where(f"PRAGMA {name}"), value, got[name][0], m, {rule} - {None}, + f"{self.where()}: SQLite {value}, the tool {got[name][0]}") + + def roles_of(self, h): + """The pointer-map pages (1.8) when h has a largest root page, and + the freelist pages, reached from the first trunk page h gives + (1.5): page -> "ptrmap", "trunk" or "leaf"; and why the freelist + could not be followed, or None.""" + f, npages = self.data, self.facts["page_count"] + ps, U = h[H_PAGE_SIZE], h[H_PAGE_SIZE] - h[H_RESERVED] + roles, problem = {}, None + if h[H_LARGEST]: + p = 2 + while p <= npages: + roles[p] = "ptrmap" + p += U // 5 + 1 + trunk, seen = h[H_TRUNK], set() + while trunk: + if trunk in seen or not 1 <= trunk <= npages: + problem = f"freelist trunk page {trunk} past the end or reached again" + break + seen.add(trunk) + roles[trunk] = "trunk" + page = f[(trunk - 1) * ps:trunk * ps] + for k in range(min(be(page[4:8]), (U - 8) // 4)): + roles[be(page[8 + 4 * k:12 + 4 * k])] = "leaf" + trunk = be(page[:4]) + return roles, problem + + def walk(self, r, check): + """Walk every b-tree from the root sqlite_schema gives, by the + reading r: its child pointers, and each overflow chain from the + first overflow page it reads. Fills in r (see Reading).""" + F, f, h, pages = self.facts, self.data, r.h, r.pages + ps = h[H_PAGE_SIZE] + npages = len(f) // ps + + def bad(where, want, got): + check.failures.append((self.where(f"{r.who}'s reading, {where}"), want, got)) + + for tree in F["trees"]: + entries = [] + r.entries.append(entries) + table = tree["kind"] == "table" + types = (TABLE_INTERIOR, TABLE_LEAF) if table else (INDEX_INTERIOR, INDEX_LEAF) + stack = [("page", tree["root"], "/", None)] + order, keys = [], [] + while stack: + item = stack.pop() + if item[0] == "key": + keys.append((len(order), item[1], item[2])) + continue + _, n, path, parent = item + if n in r.rows or not 1 <= n <= npages: + bad(f"{tree['name']} page {n}, path {path}", "a page of the file reached once", + "a page reached again or past the end") + continue + pg = pages.get(n) + if pg is None: + r.rows[n] = r.rows_stored[n] = (tree["name"], path, "FAIL") + continue + t, ffb, count, content, frag, right, pointers, cells = pg + if t not in types: + bad(f"page {n} of {tree['name']}", f"page type {' or '.join(map(str, types))}", + f"page type {t}") + r.parent[n] = (1, 0) if parent is None else (5, parent) + kept = sum(len(c[3]) for c in cells) + biggest = max([c[1] or 0 for c in cells] + [0]) + hdr = (100 if n == 1 else 0) + (12 if t in INTERIOR else 8) + free, at, seen = 0, ffb, set() + pb = f[(n - 1) * ps:n * ps] + while at and at not in seen and at + 4 <= ps: + seen.add(at) + free += be(pb[at + 2:at + 4]) + at = be(pb[at:at + 2]) + row = (tree["name"], path, "internal" if t in INTERIOR else "leaf", count, kept) + r.rows[n] = row + (content - hdr - 2 * count + frag + free, biggest) + r.rows_stored[n] = row + (content % 65536 - hdr - 2 * count + frag + free, biggest) + children = [] + for i, (p, c) in enumerate(zip(pointers, cells)): + left, size, rowid, local, overflow = c + if overflow is not None: + payload = self.follow(r, tree, n, path, i, c, npages, bad) + if payload is not None: + r.spills.append(payload) + ref = ("spill", len(r.spills) - 1) + else: + ref = ("broken", None) + else: + ref = ("dump", (n, p)) + if t in INTERIOR: + children.append(("page", left, f"{path}{i:03x}/", n)) + if table: + children.append(("key", rowid, f"page {n}, cell {i}")) + if table and t == TABLE_LEAF: + order.append((rowid, ref)) + elif not table: + order.append((None, ref)) + if t in INTERIOR: + children.append(("page", right, f"{path}{count:03x}/", n)) + stack.extend(reversed(children)) + entries.extend(order) + if table: + r.keys.append((tree, keys, [rowid for rowid, _ in order])) + r.roles, r.roles_problem = self.roles_of(h) + for p, role in r.roles.items(): + if role in ("trunk", "leaf"): + r.parent.setdefault(p, (2, 0)) + return r + + def follow(self, r, tree, n, path, i, cell, npages, bad): + """The whole payload of a cell that spills: the part the reading r + keeps on the page and the overflow pages, from the first one it + reads (1.7). Adds the overflow pages' dbstat rows.""" + ps, U = r.h[H_PAGE_SIZE], r.h[H_PAGE_SIZE] - r.h[H_RESERVED] + left, size, rowid, local, first = cell + rest = size - len(local) + count = -(-rest // (U - 4)) + out, at, prev = bytearray(local), first, n + for j in range(count): + if not 1 <= at <= npages or at in r.rows: + bad(f"{tree['name']} page {n} cell {i}, overflow page {j}", "a page of the file not used before", + f"page {at}") + return None + page = self.data[(at - 1) * ps:at * ps] + part = min(U - 4, rest - len(out) + len(local)) + out += page[4:4 + part] + last = j == count - 1 + r.rows[at] = r.rows_stored[at] = (tree["name"], f"{path}{i:03x}+{j:06x}", "overflow", 0, part, + (U - 4 - part) if last else 0, 0) + r.parent[at] = (3, n) if j == 0 else (4, prev) + prev, at = at, be(page[:4]) + if at != 0: + bad(f"{tree['name']} page {n} cell {i}", "the overflow chain ends where the payload size says", + f"page {prev} links on to page {at}") + return bytes(out) + + def compare_keys(self, r, check): + """The keys of table interior cells, as the reading r gives them, + among the rowids of the leaves: "pointers to the left of a X refer + to b-tree pages on which all keys are less than or equal to X. + Pointers to the right of X refer to pages where all keys are + greater than X" (1.6).""" + what = "table interior keys" + for tree, keys, rowids in r.keys: + last = None + for pos, key, where in keys: + before = rowids[pos - 1] if pos > 0 else None + after = rowids[pos] if pos < len(rowids) else None + if ((before is None or before <= key) and (after is None or key < after) + and (last is None or last < key)): + check.ok(what) + else: + check.fail(what, self.where(f"table {tree['name']}, {where}"), + f"a key from {before} to below {after}, above {last}", f"key {key}") + last = key + + def compare_pages(self, check): + """dbstat, page by page; every page accounted for; the pointer-map + entries; and the count of overflow, freelist and pointer-map pages + the tool reads as b-tree pages.""" + F, f, tool, model = self.facts, self.data, self.tool, self.model + stat = F["dbstat"] + npages = F["page_count"] + what = "dbstat rows" + listed = set(stat).union(*(set(r.rows) for r in (tool, model) if r is not None)) + for n in sorted(listed): + where = self.where(f"page {n}") + theirs = stat.get(n) + mine = model.rows_stored.get(n) if model else None + if tool is None: + check.model(what, where, theirs, mine) + continue + rules = {NAMED_UNUSED} if mine != model.rows.get(n) else set() + check.compare(what, where, theirs, tool.rows.get(n), mine, rules, + f"{where}: dbstat {theirs}, the tool's reading {tool.rows.get(n)}") + if tool is not None: + roles = tool.roles + free = [p for p, role in roles.items() if role in ("trunk", "leaf")] + overlap = sorted(set(roles) & set(stat)) + missing = sorted(set(range(1, npages + 1)) - set(stat) - set(roles)) + if (tool.roles_problem or overlap or missing or len(free) != tool.h[H_FREE] + or max(list(roles) + [0]) > npages): + check.fail("pages accounted for", self.where(), "every page once: in dbstat, on the freelist " + "or a pointer-map page", f"in two of them {overlap[:5]}, in none {missing[:5]}, " + f"{len(free)} freelist pages for {tool.h[H_FREE]}; {tool.roles_problem or ''}") + else: + check.ok("pages accounted for") + # Not a check: SQLite gives no b-tree reading of these pages. + for n in range(1, npages + 1): + role = "overflow" if stat.get(n, ("", "", ""))[2] == "overflow" else roles.get(n) + if role is not None: + check.other[role] += 1 + check.other_read[role] += tool.pages.get(n) is not None + # The pointer-map entries SQLite wrote (1.8), against the parent + # each page has in the walk. + what = "pointer-map entries" + ref = model or tool + if F["auto_vacuum"] and ref is not None: + ps, U = ref.h[H_PAGE_SIZE], ref.h[H_PAGE_SIZE] - ref.h[H_RESERVED] + J = U // 5 + for n in range(3, npages + 1): + if (n - 2) % (J + 1) == 0: + continue # a pointer-map page + base = 2 + ((n - 3) // (J + 1)) * (J + 1) + at = (base - 1) * ps + 5 * (n - base - 1) + theirs = (f[at], be(f[at + 1:at + 5])) + mine = model.parent.get(n) if model else None + where = self.where(f"page {n}") + if tool is None: + check.model(what, where, theirs, mine) + else: + check.compare(what, where, theirs, tool.parent.get(n), mine, set(), None) + + def values(self, r, spilled): + """For each tree, the entries the reading r gives: (rowid, the + values of the record, or None when it is not read). `spilled(k)` + gives the values of the kth payload that spills.""" + out = [] + for entries in r.entries: + vs = [] + for rowid, (src, key) in entries: + v = r.records.get(key) if src == "dump" else spilled(key) if src == "spill" else None + vs.append((rowid, None if v is None else tuple(v))) + out.append(vs) + return out + + def compare_entries(self, answers, check): + """The rows of each table and the entries of each index against + SQLite's. `answers` are the tool's answers to `record` for the + payloads that spill in its reading; the model reads its own with + read_record.""" + tool = model = None + if self.tool is not None: + def from_tool(k): + a = answers.get(k, "") + return a[3:].split(" ") if a.startswith("OK ") else None + tool = self.values(self.tool, from_tool) + if self.model is not None: + model = self.values(self.model, lambda k: read_record(self.model.spills[k])[0]) + for k, tree in enumerate(self.facts["trees"]): + t = tool[k] if tool is not None else None + m = model[k] if model is not None else [] + if tree["kind"] == "table": + self.compare_table(tree, t, m, check) + else: + self.compare_index(tree, t, m, check) + + def compare_table(self, tree, tool, model, check): + """Row by row, in rowid order: SQLite's rowid and values, the + model's read with SQLite's rules, and the tool's.""" + what = "table rows" + rows, cols = tree["rows"], tree["cols"] + for k in range(max(len(rows), len(tool or []), len(model))): + theirs = (rows[k][0], rows[k][1]) if k < len(rows) else None + got = tool[k] if tool is not None and k < len(tool) else None + mine, rules = None, {} + if k < len(model): + values, rules = sqlite_reads_row(model[k][1], cols, model[k][0]) + mine = (model[k][0], values) + rowid = (theirs or got or mine or (None,))[0] + where = self.where(f"table {tree['name']}, rowid {rowid}") + if tool is None: + check.model(what, where, theirs, mine) + continue + example = "; ".join( + f"{where}, column {cols[j][0]}: SQLite {theirs[1][j] if theirs else None}, the tool " + f"{got[1][j] if got and got[1] and j < len(got[1]) else None}" + for _, j in sorted(rules.items())) + check.compare(what, where, theirs, got, mine, set(rules), example) + + def compare_index(self, tree, tool, model, check): + """The entries as multisets: SQLite's, the model's read with + SQLite's rules, and the tool's. An entry of the tool's that is not + SQLite's is a named difference when the model's entry in its place, + read with SQLite's rules, is one of SQLite's and needed them.""" + what = "index entries" + where = self.where(f"index {tree['name']}") + cols = [(name, aff, False) for name, aff, _ in tree["cols"]] + mine = [sqlite_reads_row(values, cols, None) for _, values in model] + left = collections.Counter(tree["rows"]) + for values, _ in mine: + if values is not None and left[values] > 0: + left[values] -= 1 + check.model(what, where, values, values) + else: + check.model(what, where, "no such entry in SQLite", values) + for entry, n in left.items(): + for _ in range(n): + check.model(what, where, entry, "no such entry read by the model") + if tool is None: + return + left = collections.Counter(tree["rows"]) + later = [] + for k, (_, values) in enumerate(tool): + if values is not None and left[values] > 0: + left[values] -= 1 + check.ok(what) + else: + later.append(k) + for k in later: + values = tool[k][1] + read, rules = mine[k] if k < len(mine) else (None, {}) + if values is not None and read is not None and rules and left[read] > 0: + left[read] -= 1 + check.name(what, "; ".join(sorted(rules)), + f"{where}: SQLite {' '.join(read)[:100]}, the tool {' '.join(values)[:100]}") + else: + check.fail(what, where, "no such entry in SQLite", " ".join(values or ["(not read)"])[:160]) + for entry, n in left.items(): + for _ in range(n): + check.fail(what, where, " ".join(entry)[:160], "no such entry read by the tool") + + +# --------------------------------------------------------------------- +# Running a program over request lines. + + +def run_lines(cmd, lines, timeout=TIMEOUT, cwd=None, env=None): + """Send `lines` to `cmd` and return its answer lines and a note on + how the run failed, or None.""" + raw = "".join(lines).encode("utf-8") + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, cwd=cwd, env=env) + out, err = bytearray(), bytearray() + + def feed(): + try: + proc.stdin.write(raw) + proc.stdin.close() + except (BrokenPipeError, ValueError): + pass + + def drain(stream, buf): + while chunk := stream.read1(1 << 16): + buf += chunk + + threads = [threading.Thread(target=feed), + threading.Thread(target=drain, args=(proc.stdout, out)), + threading.Thread(target=drain, args=(proc.stderr, err))] + for t in threads: + t.start() + problem = None + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + problem = f"no end after {timeout} seconds" + for t in threads: + t.join() + proc.wait() + if proc.returncode != 0 and not problem: + tail = " / ".join(err.decode("utf-8", "backslashreplace").splitlines()[-3:]) + problem = f"exit status {proc.returncode}: {tail}" + text = bytes(out).decode("utf-8", "backslashreplace") + if text and not text.endswith("\n"): + problem = problem or "the last answer has no LF" + answers = text.split("\n") + if answers and answers[-1] == "": + answers.pop() + return answers, problem + + +def run_parallel(cmd, lines, jobs): + """`run_lines` over `jobs` processes, each given a run of the lines + in order; the answers in order, and the problems.""" + size = -(-len(lines) // jobs) if lines else 0 + chunks = [lines[k:k + size] for k in range(0, len(lines), size)] if size else [[]] + results = [None] * len(chunks) + + def one(k): + results[k] = run_lines(cmd, chunks[k]) + + threads = [threading.Thread(target=one, args=(k,)) for k in range(len(chunks))] + for t in threads: + t.start() + for t in threads: + t.join() + answers, problems = [], [] + for chunk, (got, problem) in zip(chunks, results): + if problem: + problems.append(problem) + if len(got) != len(chunk): + problems.append(f"{len(got)} answers for {len(chunk)} requests") + got = (got + [""] * len(chunk))[:len(chunk)] + answers += got + return answers, problems + + +def run_dump(path): + """`sqlite dump path`: its lines, exit status, and a problem or None.""" + try: + p = subprocess.run([str(EXE), "dump", str(path)], capture_output=True, timeout=TIMEOUT) + except (OSError, subprocess.SubprocessError) as e: + return [], None, str(e) + text = p.stdout.decode("utf-8", "backslashreplace") + lines = text.split("\n") + problem = None + if text and not text.endswith("\n"): + problem = "the last line has no LF" + if lines and lines[-1] == "": + lines.pop() + return lines, p.returncode, problem + + +# --------------------------------------------------------------------- +# Malformed and odd requests, and the command line. + +MALFORMED = [ + "", " ", "\t", "varint", "varint 00 00", "varint 0", "varint 0A", "varint 0x00", "varint zz", + "varint ", "varint 00", " varint 00", "VARINT 00", "Varint 00", "varint\t00", "varint --", + "varint +00", "varint é0", "header", "header 53 51", "header -0", "page 1", "page 1 00 00", + "page -1 00", "page +1 00", "page 1.0 00", "page x 00", "page 00", "page 1e3 00", "page ٣ 00", + "page 1_0 00", "page ０ 00", "page 1 0G", "record", "record 0", "record 00 00", "record 02 0105", + "local 13 4096", "local 13 4096 100 1", "local 3 4096 100", "local 0 4096 100", "local 14 4096 100", + "local 256 4096 100", "local 269 4096 100", "local 013x 4096 100", "local 13 -1 100", + "local 13 4096 -", "local - 4096 100", "local 13 4096 1e3", "local 13 4096 ９", "local 13 4096 1", + "local 13 4096 1 ", "dump x", "serve", "pages 1 00", "records 00", "varint00", "local13 1 1", + "varint FF", "header 53514C69", "record 0A01", "page 1 0D00", "varint Ff", "varint fF", +] +ODD = ["varint -", "header -", "record -", "page 0 -", "page 1 -", "local 013 04096 010000", + "local 2 0 0", "local 5 0 0", "local 10 4 5", "local 13 99999999999999999999999 5", + "page 99999999999999999999999 " + SPEC_HEADER.hex(), "page 0001 " + ONE_PAGE_FILE.hex(), + "varint " + "ff" * 20, "record " + "ff" * 9] +# Last lines with no LF, with CR LF, with a lone CR: (request, answer). +AT_EOF = [("varint 00", "OK 0 1 -"), ("record 020105\r\n", "OK int:5"), ("local 13 4096 10000", "OK 1816"), + ("varint 00\r", "error")] + + +def cli_cases(folder, sample): + """Runs of the command line: (arguments, standard input, the lines + the run must print on standard output, its exit status), from the + model and the header of Main.lean.""" + folder = Path(folder) + cases = [] + lines, status = model_dump(sample) + db = folder / "cli.db" + db.write_bytes(sample) + cases.append((["dump", str(db)], b"", lines, status)) + for name, data in (("noheader.db", b"not a database file at all\n" * 40), ("empty.db", b""), + ("headeronly.db", sample[:100]), ("partial.db", sample + b"\x0d" * 300), + ("cut.db", sample[:99])): + path = folder / name + path.write_bytes(data) + lines, status = model_dump(data) + cases.append((["dump", str(path)], b"", lines, status)) + cases.append((["dump", str(folder / "does-not-exist.db")], b"", [], 2)) + sub = folder / "a-folder" + sub.mkdir(exist_ok=True) + cases.append((["dump", str(sub)], b"", [], 2)) + cases.append((["serve"], b"", [], 0)) + for args in ([], ["dump"], ["dump", str(db), str(db)], ["serve", "x"], ["Dump", str(db)], ["bogus"], + ["header", "00"], ["--help"], ["dump", ""]): + cases.append((args, b"", [], 2)) + return cases + + +def run_cli(cases): + right, failures = 0, [] + for args, stdin, want, status in cases: + try: + p = subprocess.run([str(EXE)] + args, input=stdin, capture_output=True, timeout=600) + text = p.stdout.decode("utf-8", "backslashreplace") + got, code_ = (text[:-1].split("\n") if text.endswith("\n") else + ([text] if text else [])), p.returncode + except (OSError, subprocess.SubprocessError) as e: + got, code_ = [str(e)], None + if (got, code_) == (want, status): + right += 1 + else: + first = next((i for i, (a, b) in enumerate(zip(want, got)) if a != b), min(len(want), len(got))) + failures.append((f"sqlite {' '.join(args)}"[:120], + f"{len(want)} lines, exit {status}; line {first + 1}: " + f"{(want[first] if first < len(want) else '(none)')[:120]}", + f"{len(got)} lines, exit {code_}; line {first + 1}: " + f"{(got[first] if first < len(got) else '(none)')[:120]}")) + return right, failures + + +# --------------------------------------------------------------------- + + +def lean_version(): + try: + out = subprocess.run(["lean", "--version"], cwd=ROOT, capture_output=True, text=True, + errors="backslashreplace", timeout=300) + first = out.stdout.strip().splitlines()[:1] + return first[0] if first else "unknown" + except (OSError, subprocess.TimeoutExpired): + return "unknown" + + +def first_line(cmd): + try: + out = subprocess.run(cmd, capture_output=True, text=True, timeout=300) + return (out.stdout or out.stderr).strip().splitlines()[0] + except (OSError, subprocess.SubprocessError, IndexError): + return "unknown" + + +def count_arg(text): + n = int(text) + if n < 1: + raise argparse.ArgumentTypeError("the count must be at least 1") + return n + + +def write(lines): + RESULTS.write_text("\n".join(lines), encoding="utf-8") + + +def code(text): + """`text` as Markdown inline code, whatever backticks it holds.""" + return f"`` {text} ``" if "`" in text else f"`{text}`" + + +def short(text, n=200): + return text if len(text) <= n else text[:n] + f"... ({len(text)} characters)" + + +def tally(counter): + return ", ".join(f"{k} {v}" for k, v in sorted(counter.items(), key=lambda kv: (-kv[1], kv[0]))) + + +# --------------------------------------------------------------------- +# All the requests sent to `sqlite serve`, after the databases are made. + + +def every_byte_value(label, data): + for i in range(len(data)): + for v in range(256): + if v != data[i]: + yield (label, "header", [data[:i] + bytes([v]) + data[i + 1:]]) + + +def inputs(seed, count, dbs): + rng = random.Random(seed) + for op, args, _ in SPEC_CASES: + yield ("Spec.lean's examples and rejects", op, args) + yield from varint_sets(rng) + for _ in range(count): + yield ("random bytes", "varint", [random_varint(rng)]) + # Headers. + for db in dbs: + yield ("the header of a database made by SQLite", "header", + [db.data[:100 + rng.choice((0, 0, 1, 28))]]) + for db in dbs[:2]: + yield from every_byte_value("every value of every byte of a real header", db.data[:100]) + # The header of a database with no schema: its schema format and text + # encoding are 0, so this gives a text encoding of 0 with every schema + # format, and every text encoding with a schema format of 0. + no_schema = [db for db in dbs if be(db.data[44:48]) == 0 and be(db.data[56:60]) == 0][:1] + for db in no_schema: + yield from every_byte_value("every value of every byte of the header of a database with no schema", + db.data[:100]) + for base in [dbs[0].data[:100]] + [db.data[:100] for db in no_schema]: + for sf in range(6): + for enc in range(5): + yield ("each schema format from 0 to 5 with each text encoding from 0 to 4, in a real header", + "header", [base[:44] + sf.to_bytes(4, "big") + base[48:56] + enc.to_bytes(4, "big") + + base[60:]]) + for v in range(65536): + yield ("every value of the page size field", "header", + [SPEC_HEADER[:16] + v.to_bytes(2, "big") + SPEC_HEADER[18:]]) + for ps in PAGE_SIZES: + h = SPEC_HEADER[:16] + (1 if ps == 65536 else ps).to_bytes(2, "big") + SPEC_HEADER[18:] + for i in range(18, 24): + for v in range(256): + yield ("every value of bytes 18 to 23, for each page size", "header", + [h[:i] + bytes([v]) + h[i + 1:]]) + for k in range(100): + yield ("a real header cut short", "header", [dbs[0].data[:k]]) + for _ in range(count // 5): + yield ("a header from random fields", "header", [random_header(rng)]) + # Records. + for _ in range(count): + cols, rec = random_record(rng) + yield ("random record", "record", [rec]) + if rng.random() < 0.6: + label, b = changed_record(rng, cols, rec) + yield ("random record, " + label, "record", [b]) + for _ in range(count // 10): + yield ("random bytes", "record", [rng.randbytes(rng.choice((1, 2, 3, 8, 30)))]) + # `local`. + yield from local_sets(rng) + for _ in range(count // 2): + yield ("random", "local", random_local(rng)) + # Pages of the databases. + for db in dbs: + ps = db.settings.page_size + npages = len(db.data) // ps + if ps <= 2048: + for n in range(0, min(npages, 8) + 1): + yield ("page 0 and each of the first 8 pages of a database made by SQLite with pages " + "up to 2048 bytes, the file cut after it", "page", [n, db.data[:max(1, n) * ps]]) + if len(db.data) <= 131072: + yield ("the last page of a database made by SQLite of up to 128 KB, and the page after it", + "page", [npages, db.data]) + yield ("the last page of a database made by SQLite of up to 128 KB, and the page after it", + "page", [npages + 1, db.data]) + for db in dbs: + ps = db.settings.page_size + npages = len(db.data) // ps + picks = list(range(2, npages + 1)) + rng.shuffle(picks) + for x in picks[:3 if ps >= 16384 else 40]: + f = db.data[:ps] + db.data[(x - 1) * ps:x * ps] + yield ("a page of a database made by SQLite, as page 2", "page", [2, f]) + if rng.random() < 0.25: + yield ("a page of a database made by SQLite, as page 2, the file one byte short", + "page", [2, f[:-1]]) + readable = [db for db in dbs if read_header(db.data)[0] is not None] + for label, op, (n, f) in edge_pages(rng): + yield (label, op, [n, f]) + yield (label + ", the file one byte short", op, [n, f[:-1]]) + small = [db for db in readable if db.settings.page_size <= 4096] + large = [db for db in readable if db.settings.page_size > 4096] + for _ in range(count // 3): + db = (rng.choice(large) if large and rng.random() < 0.01 else + rng.choices(small, [1 / d.settings.page_size for d in small])[0]) + ps = db.settings.page_size + npages = len(db.data) // ps + x = rng.randint(1, npages) + if x == 1 or rng.random() < 0.2: + f, n = db.data[:ps], 1 + else: + f, n = db.data[:ps] + db.data[(x - 1) * ps:x * ps], 2 + h, _ = read_header(f) + label, b = changed_page(rng, h, f, n) + yield ("a page of a database made by SQLite, " + label, "page", [n, b]) + for _ in range(count // 5): + r = rng.random() + ps = 512 if r < 0.55 else 1024 if r < 0.75 else rng.choice((2048, 4096)) if r < 0.995 else \ + rng.choice((8192, 16384, 32768, 65536)) + reserved = random_reserved(rng, ps) + if ps == 65536 and rng.random() < 0.5: + reserved = 0 + f, defect = built_page(rng, ps, reserved, rng.choice((1, 2))) + yield ("a built page, " + defect, "page", [2 if len(f) > ps else 1, f]) + + +def changed_databases(rng, dbs, folder, k): + """k copies of small databases, to dump, each with one to three of + its pages changed in one way each, in the ways of changed_page that + keep the file's length and its 100-byte header. Each copy differs + from its database.""" + readable = [db for db in dbs if read_header(db.data)[0] is not None] + small = [db for db in readable if len(db.data) <= 262144] or readable + out = [] + for i in range(k): + db = rng.choice(small) + h, _ = read_header(db.data) + f = db.data + ps = h[H_PAGE_SIZE] + changes = rng.randint(1, 3) + while changes or f == db.data: + n = rng.randint(1, len(f) // ps) + _, f = changed_page(rng, h, f, n, keep_shape=True) + changes = max(0, changes - 1) + assert len(f) == len(db.data) and f[:100] == db.data[:100] + path = Path(folder) / f"changed{i}.db" + path.write_bytes(f) + out.append((path, f)) + return out + + +# --------------------------------------------------------------------- + +# How the model answers a `header` request with a text encoding of 0. +ENC0_ANSWERS = ("read", "refused for the text encoding", "refused for another rule") + + +def main(): + global EXE + sys.set_int_max_str_digits(0) + ap = argparse.ArgumentParser() + ap.add_argument("--seed", type=int, default=20261028) + ap.add_argument("--count", type=count_arg, default=100000) + ap.add_argument("--no-oracles", action="store_true", + help="compare with the model only, as tests/plant_bugs.py does") + ap.add_argument("--no-build", action="store_true", + help="use .lake/build/bin/sqlite as it is, without `lake build sqlite`") + args = ap.parse_args() + os.environ["PATH"] = f"{Path.home() / '.elan' / 'bin'}{os.pathsep}{os.environ.get('PATH', '')}" + started = time.monotonic() + + def give_up(why): + write(["# Test results", "", "- Oracle: none ran", f"- Seed: {args.seed}", "- Inputs: 0", + f"- Failures: 1 ({why})", ""]) + sys.exit(why) + + if args.no_build: + built = "`.lake/build/bin/sqlite` as it was, not rebuilt (`--no-build`)" + newest = max(p.stat().st_mtime for p in (ROOT / "Main.lean", ROOT / "Sqlite" / "Basic.lean", + ROOT / "Spec" / "Defs.lean")) + if not BUILT_EXE.exists(): + give_up("`--no-build`, and there is no .lake/build/bin/sqlite") + if BUILT_EXE.stat().st_mtime < newest: + built += "; it is older than the sources" + else: + built = "built by `lake build sqlite`" + try: + build = subprocess.run(["lake", "build", "sqlite"], cwd=ROOT, capture_output=True, + text=True, errors="backslashreplace", timeout=BUILD_TIMEOUT) + if build.returncode != 0: + print(build.stdout + build.stderr) + give_up("`lake build sqlite` failed") + except (OSError, subprocess.SubprocessError) as e: + give_up(f"`lake build sqlite` failed: {e}") + + failures = [] + with tempfile.TemporaryDirectory() as tmpname: + tmp = Path(tmpname) + EXE = tmp / "sqlite" + shutil.copy2(BUILT_EXE, EXE) + exe_hash = hashlib.sha256(EXE.read_bytes()).hexdigest() + try: + lib, lib_why = Lib(tmp), None + except Exception as e: # without ctypes, no reserved bytes and no legacy format + lib, lib_why = None, str(e).splitlines()[0][:200] if str(e) else type(e).__name__ + left_out = [] + oracle = not args.no_oracles + if args.no_oracles: + left_out.append(("SQLite", "--no-oracles")) + elif sqlite3.sqlite_version != SQLITE_PINNED: + left_out.append(("SQLite", f"{sqlite3.sqlite_version} is installed, not {SQLITE_PINNED}")) + oracle = False + else: + try: + sqlite3.connect(":memory:").execute("SELECT * FROM dbstat").fetchall() + except sqlite3.Error as e: + left_out.append(("SQLite", f"no dbstat: {e}")) + oracle = False + for name, why in left_out: + print(f"oracle left out: {name}: {why}") + + # The databases, each made twice: the same seed must make the + # same file, or the record cannot be checked by a rerun. + dbs = [] + made_twice = 0 + for i, (label, s, sd) in enumerate(all_databases(args.seed, args.count, lib)): + path = tmp / f"db{i}.db" + meta = make_database(path, s, random.Random(sd), lib) + data = path.read_bytes() + again = tmp / f"db{i}-again.db" + make_database(again, s, random.Random(sd), lib) + if again.read_bytes() == data: + made_twice += 1 + else: + failures.append((f"database {i} ({label})", "the same bytes when made again from the same seed", + "other bytes")) + again.unlink() + facts = None + if oracle: + facts = ask_sqlite(path, meta) + if path.read_bytes() != data: + failures.append((f"database {i}", "the same file after SQLite is asked about it", + "a changed file")) + dbs.append(Database(i, label, s, path, data, facts)) + made = time.monotonic() + print(f"{len(dbs)} databases made in {made - started:.0f} s", flush=True) + changed = changed_databases(random.Random(args.seed + 5), dbs, tmp, 4 + args.count // 2000) + + # The requests. + items = list(inputs(args.seed, args.count, dbs)) + lines = [request(op, a) + ("\r\n" if i % 7 == 6 else "\n") for i, (_, op, a) in enumerate(items)] + digest = hashlib.sha256() + for data in [db.data for db in dbs] + [f for _, f in changed]: + digest.update(hashlib.sha256(data).digest()) + for line in lines: + digest.update(line.encode("utf-8")) + inputs_digest = digest.hexdigest() + odd = MALFORMED + ODD + served = {} + + def serve(): + served["out"] = run_parallel([str(EXE), "serve"], lines + [m + "\n" for m in odd], JOBS) + + server = threading.Thread(target=serve) + server.start() + wants = [model(op, a) for _, op, a in items] + server.join() + out, problems = served["out"] + print(f"{len(items)} requests answered, {time.monotonic() - made:.0f} s", flush=True) + for problem in problems: + failures.append(("(whole run)", "a clean run", problem)) + tool, odd_out = out[:len(items)], out[len(items):] + by_label = collections.Counter(f"{label} ({op})" for label, op, _ in items) + by_op = collections.Counter(op for _, op, _ in items) + refused = collections.Counter() + enc0 = collections.Counter() + differ_model = 0 + spec_right = 0 + for i, ((label, op, a), (want, why), got) in enumerate(zip(items, wants, tool)): + if why is not None: + refused[(op, why)] += 1 + if op == "header" and len(a[0]) >= 100 and be(a[0][56:60]) == 0: + enc0[(min(5, be(a[0][44:48])), ENC0_ANSWERS[0] if why is None else ENC0_ANSWERS[1] + if why == WHY_H_ENCODING_0 else ENC0_ANSWERS[2])] += 1 + if label == "Spec.lean's examples and rejects": + spec_right += want == SPEC_CASES[i][2] + if want != SPEC_CASES[i][2]: + failures.append((f"the model on {short(request(op, a), 120)}", SPEC_CASES[i][2], want)) + if got != want: + differ_model += 1 + failures.append((short(lines[i].rstrip("\r\n"), 160), short(want, 160), short(got, 160))) + del lines, wants, tool + malformed_right = sum(1 for a in odd_out[:len(MALFORMED)] if a == "error") + for m in MALFORMED: + if model_line(m + "\n") != "error": + failures.append((f"the model on {m!r}", "error", model_line(m + "\n"))) + odd_right = 0 + for m, a in zip(odd, odd_out): + want = model_line(m + "\n") + if a == want: + odd_right += m in ODD + else: + failures.append((f"request {short(m, 100)!r}", short(want, 100), short(a, 100))) + eof_right = 0 + for req, want in AT_EOF: + if model_line(req) != want: + failures.append((f"the model on {req!r} at end of input", want, model_line(req))) + got, problem = run_lines([str(EXE), "serve"], [req], timeout=60) + if got == [want] and not problem: + eof_right += 1 + else: + failures.append((repr(req) + " (at end of input)", want, f"{got} {problem or ''}".strip())) + + # The dumps. + check = Check() + dump_kinds = collections.Counter() + page_kinds = collections.Counter() + dumps_right = 0 + dump_lines = 0 + # Databases SQLite made with no schema, so a schema format and a + # text encoding of 0, and how many of them the tool's dump reads. + no_schema = no_schema_read = 0 + jobs = [(db.path, db.data, db) for db in dbs] + [(p, f, None) for p, f in changed] + with concurrent.futures.ThreadPoolExecutor(JOBS) as pool: + results = pool.map(lambda job: run_dump(job[0]), jobs) + for (path, data, db), (got, status, problem) in zip(jobs, results): + label = "a database made by SQLite" if db else "a database made by SQLite, bytes changed" + by_label[f"{label} (dump)"] += 1 + by_op["dump"] += 1 + want, want_status = model_dump(data) + dump_lines += len(want) + for line in want: + if line.startswith("page "): + page_kinds["FAIL" if line.endswith(" FAIL") else line.split(" ")[2]] += 1 + elif line.startswith("record ") and line.endswith(" FAIL"): + page_kinds["record FAIL"] += 1 + if (got, status, problem) == (want, want_status, None): + dumps_right += 1 + else: + differ_model += 1 + first = next((k for k, (a, b) in enumerate(zip(want, got)) if a != b), + min(len(want), len(got))) + failures.append((f"sqlite dump {path.name} ({db.where() if db else 'bytes changed'})", + f"{len(want)} lines, exit {want_status}; line {first + 1}: " + + short(want[first] if first < len(want) else "(none)", 160), + f"{len(got)} lines, exit {status}{', ' + problem if problem else ''}; " + f"line {first + 1}: " + short(got[first] if first < len(got) else "(none)", + 160))) + if db is not None and be(data[44:48]) == 0 and be(data[56:60]) == 0: + no_schema += 1 + no_schema_read += status == 0 and got[:1] == want[:1] and bool(want) + if db is not None and oracle: + try: + db.read_back(got, want, check) + except Exception as e: # a dump that does not read back + check.fail("databases read back", db.where(), "a dump that reads back", + f"{type(e).__name__}: {e}") + print(f"{len(jobs)} dumps, {time.monotonic() - made:.0f} s", flush=True) + if not no_schema: + failures.append(("(inputs)", "a database made by SQLite with no schema, so a schema format and " + "a text encoding of 0", "none")) + + # Payloads that spill, joined from their overflow pages. + spill_items = [(db, k, p) for db in dbs if db.tool is not None for k, p in enumerate(db.tool.spills)] + spill_lines = [request("record", [p]) + "\n" for _, _, p in spill_items] + spill_out, problems = run_parallel([str(EXE), "serve"], spill_lines, JOBS) + for problem in problems: + failures.append(("(payloads that spill)", "a clean run", problem)) + spilled = {} + for (db, k, p), got in zip(spill_items, spill_out): + want, why = model("record", [p]) + by_label["a payload that spills, joined from its overflow pages (record)"] += 1 + by_op["record"] += 1 + if why is not None: + refused[("record", why)] += 1 + if got != want: + differ_model += 1 + failures.append((short(request("record", [p]), 160), short(want, 160), short(got, 160))) + spilled.setdefault(db.index, {})[k] = got + if oracle: + for db in dbs: + if db.read: + try: + db.compare_entries(spilled.get(db.index, {}), check) + except Exception as e: + check.fail("databases read back", db.where(), "rows that read back", + f"{type(e).__name__}: {e}") + failures += check.failures + + # The command line. + sample = min((db.data for db in dbs if read_header(db.data)[0] is not None + and 2 * db.settings.page_size <= len(db.data)), key=len) + cli = cli_cases(tmp, sample) + cli_right, cli_failures = run_cli(cli) + failures += cli_failures + + n_inputs = sum(by_label.values()) + settings = [db.settings for db in dbs] + pages_total = sum(len(db.data) // db.settings.page_size for db in dbs) + lines_out = [ + "# Test results", + "", + "Written by `tests/run_tests.py`. Rerun it to check these numbers.", + "", + f"- Lean: {lean_version()}", + f"- Executable: {built}; SHA-256 of the copy run {exe_hash}", + "- Oracle: the model of the file format in `tests/run_tests.py`" + + ("" if not oracle else f"; SQLite {sqlite3.sqlite_version}"), + ] + if oracle: + con = sqlite3.connect(":memory:") + source = con.execute("SELECT sqlite_source_id()").fetchone()[0] + con.close() + package = first_line(["dpkg-query", "-W", "-f", "${Package} ${Version}\n", "libsqlite3-0"]) + lines_out.append( + f"- SQLite: {sqlite3.sqlite_version} ({source})" + + (f", the library of Debian package {package}" if package != "unknown" else "") + + f", through the sqlite3 module of Python {sys.version.split()[0]}, with dbstat") + lines_out += [ + "- Oracles left out: " + ("; ".join(f"{n} ({why})" for n, why in left_out) or "none"), + f"- Seed: {args.seed}", + f"- Inputs: {n_inputs}", + f"- Inputs SHA-256: {inputs_digest}, of the files made and the requests built from them, in order; " + "a rerun with the same seed and count must give the same", + "- By kind: " + ", ".join(f"{k} {v}" for k, v in sorted(by_label.items())), + "- Requests: " + ", ".join(f"`{op}` {n}" for op, n in sorted(by_op.items())), + f"- Answers that differ from the model: {differ_model}", + f"- Spec.lean's `examples` and `rejects`, answered by the model as Spec.lean states them: " + f"{spec_right} of {len(SPEC_CASES)}", + ] + for op, name in (("varint", "Varints"), ("header", "Headers"), ("page", "Pages asked with `page`"), + ("record", "Records")): + n = by_op[op] + no = sum(v for (o, _), v in refused.items() if o == op) + reasons = collections.Counter({why: v for (o, why), v in refused.items() if o == op}) + lines_out.append(f"- {name}: {n}, of which the model reads {n - no}; why it refuses the others: " + + (tally(reasons) or "none")) + lines_out += [ + f"- Databases made by SQLite: {len(dbs)} ({len(FIXED_DATABASES)} fixed, the rest random), " + f"{pages_total} pages in all. Page sizes: " + + tally(collections.Counter(s.page_size for s in settings)) + + "; with reserved bytes: " + str(sum(1 for s in settings if s.reserve)) + + "; text encodings: " + tally(collections.Counter(s.encoding or "UTF-8" for s in settings)) + + "; auto_vacuum: " + tally(collections.Counter(("none", "full", "incremental")[s.auto_vacuum or 0] + for s in settings)) + + f"; WAL: {sum(1 for s in settings if s.wal)}; legacy file format: " + f"{sum(1 for s in settings if s.legacy)}; secure_delete off: " + f"{sum(1 for s in settings if s.secure_delete == 'OFF')}" + + ("" if lib else f"; reserved bytes and the legacy format not set ({lib_why})") + + f". Each made twice from the same seed, the same bytes both times: {made_twice} of {len(dbs)}", + f"- Dumps: {len(jobs)}, of which {len(changed)} are of copies with one to three pages changed, each " + f"different from its database; {dump_lines} lines; as the model prints them: {dumps_right} of " + f"{len(jobs)}. Pages in them, by type: " + + tally(collections.Counter({("page type " + k if k != "FAIL" else "not read (FAIL)") + if k != "record FAIL" else "records not read (FAIL)": v + for k, v in page_kinds.items()})) + + f". Databases made by SQLite with no schema (schema format 0, text encoding 0): {no_schema}, of " + f"which the tool's dump reads the header as the model does: {no_schema_read}", + "- `header` requests with a text encoding of 0, by schema format, as the model answers them (an " + "answer of the tool's that differs is a failure above): " + + "; ".join(f"schema format {'5 or more' if sf == 5 else sf}: " + + ", ".join(f"{how} {enc0[(sf, how)]}" for how in ENC0_ANSWERS if enc0[(sf, how)]) + for sf in range(6) if any(enc0[(sf, how)] for how in ENC0_ANSWERS)), + ] + if oracle: + lines_out.append( + "- Agreement with SQLite (the rest are the named differences below, or failures): " + + ", ".join(f"{what} {check.agree[what]} of {check.asked[what]}" for what in sorted(check.asked))) + lines_out.append( + "- SQLite's answers that the model's dump, read with SQLite's rules, gives, whether or not the " + "tool agrees (any other is a failure): " + + ", ".join(f"{what} {check.model_agree[what]} of {check.model_asked[what]}" + for what in sorted(check.model_asked))) + lines_out.append(f"- Payloads that spill, joined from the part the tool keeps on the page and the " + f"overflow pages, and read whole by `record`: {len(spill_items)}") + lines_out.append( + f"- Pages SQLite uses as overflow, freelist or pointer-map pages: {sum(check.other.values())} (" + + ", ".join(f"{ROLES[k]} {check.other[k]}" for k in ROLES if check.other[k]) + + f"), of which the tool reads {sum(check.other_read.values())} as b-tree pages from their bytes (" + + (", ".join(f"{ROLES[k]} {check.other_read[k]}" for k in ROLES if check.other_read[k]) or "none") + + "). A count, not a check: SQLite gives no b-tree reading of these pages, and their lines in " + "the dumps are compared with the model's like every other") + else: + lines_out.append("- Agreement with SQLite: SQLite left out") + lines_out += [ + f"- Malformed requests answered `error`: {malformed_right} of {len(MALFORMED)}; odd but " + f"well-formed requests answered as the model says: {odd_right} of {len(ODD)}", + f"- Requests ending in CR LF: every seventh, {len(items) // 7}", + f"- Last lines sent alone, with no LF, with CR LF, or with a lone CR, answered right: " + f"{eof_right} of {len(AT_EOF)}", + f"- Command line (`sqlite dump`, `sqlite serve` and wrong arguments), printing and exiting as " + f"the model says: {cli_right} of {len(cli)}", + f"- Time: {time.monotonic() - started:.0f} s", + f"- Failures: {len(failures)}", + "", + ] + if oracle: + lines_out += ["## Where SQLite and the tool differ", "", + "Each disagreement is counted under the reason the model confirms: the model's " + "dump, read with SQLite's own rules, gives SQLite's answer, and needs the rules " + "named to do so. The first case that showed it is given.", ""] + if not check.named: + lines_out.append("No disagreement.") + for reason, (n, example) in sorted(check.named.items(), key=lambda kv: -kv[1][0]): + lines_out.append(f"- {reason}: {n}; for example {short(example, 300)}") + lines_out.append("") + if failures: + lines_out += ["## Failures", ""] + lines_out += [f"- {code(short(req, 300))}: expected {code(want)}, got {code(got)}" + for req, want, got in failures[:60]] + if len(failures) > 60: + lines_out.append(f"- and {len(failures) - 60} more") + lines_out.append("") + write(lines_out) + print(f"{n_inputs} inputs, {len(failures)} failures; results in {RESULTS}") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())