macltok

public

Star0

Fork 0

Watch0

92 commits 3 branches 0 tags 0 releases3.0 MiB

Commit `c9effeac326a`

tarball

atagen <boss@atagen.co>2026-10-25 13:25 UTC

22 files changed+4455-7

search: index messages locally and search them

Encrypted rooms can't use the server's /search, so a tantivy index fed from the event cache and a paced /messages backfill answers SearchCommand for every room alike. The fork's matrix-sdk-search was bypassed because its schema is private and its IndexWriter and mmap can't run in wasm, so macltok keeps immutable SingleSegmentIndexWriter segments with tombstone bitsets and merges them itself. Segments are sealed blobs in macltok-search.sqlite3 or the ::macltok-search IndexedDB database and go with the store.

Modified file: Cargo.lock+517 additions−1 deletions@@ -51,6 +51,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
+[[package]]
 name = "android_system_properties"
 version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -86,6 +92,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "arc-swap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
 name = "archery"
 version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -290,6 +305,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6"
 
 [[package]]
+name = "bitpacking"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019"
+dependencies = [
+ "crunchy",
+]
+
+[[package]]
 name = "blake3"
 version = "1.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -321,6 +345,29 @@ dependencies = [
 ]
 
 [[package]]
+name = "bon"
+version = "3.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a05442cd7005532d0cfedec676b5b9333d2b4c7c3afabe2b1a9ce2742fbe2ee6"
+dependencies = [
+ "bon-macros",
+]
+
+[[package]]
+name = "bon-macros"
+version = "3.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93833e97ac7c46ba97045afcf247238c27057538fcf2f06344662ddd5f567be6"
+dependencies = [
+ "darling",
+ "ident_case",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "syn 3.0.5",
+]
+
+[[package]]
 name = "bs58"
 version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -381,6 +428,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "census"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
+
+[[package]]
 name = "cfg-if"
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -573,12 +626,46 @@ dependencies = [
 ]
 
 [[package]]
+name = "crossbeam-channel"
+version = "0.5.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
 name = "crossbeam-utils"
 version = "0.8.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
 
 [[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
+[[package]]
 name = "crypto-common"
 version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -637,6 +724,46 @@ dependencies = [
 ]
 
 [[package]]
+name = "darling"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff"
+dependencies = [
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 3.0.5",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn 3.0.5",
+]
+
+[[package]]
+name = "datasketches"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745"
+
+[[package]]
 name = "date_header"
 version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -723,6 +850,9 @@ name = "deranged"
 version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
+dependencies = [
+ "serde_core",
+]
 
 [[package]]
 name = "derivative"
@@ -800,6 +930,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "downcast-rs"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
+
+[[package]]
 name = "dunce"
 version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -859,6 +995,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
 
 [[package]]
+name = "erased-serde"
+version = "0.4.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e59b668157f32d3459d4bd634861a06ffa2503a461ca5f0916b9f0252605e66"
+dependencies = [
+ "serde",
+ "serde_core",
+ "typeid",
+]
+
+[[package]]
 name = "errno"
 version = "0.3.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -953,6 +1100,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "fastdivide"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
+
+[[package]]
 name = "fastrand"
 version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -994,6 +1147,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
 
 [[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
+[[package]]
 name = "form_urlencoded"
 version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1233,7 +1392,18 @@ version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 dependencies = [
- "foldhash",
+ "foldhash 0.1.5",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
 ]
 
 [[package]]
@@ -1286,6 +1456,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "htmlescape"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
+
+[[package]]
 name = "http"
 version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1498,6 +1674,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
 name = "idna"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1610,6 +1792,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "inventory"
+version = "0.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
 name = "ipnet"
 version = "2.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1750,6 +1941,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
+name = "levenshtein_automata"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
+
+[[package]]
 name = "libc"
 version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1813,12 +2010,27 @@ dependencies = [
 ]
 
 [[package]]
+name = "lru"
+version = "0.16.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
+dependencies = [
+ "hashbrown 0.16.1",
+]
+
+[[package]]
 name = "lru-slab"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
 
 [[package]]
+name = "lz4_flex"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
+
+[[package]]
 name = "macltok"
 version = "0.1.0"
 dependencies = [
@@ -1829,6 +2041,7 @@ dependencies = [
  "http",
  "imbl",
  "js-sys",
+ "log",
  "matrix-sdk",
  "matrix-sdk-base",
  "matrix-sdk-crypto",
@@ -1843,6 +2056,8 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2",
+ "tantivy",
+ "tantivy-common",
  "thiserror 2.0.20",
  "tokio",
  "tracing",
@@ -2346,6 +2561,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "measure_time"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
+dependencies = [
+ "log",
+]
+
+[[package]]
 name = "memchr"
 version = "2.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2374,6 +2598,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
 name = "miniz_oxide"
 version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2395,12 +2625,28 @@ dependencies = [
 ]
 
 [[package]]
+name = "murmurhash32"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
+
+[[package]]
 name = "new_debug_unreachable"
 version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
 
 [[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
 name = "nu-ansi-term"
 version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2471,6 +2717,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
+name = "oneshot"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
+
+[[package]]
 name = "oorandom"
 version = "11.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2489,6 +2741,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
 
 [[package]]
+name = "ordered-float"
+version = "5.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "ownedbytes"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
+dependencies = [
+ "stable_deref_trait",
+]
+
+[[package]]
 name = "parking"
 version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2677,6 +2947,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
 
 [[package]]
+name = "prettyplease"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
+dependencies = [
+ "proc-macro2",
+ "syn 3.0.5",
+]
+
+[[package]]
 name = "proc-macro-crate"
 version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2911,6 +3191,26 @@ dependencies = [
 ]
 
 [[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
 name = "readlock"
 version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3551,6 +3851,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
 
 [[package]]
+name = "sketches-ddsketch"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "513c3f5f732bfd6fbb187619c2dfe9d2f25f1a2976f01d575f0fd329d565df56"
+dependencies = [
+ "serde",
+]
+
+[[package]]
 name = "slab"
 version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3616,6 +3925,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
+[[package]]
 name = "subtle"
 version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3675,6 +3990,149 @@ dependencies = [
 ]
 
 [[package]]
+name = "tantivy"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "861facfabd71044968f364837f9a083b56464ba5a59079f88706ee5c451ca069"
+dependencies = [
+ "aho-corasick",
+ "arc-swap",
+ "base64",
+ "bitpacking",
+ "bon",
+ "byteorder",
+ "census",
+ "crc32fast",
+ "crossbeam-channel",
+ "datasketches",
+ "downcast-rs",
+ "fastdivide",
+ "fnv",
+ "htmlescape",
+ "itertools 0.14.0",
+ "levenshtein_automata",
+ "log",
+ "lru",
+ "lz4_flex",
+ "measure_time",
+ "once_cell",
+ "oneshot",
+ "rayon",
+ "regex",
+ "rustc-hash",
+ "serde",
+ "serde_json",
+ "sketches-ddsketch",
+ "smallvec",
+ "tantivy-bitpacker",
+ "tantivy-columnar",
+ "tantivy-common",
+ "tantivy-fst",
+ "tantivy-query-grammar",
+ "tantivy-stacker",
+ "tantivy-tokenizer-api",
+ "thiserror 2.0.20",
+ "time",
+ "typetag",
+ "uuid",
+ "winapi",
+]
+
+[[package]]
+name = "tantivy-bitpacker"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4"
+dependencies = [
+ "bitpacking",
+]
+
+[[package]]
+name = "tantivy-columnar"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc"
+dependencies = [
+ "downcast-rs",
+ "fastdivide",
+ "itertools 0.14.0",
+ "serde",
+ "tantivy-bitpacker",
+ "tantivy-common",
+ "tantivy-sstable",
+ "tantivy-stacker",
+]
+
+[[package]]
+name = "tantivy-common"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5"
+dependencies = [
+ "async-trait",
+ "byteorder",
+ "ownedbytes",
+ "serde",
+ "time",
+]
+
+[[package]]
+name = "tantivy-fst"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
+dependencies = [
+ "byteorder",
+ "regex-syntax",
+ "utf8-ranges",
+]
+
+[[package]]
+name = "tantivy-query-grammar"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82"
+dependencies = [
+ "fnv",
+ "nom",
+ "ordered-float",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "tantivy-sstable"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606"
+dependencies = [
+ "futures-util",
+ "itertools 0.14.0",
+ "tantivy-bitpacker",
+ "tantivy-common",
+ "tantivy-fst",
+]
+
+[[package]]
+name = "tantivy-stacker"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951"
+dependencies = [
+ "murmurhash32",
+ "tantivy-common",
+]
+
+[[package]]
+name = "tantivy-tokenizer-api"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98"
+dependencies = [
+ "serde",
+]
+
+[[package]]
 name = "tempfile"
 version = "3.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4024,12 +4482,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
 [[package]]
+name = "typeid"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
+
+[[package]]
 name = "typenum"
 version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
 
 [[package]]
+name = "typetag"
+version = "0.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2"
+dependencies = [
+ "erased-serde",
+ "inventory",
+ "once_cell",
+ "serde",
+ "typetag-impl",
+]
+
+[[package]]
+name = "typetag-impl"
+version = "0.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.5",
+]
+
+[[package]]
 name = "typewit"
 version = "1.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4114,6 +4602,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
 
 [[package]]
+name = "utf8-ranges"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"
+
+[[package]]
 name = "utf8_iter"
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4396,6 +4890,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "29333c3ea1ba8b17211763463ff24ee84e41c78224c16b001cd907e663a38c68"
 
 [[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
 name = "winapi-util"
 version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4405,6 +4915,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
 name = "windows-core"
 version = "0.62.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"

Modified file: Cargo.toml+6 additions@@ -37,6 +37,9 @@ serde_json = { version = "1.0.143", features = ["raw_value"] }
 thiserror = "2.0.16"
 mime = "0.3.17"
 sha2 = "0.10.9"
+# no mmap, which wasm can't have, and no zstd, which is C
+tantivy = { version = "0.26.1", default-features = false, features = ["lz4-compression"] }
+tantivy-common = "0.11"
 tokio = { version = "1.47.1", features = ["macros", "sync"] }
 tracing = "0.1.41"
 url = { version = "2.5.7", optional = true }
@@ -48,6 +51,9 @@ getrandom3 = { package = "getrandom", version = "0.3", features = ["wasm_js"] }
 getrandom4 = { package = "getrandom", version = "0.4", features = ["wasm_js"] }
 indexed_db_futures = { package = "matrix_indexed_db_futures", version = "0.7.0", optional = true, features = ["typed-arrays"] }
 js-sys = { version = "0.3.105", optional = true }
+# tantivy 0.26's merger reads Instant::now() whenever debug logging is on, and that panics in
+# wasm. both caps, since a release_max_level a host sets would otherwise win in release
+log = { version = "0.4.29", features = ["max_level_info", "release_max_level_info"] }
 matrix-sdk = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "5ec7600f0a5f648c6e2a34b5be0a1494a4e2eafd", default-features = false, features = ["js"] }
 matrix-sdk-ui = { git = "ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk", rev = "5ec7600f0a5f648c6e2a34b5be0a1494a4e2eafd", default-features = false, features = ["js"] }
 # the sdk's own wasm reqwest leaves out stream, which the bounded media download needs

Modified file: harness/web/Cargo.lock+517 additions−1 deletions@@ -51,6 +51,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "allocator-api2"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
+
+[[package]]
 name = "android_system_properties"
 version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -86,6 +92,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "arc-swap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
 name = "archery"
 version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -268,6 +283,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d084b0137aaa901caf9f1e8b21daa6aa24d41cd806e111335541eff9683bd6"
 
 [[package]]
+name = "bitpacking"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019"
+dependencies = [
+ "crunchy",
+]
+
+[[package]]
 name = "blake3"
 version = "1.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -299,6 +323,29 @@ dependencies = [
 ]
 
 [[package]]
+name = "bon"
+version = "3.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a05442cd7005532d0cfedec676b5b9333d2b4c7c3afabe2b1a9ce2742fbe2ee6"
+dependencies = [
+ "bon-macros",
+]
+
+[[package]]
+name = "bon-macros"
+version = "3.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93833e97ac7c46ba97045afcf247238c27057538fcf2f06344662ddd5f567be6"
+dependencies = [
+ "darling",
+ "ident_case",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "syn 3.0.5",
+]
+
+[[package]]
 name = "bs58"
 version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -359,6 +406,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "census"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
+
+[[package]]
 name = "cfg-if"
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -540,12 +593,46 @@ dependencies = [
 ]
 
 [[package]]
+name = "crossbeam-channel"
+version = "0.5.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
 name = "crossbeam-utils"
 version = "0.8.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
 
 [[package]]
+name = "crunchy"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
+
+[[package]]
 name = "crypto-common"
 version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -594,6 +681,46 @@ dependencies = [
 ]
 
 [[package]]
+name = "darling"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff"
+dependencies = [
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 3.0.5",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn 3.0.5",
+]
+
+[[package]]
+name = "datasketches"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745"
+
+[[package]]
 name = "date_header"
 version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -662,6 +789,9 @@ name = "deranged"
 version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
+dependencies = [
+ "serde_core",
+]
 
 [[package]]
 name = "derivative"
@@ -739,6 +869,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "downcast-rs"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
+
+[[package]]
 name = "dunce"
 version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -792,6 +928,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
 
 [[package]]
+name = "erased-serde"
+version = "0.4.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e59b668157f32d3459d4bd634861a06ffa2503a461ca5f0916b9f0252605e66"
+dependencies = [
+ "serde",
+ "serde_core",
+ "typeid",
+]
+
+[[package]]
 name = "errno"
 version = "0.3.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -886,6 +1033,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "fastdivide"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
+
+[[package]]
 name = "fastrand"
 version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -927,6 +1080,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
 
 [[package]]
+name = "foldhash"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
+
+[[package]]
 name = "form_urlencoded"
 version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1150,7 +1309,18 @@ version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 dependencies = [
- "foldhash",
+ "foldhash 0.1.5",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
+dependencies = [
+ "allocator-api2",
+ "equivalent",
+ "foldhash 0.2.0",
 ]
 
 [[package]]
@@ -1203,6 +1373,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "htmlescape"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
+
+[[package]]
 name = "http"
 version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1408,6 +1584,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
 name = "idna"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1505,6 +1687,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "inventory"
+version = "0.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
 name = "ipnet"
 version = "2.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1645,6 +1836,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
+name = "levenshtein_automata"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
+
+[[package]]
 name = "libc"
 version = "0.2.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1707,12 +1904,27 @@ dependencies = [
 ]
 
 [[package]]
+name = "lru"
+version = "0.16.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
+dependencies = [
+ "hashbrown 0.16.1",
+]
+
+[[package]]
 name = "lru-slab"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
 
 [[package]]
+name = "lz4_flex"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
+
+[[package]]
 name = "macltok"
 version = "0.1.0"
 dependencies = [
@@ -1723,6 +1935,7 @@ dependencies = [
  "http",
  "imbl",
  "js-sys",
+ "log",
  "matrix-sdk",
  "matrix-sdk-base",
  "matrix-sdk-crypto",
@@ -1735,6 +1948,8 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2",
+ "tantivy",
+ "tantivy-common",
  "thiserror 2.0.20",
  "tokio",
  "tracing",
@@ -2211,6 +2426,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "measure_time"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
+dependencies = [
+ "log",
+]
+
+[[package]]
 name = "memchr"
 version = "2.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2239,6 +2463,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
 name = "miniz_oxide"
 version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2260,12 +2490,28 @@ dependencies = [
 ]
 
 [[package]]
+name = "murmurhash32"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
+
+[[package]]
 name = "new_debug_unreachable"
 version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
 
 [[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
 name = "nu-ansi-term"
 version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2336,6 +2582,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
+name = "oneshot"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
+
+[[package]]
 name = "oorandom"
 version = "11.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2354,6 +2606,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
 
 [[package]]
+name = "ordered-float"
+version = "5.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "ownedbytes"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
+dependencies = [
+ "stable_deref_trait",
+]
+
+[[package]]
 name = "parking"
 version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2500,6 +2770,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
 
 [[package]]
+name = "prettyplease"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0"
+dependencies = [
+ "proc-macro2",
+ "syn 3.0.5",
+]
+
+[[package]]
 name = "proc-macro-crate"
 version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2734,6 +3014,26 @@ dependencies = [
 ]
 
 [[package]]
+name = "rayon"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
 name = "readlock"
 version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3358,6 +3658,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
 
 [[package]]
+name = "sketches-ddsketch"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "513c3f5f732bfd6fbb187619c2dfe9d2f25f1a2976f01d575f0fd329d565df56"
+dependencies = [
+ "serde",
+]
+
+[[package]]
 name = "slab"
 version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3423,6 +3732,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
+[[package]]
 name = "subtle"
 version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3482,6 +3797,149 @@ dependencies = [
 ]
 
 [[package]]
+name = "tantivy"
+version = "0.26.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "861facfabd71044968f364837f9a083b56464ba5a59079f88706ee5c451ca069"
+dependencies = [
+ "aho-corasick",
+ "arc-swap",
+ "base64",
+ "bitpacking",
+ "bon",
+ "byteorder",
+ "census",
+ "crc32fast",
+ "crossbeam-channel",
+ "datasketches",
+ "downcast-rs",
+ "fastdivide",
+ "fnv",
+ "htmlescape",
+ "itertools 0.14.0",
+ "levenshtein_automata",
+ "log",
+ "lru",
+ "lz4_flex",
+ "measure_time",
+ "once_cell",
+ "oneshot",
+ "rayon",
+ "regex",
+ "rustc-hash",
+ "serde",
+ "serde_json",
+ "sketches-ddsketch",
+ "smallvec",
+ "tantivy-bitpacker",
+ "tantivy-columnar",
+ "tantivy-common",
+ "tantivy-fst",
+ "tantivy-query-grammar",
+ "tantivy-stacker",
+ "tantivy-tokenizer-api",
+ "thiserror 2.0.20",
+ "time",
+ "typetag",
+ "uuid",
+ "winapi",
+]
+
+[[package]]
+name = "tantivy-bitpacker"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4"
+dependencies = [
+ "bitpacking",
+]
+
+[[package]]
+name = "tantivy-columnar"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc"
+dependencies = [
+ "downcast-rs",
+ "fastdivide",
+ "itertools 0.14.0",
+ "serde",
+ "tantivy-bitpacker",
+ "tantivy-common",
+ "tantivy-sstable",
+ "tantivy-stacker",
+]
+
+[[package]]
+name = "tantivy-common"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5"
+dependencies = [
+ "async-trait",
+ "byteorder",
+ "ownedbytes",
+ "serde",
+ "time",
+]
+
+[[package]]
+name = "tantivy-fst"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
+dependencies = [
+ "byteorder",
+ "regex-syntax",
+ "utf8-ranges",
+]
+
+[[package]]
+name = "tantivy-query-grammar"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82"
+dependencies = [
+ "fnv",
+ "nom",
+ "ordered-float",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "tantivy-sstable"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606"
+dependencies = [
+ "futures-util",
+ "itertools 0.14.0",
+ "tantivy-bitpacker",
+ "tantivy-common",
+ "tantivy-fst",
+]
+
+[[package]]
+name = "tantivy-stacker"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951"
+dependencies = [
+ "murmurhash32",
+ "tantivy-common",
+]
+
+[[package]]
+name = "tantivy-tokenizer-api"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98"
+dependencies = [
+ "serde",
+]
+
+[[package]]
 name = "tempfile"
 version = "3.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3829,12 +4287,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
 [[package]]
+name = "typeid"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
+
+[[package]]
 name = "typenum"
 version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
 
 [[package]]
+name = "typetag"
+version = "0.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2"
+dependencies = [
+ "erased-serde",
+ "inventory",
+ "once_cell",
+ "serde",
+ "typetag-impl",
+]
+
+[[package]]
+name = "typetag-impl"
+version = "0.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.5",
+]
+
+[[package]]
 name = "typewit"
 version = "1.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3913,6 +4401,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
 
 [[package]]
+name = "utf8-ranges"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"
+
+[[package]]
 name = "utf8_iter"
 version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4195,6 +4689,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "29333c3ea1ba8b17211763463ff24ee84e41c78224c16b001cd907e663a38c68"
 
 [[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
 name = "winapi-util"
 version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4204,6 +4714,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
 name = "windows-core"
 version = "0.62.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"

Modified file: src/account.rs+15 additions@@ -324,6 +324,17 @@ impl StoreConfig {
         format!("{}::macltok-mentions", self.indexeddb_name())
     }
 
+    #[cfg(feature = "sqlite")]
+    pub(crate) fn search_location(&self) -> PathBuf {
+        let StoreLocation::Sqlite(path) = &self.location;
+        path.join("macltok-search.sqlite3")
+    }
+
+    #[cfg(feature = "indexeddb")]
+    pub(crate) fn search_location(&self) -> String {
+        format!("{}::macltok-search", self.indexeddb_name())
+    }
+
     #[cfg(feature = "indexeddb")]
     pub(crate) fn indexeddb_name(&self) -> &str {
         let StoreLocation::IndexedDb(name) = &self.location;
@@ -854,6 +865,9 @@ impl Session {
                 .ok_or(AccountError::MissingSession)?
                 .to_owned(),
         };
+        let search =
+            crate::search::store::SearchStore::open(self.tracking.store.store(), account.clone())
+                .await;
         let opener = crate::mentions::store::Opener::new(
             self.tracking.store.store(),
             account,
@@ -878,6 +892,7 @@ impl Session {
             self.tracking.persistence,
             Some(self.store_lock),
             Some(mentions),
+            Some(search),
         )
         .await
         .map_err(AccountError::from)

Modified file: src/engine/context.rs+21 additions@@ -412,6 +412,27 @@ async fn fetch_side(
     })
 }
 
+/// a search hit as event context would show it, `values` its original and then its latest edit
+pub(crate) async fn preview(
+    room: &Room,
+    current_user_id: Option<&ruma::UserId>,
+    values: &[Value],
+    highlight: bool,
+) -> Option<(TimelineEvent, Vec<MediaRegistration>)> {
+    let mut page = Assembly::new(
+        room,
+        current_user_id,
+        values,
+        &vec![highlight; values.len()],
+        Some(0),
+    )
+    .await;
+    match page.take(0..1).pop()? {
+        TimelineItem::Event(event) => Some((*event, page.media)),
+        _ => None,
+    }
+}
+
 /// a window of raw events normalised together, so relations inside it land on their targets
 struct Assembly {
     items: Vec<(usize, TimelineItem)>,

Modified file: src/engine/mod.rs+62 additions−1 deletions@@ -40,6 +40,11 @@ pub(crate) use crate::mentions::store::MentionsStore;
 #[cfg(not(any(feature = "sqlite", feature = "indexeddb")))]
 pub(crate) enum MentionsStore {}
 
+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]
+pub(crate) use crate::search::store::SearchStore;
+#[cfg(not(any(feature = "sqlite", feature = "indexeddb")))]
+pub(crate) enum SearchStore {}
+
 use self::{
     commands::Job,
     executor::{EngineBoxFuture, TaskSet},
@@ -73,6 +78,13 @@ pub struct EngineConfig {
     pub max_mention_candidates: usize,
     /// warm a room up for sending when it's opened or someone starts typing in it
     pub prepare_to_send: bool,
+    /// `/messages` pages each joined room gets per engine run for search backfill. 0 for none
+    pub search_backfill_pages_per_room: u16,
+    pub search_backfill_page_size: u16,
+    /// the least time between two backfill requests, across every room
+    pub search_backfill_interval_ms: u64,
+    /// undecryptable messages kept to index once their keys turn up, newest first
+    pub max_search_undecrypted: usize,
 }
 
 impl EngineConfig {
@@ -135,6 +147,30 @@ impl EngineConfig {
         self.prepare_to_send = prepare;
         self
     }
+
+    #[must_use]
+    pub const fn with_search_backfill_pages_per_room(mut self, pages: u16) -> Self {
+        self.search_backfill_pages_per_room = pages;
+        self
+    }
+
+    #[must_use]
+    pub const fn with_search_backfill_page_size(mut self, events: u16) -> Self {
+        self.search_backfill_page_size = events;
+        self
+    }
+
+    #[must_use]
+    pub const fn with_search_backfill_interval_ms(mut self, interval_ms: u64) -> Self {
+        self.search_backfill_interval_ms = interval_ms;
+        self
+    }
+
+    #[must_use]
+    pub const fn with_max_search_undecrypted(mut self, events: usize) -> Self {
+        self.max_search_undecrypted = events;
+        self
+    }
 }
 
 impl Default for EngineConfig {
@@ -150,6 +186,10 @@ impl Default for EngineConfig {
             max_mention_context_events: 25_000,
             max_mention_candidates: 5_000,
             prepare_to_send: true,
+            search_backfill_pages_per_room: 10,
+            search_backfill_page_size: 100,
+            search_backfill_interval_ms: 2_000,
+            max_search_undecrypted: 2_000,
         }
     }
 }
@@ -329,6 +369,8 @@ pub struct Engine {
     mentions: crate::mentions::service::Handle,
     mentions_service: Option<crate::mentions::service::Service>,
     mentions_updates: Option<watch::Receiver<crate::MentionsPage>>,
+    search: crate::search::Handle,
+    search_service: Option<crate::search::Service>,
 }
 
 impl Engine {
@@ -338,7 +380,7 @@ impl Engine {
         config: EngineConfig,
     ) -> Result<(Self, ClientHandle, EventStream), EngineError> {
         let session_changes = client.subscribe_to_session_changes();
-        Self::with_persistence(client, config, session_changes, None, None, None).await
+        Self::with_persistence(client, config, session_changes, None, None, None, None).await
     }
 
     /// `session_changes` is taken when the client is built, since the sdk can refresh
@@ -350,6 +392,7 @@ impl Engine {
         persistence: Option<Persistence>,
         store_lock: Option<StoreLock>,
         mentions: Option<Result<MentionsStore, crate::mentions::service::Unopened>>,
+        search: Option<Result<SearchStore, crate::Error>>,
     ) -> Result<(Self, ClientHandle, EventStream), EngineError> {
         if client.user_id().is_none() {
             return Err(EngineError::Unauthenticated);
@@ -379,6 +422,8 @@ impl Engine {
             .await
             // a missing user id is all it still refuses, and that was turned away above
             .map_err(|_| EngineError::Unauthenticated)?;
+        let (search, search_service) =
+            crate::search::Service::new(client.clone(), media.clone(), search, &config);
         let sync = Arc::new(
             SyncService::builder(client.clone())
                 .with_offline_mode()
@@ -443,6 +488,8 @@ impl Engine {
             mentions,
             mentions_service: Some(mentions_service),
             mentions_updates: Some(mentions_updates),
+            search,
+            search_service: Some(search_service),
         };
         Ok((engine, handle, events))
     }
@@ -532,6 +579,10 @@ impl Engine {
             self.tasks
                 .spawn(TaskContext::Watcher("mentions service"), service.run());
         }
+        if let Some(service) = self.search_service.take() {
+            self.tasks
+                .spawn(TaskContext::Watcher("search service"), service.run());
+        }
         if let Some(updates) = self.mentions_updates.take() {
             let events = self.port.event_sender();
             self.tasks.spawn(
@@ -855,6 +906,13 @@ impl Engine {
                         reply.send(mentions.execute(command).await);
                     });
             }
+            Command::Search(command) => {
+                let search = self.search.clone();
+                self.tasks
+                    .spawn(TaskContext::Command("search command"), async move {
+                        reply.send(search.execute(command).await);
+                    });
+            }
             Command::HistoryRepair(command) => self.route_repair(command, reply),
             Command::Media(MediaCommand::Upload(upload)) => self.upload(upload, reply),
             Command::Encryption(command) => enqueue(&self.encryption, Job { command, reply }),
@@ -1162,6 +1220,9 @@ impl Engine {
         if self.mentions_service.is_none() {
             self.mentions.stop().await;
         }
+        if self.search_service.is_none() {
+            self.search.stop().await;
+        }
         self.tasks.shutdown().await;
         self.rooms.clear();
         self.contexts.clear().await;

Modified file: src/error.rs+1

@@ -54,6 +54,7 @@ pub enum ErrorKind {

5454 RoomNotOpen,

5555 TimelineNotOpen,

5656 EventContextNotOpen,

57+ SearchNotOpen,

5758 EventNotFound,

5859 RoomBusy,

5960 QueueFull,

Modified file: src/lib.rs+6 additions@@ -10,6 +10,8 @@ mod mirror;
 mod model;
 mod normalise;
 mod repair_model;
+mod search;
+mod search_model;
 mod space_order;
 mod transport;
 mod uiaa;
@@ -79,6 +81,10 @@ pub use repair_model::{
     HistoryRepairReport, HistoryRepairStop,
 };
 pub use ruma;
+pub use search_model::{
+    RoomSearchCoverage, SearchBackfill, SearchCommand, SearchCoverage, SearchHit, SearchMatch,
+    SearchPage, SearchScope, SearchSession,
+};
 #[doc(hidden)]
 pub use transport::PublicationAck;
 pub use transport::{ClientHandle, EventStream, Pending, SpaceHierarchy};

Modified file: src/model.rs+4 additions@@ -25,6 +25,7 @@ pub enum Command {
     Timeline(TimelineCommand),
     EventContext(EventContextCommand),
     Mentions(crate::MentionsCommand),
+    Search(crate::SearchCommand),
     Media(MediaCommand),
     Encryption(EncryptionCommand),
     Notification(NotificationCommand),
@@ -697,6 +698,9 @@ pub enum CommandOutput {
     EventContextPage(EventContextPage),
     MentionsPage(crate::MentionsPage),
     MentionContext(crate::SavedMentionContext),
+    SearchSession(crate::SearchSession),
+    SearchPage(crate::SearchPage),
+    SearchCoverage(crate::SearchCoverage),
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]

Modified file: src/normalise/mod.rs+1−1

@@ -16,7 +16,7 @@ pub(crate) use image_pack::{

1616 pub(crate) use link_preview::{ROOM_PREVIEW_URLS, server_preview, url_previews_allowed};

1717 pub(crate) use media::{MediaRegistration, MediaScope, MediaSourceInfo};

1818 pub(crate) use member::{member_sender, member_summary};

19-pub(crate) use observed::normalise_observed;

19+pub(crate) use observed::{normalise_observed, poll_question};

2020 pub(crate) use room::notification_mode;

2121 pub(crate) use room::{room_summaries, space_child, stored_space_order};

2222 pub(crate) use thread::{recount_thread_unreads, stored_thread_unreads};

Modified file: src/normalise/observed.rs+1−1

@@ -219,7 +219,7 @@ fn raw_relation_refs(content: &Value) -> (Option<OwnedEventId>, Option<OwnedEven

219219 (reply_to, thread_root)

220220 }

221221

222-fn poll_question(content: &Value, event_type: &str) -> Option<String> {

222+pub(crate) fn poll_question(content: &Value, event_type: &str) -> Option<String> {

223223 let typed_question = match event_type {

224224 "m.poll.start" => serde_json::from_value::<PollStartEventContent>(content.clone())

225225 .ok()

Added file: src/search/feed.rs+289 additions@@ -0,0 +1,289 @@
+use std::collections::{HashSet, VecDeque};
+
+use ruma::events::room::message::sanitize::remove_plain_reply_fallback;
+use serde_json::Value;
+
+use super::{MessageIndex, index::Doc};
+use crate::normalise::poll_question;
+
+const REMEMBERED_REDACTIONS: usize = 10_000;
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct Edit {
+    pub(crate) id: String,
+    pub(crate) target: String,
+    pub(crate) sender: String,
+    pub(crate) ts: u64,
+    pub(crate) text: String,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) enum Observed {
+    Message {
+        key: String,
+        sender: String,
+        ts: u64,
+        text: String,
+        bundled: Option<Edit>,
+    },
+    Edit(Edit),
+    Redaction {
+        target: String,
+    },
+    Redacted {
+        key: String,
+    },
+    Encrypted {
+        key: String,
+        session_id: String,
+        ts: u64,
+    },
+    Other,
+}
+
+#[derive(Debug, Eq, PartialEq)]
+pub(crate) enum Effect {
+    Indexed(u64),
+    Removed,
+    /// an edit was redacted out from under its original, which has to be read again
+    Refetch(String),
+    Nothing,
+}
+
+/// redactions seen lately, so a stale copy of what they removed can't come back
+#[derive(Default)]
+pub(crate) struct Redactions {
+    order: VecDeque<String>,
+    seen: HashSet<String>,
+}
+
+impl Redactions {
+    fn insert(&mut self, key: String) {
+        if self.seen.insert(key.clone()) {
+            self.order.push_back(key);
+            if self.order.len() > REMEMBERED_REDACTIONS
+                && let Some(oldest) = self.order.pop_front()
+            {
+                self.seen.remove(&oldest);
+            }
+        }
+    }
+
+    fn contains(&self, key: &str) -> bool {
+        self.seen.contains(key)
+    }
+}
+
+fn text_of(event_type: &str, content: &Value) -> Option<String> {
+    let text = match event_type {
+        "m.room.message" => {
+            let msgtype = content.get("msgtype")?.as_str()?;
+            if msgtype == "m.key.verification.request" {
+                return None;
+            }
+            let mut body = content.get("body")?.as_str()?.to_owned();
+            if content.pointer("/m.relates_to/m.in_reply_to").is_some() {
+                body = remove_plain_reply_fallback(&body).to_owned();
+            }
+            match content.get("filename").and_then(Value::as_str) {
+                Some(filename) if filename != body => format!("{body}\n{filename}"),
+                _ => body,
+            }
+        }
+        "m.sticker" => content.get("body")?.as_str()?.to_owned(),
+        "m.poll.start" | "org.matrix.msc3381.poll.start" => poll_question(content, event_type)?,
+        _ => return None,
+    };
+    (!text.trim().is_empty()).then_some(text)
+}
+
+fn edit_of(value: &Value, event_type: &str) -> Option<Edit> {
+    let content = value.get("content")?;
+    if content.pointer("/m.relates_to/rel_type")?.as_str()? != "m.replace" {
+        return None;
+    }
+    Some(Edit {
+        id: value.get("event_id")?.as_str()?.to_owned(),
+        target: content
+            .pointer("/m.relates_to/event_id")?
+            .as_str()?
+            .to_owned(),
+        sender: value.get("sender")?.as_str()?.to_owned(),
+        ts: value.get("origin_server_ts")?.as_u64()?,
+        text: text_of(event_type, content.get("m.new_content")?)?,
+    })
+}
+
+pub(crate) fn observe(value: &Value) -> Observed {
+    observe_inner(value).unwrap_or(Observed::Other)
+}
+
+fn observe_inner(value: &Value) -> Option<Observed> {
+    let event_type = value.get("type")?.as_str()?;
+    let key = value.get("event_id")?.as_str()?.to_owned();
+    if event_type == "m.room.redaction" {
+        let target = value
+            .get("redacts")
+            .or_else(|| value.pointer("/content/redacts"))?
+            .as_str()?
+            .to_owned();
+        return Some(Observed::Redaction { target });
+    }
+    if value.pointer("/unsigned/redacted_because").is_some() {
+        return Some(Observed::Redacted { key });
+    }
+    let ts = value.get("origin_server_ts")?.as_u64()?;
+    if event_type == "m.room.encrypted" {
+        let session_id = value.pointer("/content/session_id")?.as_str()?.to_owned();
+        return Some(Observed::Encrypted {
+            key,
+            session_id,
+            ts,
+        });
+    }
+    if value
+        .pointer("/content/m.relates_to/rel_type")
+        .and_then(Value::as_str)
+        == Some("m.replace")
+    {
+        return edit_of(value, event_type).map(Observed::Edit);
+    }
+    let content = value.get("content")?;
+    let text = text_of(event_type, content)?;
+    let sender = value.get("sender")?.as_str()?.to_owned();
+    let bundled = value
+        .pointer("/unsigned/m.relations/m.replace")
+        .and_then(|edit| edit_of(edit, event_type))
+        .filter(|edit| edit.target == key);
+    Some(Observed::Message {
+        key,
+        sender,
+        ts,
+        text,
+        bundled,
+    })
+}
+
+fn newer(ts: u64, id: &str, than: &Doc) -> bool {
+    (ts, id) > (than.content_ts, than.content.as_str())
+}
+
+/// what `observed` does to `room`'s part of the index. edits land on their original under
+/// its key, the latest by its own sender wins, and one arriving first holds the key
+/// provisionally until the original turns up
+pub(crate) fn apply(
+    index: &mut dyn MessageIndex,
+    redactions: &mut Redactions,
+    room: &str,
+    observed: Observed,
+) -> Effect {
+    match observed {
+        Observed::Message {
+            key,
+            sender,
+            ts,
+            text,
+            bundled,
+        } => {
+            if redactions.contains(&key) {
+                return Effect::Nothing;
+            }
+            let mut doc = Doc {
+                key: key.clone(),
+                room: room.to_owned(),
+                sender: sender.clone(),
+                ts,
+                content: key.clone(),
+                content_ts: ts,
+                text,
+                provisional: false,
+            };
+            if let Some(edit) =
+                bundled.filter(|edit| edit.sender == sender && !redactions.contains(&edit.id))
+            {
+                doc.content = edit.id;
+                doc.content_ts = edit.ts;
+                doc.text = edit.text;
+            }
+            match index.get(&key) {
+                Some(existing) if !existing.provisional => {
+                    if !newer(doc.content_ts, &doc.content, &existing) {
+                        return Effect::Nothing;
+                    }
+                }
+                Some(existing) => {
+                    if existing.sender == sender && !newer(doc.content_ts, &doc.content, &existing)
+                    {
+                        doc.content = existing.content;
+                        doc.content_ts = existing.content_ts;
+                        doc.text = existing.text;
+                    }
+                }
+                None => {}
+            }
+            index.put(doc);
+            Effect::Indexed(ts)
+        }
+        Observed::Edit(edit) => {
+            if redactions.contains(&edit.target) || redactions.contains(&edit.id) {
+                return Effect::Nothing;
+            }
+            let doc = match index.get(&edit.target) {
+                Some(existing) => {
+                    if existing.sender != edit.sender || !newer(edit.ts, &edit.id, &existing) {
+                        return Effect::Nothing;
+                    }
+                    Doc {
+                        content: edit.id,
+                        content_ts: edit.ts,
+                        text: edit.text,
+                        ..existing
+                    }
+                }
+                None => Doc {
+                    key: edit.target,
+                    room: room.to_owned(),
+                    sender: edit.sender,
+                    ts: edit.ts,
+                    content: edit.id,
+                    content_ts: edit.ts,
+                    text: edit.text,
+                    provisional: true,
+                },
+            };
+            let ts = doc.ts;
+            index.put(doc);
+            Effect::Indexed(ts)
+        }
+        Observed::Redaction { target } => {
+            redactions.insert(target.clone());
+            if index.remove(&target) {
+                return Effect::Removed;
+            }
+            match index.by_content(&target) {
+                Some(doc) => {
+                    index.remove(&doc.key);
+                    if doc.provisional {
+                        Effect::Removed
+                    } else {
+                        Effect::Refetch(doc.key)
+                    }
+                }
+                None => Effect::Nothing,
+            }
+        }
+        Observed::Redacted { key } => {
+            redactions.insert(key.clone());
+            if index.remove(&key) {
+                Effect::Removed
+            } else {
+                Effect::Nothing
+            }
+        }
+        Observed::Encrypted { .. } | Observed::Other => Effect::Nothing,
+    }
+}
+
+#[cfg(test)]
+#[path = "feed_tests.rs"]
+mod tests;

Added file: src/search/feed_tests.rs+187 additions@@ -0,0 +1,187 @@
+use serde_json::json;
+
+use super::*;
+use crate::search::index::KeywordIndex;
+
+const ROOM: &str = "!room:x";
+
+fn message(id: &str, sender: &str, ts: u64, body: &str) -> Value {
+    json!({
+        "type": "m.room.message",
+        "event_id": id,
+        "sender": sender,
+        "origin_server_ts": ts,
+        "content": { "msgtype": "m.text", "body": body },
+    })
+}
+
+fn edit(id: &str, sender: &str, ts: u64, target: &str, body: &str) -> Value {
+    json!({
+        "type": "m.room.message",
+        "event_id": id,
+        "sender": sender,
+        "origin_server_ts": ts,
+        "content": {
+            "msgtype": "m.text",
+            "body": format!("* {body}"),
+            "m.new_content": { "msgtype": "m.text", "body": body },
+            "m.relates_to": { "rel_type": "m.replace", "event_id": target },
+        },
+    })
+}
+
+fn redaction(id: &str, target: &str) -> Value {
+    json!({
+        "type": "m.room.redaction",
+        "event_id": id,
+        "sender": "@a:x",
+        "origin_server_ts": 99,
+        "redacts": target,
+        "content": {},
+    })
+}
+
+fn feed(index: &mut KeywordIndex, redactions: &mut Redactions, value: Value) -> Effect {
+    apply(index, redactions, ROOM, observe(&value))
+}
+
+fn keys(index: &mut KeywordIndex, query: &str) -> Vec<String> {
+    let (hits, _) = index.search(query, None, None, 10).unwrap();
+    hits.into_iter().map(|hit| hit.key).collect()
+}
+
+/// backfill walks newest first, so an edit routinely lands before what it edits
+#[test]
+fn an_edit_before_its_original_keeps_the_edit_and_takes_the_original_timestamp() {
+    let mut index = KeywordIndex::new(false);
+    let mut redactions = Redactions::default();
+    feed(
+        &mut index,
+        &mut redactions,
+        edit("$e", "@a:x", 20, "$o", "fixed words"),
+    );
+    let provisional = index.get("$o").unwrap();
+    assert!(provisional.provisional);
+    assert_eq!(provisional.ts, 20);
+    feed(
+        &mut index,
+        &mut redactions,
+        message("$o", "@a:x", 10, "typo wrds"),
+    );
+    let doc = index.get("$o").unwrap();
+    assert!(!doc.provisional);
+    assert_eq!((doc.ts, doc.content.as_str()), (10, "$e"));
+    assert_eq!(keys(&mut index, "fixed"), vec!["$o"]);
+    assert!(keys(&mut index, "wrds").is_empty());
+}
+
+#[test]
+fn edits_from_someone_else_and_older_edits_lose() {
+    let mut index = KeywordIndex::new(false);
+    let mut redactions = Redactions::default();
+    feed(
+        &mut index,
+        &mut redactions,
+        message("$o", "@a:x", 10, "original"),
+    );
+    feed(
+        &mut index,
+        &mut redactions,
+        edit("$e2", "@a:x", 30, "$o", "newest"),
+    );
+    assert_eq!(
+        feed(
+            &mut index,
+            &mut redactions,
+            edit("$e1", "@a:x", 20, "$o", "older")
+        ),
+        Effect::Nothing
+    );
+    assert_eq!(
+        feed(
+            &mut index,
+            &mut redactions,
+            edit("$h", "@b:x", 40, "$o", "hijack")
+        ),
+        Effect::Nothing
+    );
+    assert_eq!(keys(&mut index, "newest"), vec!["$o"]);
+    assert!(keys(&mut index, "hijack").is_empty());
+    assert!(keys(&mut index, "original").is_empty());
+}
+
+#[test]
+fn a_provisional_edit_from_someone_else_gives_way_to_the_original() {
+    let mut index = KeywordIndex::new(false);
+    let mut redactions = Redactions::default();
+    feed(
+        &mut index,
+        &mut redactions,
+        edit("$h", "@b:x", 20, "$o", "hijack"),
+    );
+    feed(
+        &mut index,
+        &mut redactions,
+        message("$o", "@a:x", 10, "genuine"),
+    );
+    assert_eq!(keys(&mut index, "genuine"), vec!["$o"]);
+    assert!(keys(&mut index, "hijack").is_empty());
+}
+
+#[test]
+fn a_redaction_removes_the_message_and_a_stale_copy_cannot_bring_it_back() {
+    let mut index = KeywordIndex::new(false);
+    let mut redactions = Redactions::default();
+    feed(
+        &mut index,
+        &mut redactions,
+        message("$o", "@a:x", 10, "secret"),
+    );
+    assert_eq!(
+        feed(&mut index, &mut redactions, redaction("$r", "$o")),
+        Effect::Removed
+    );
+    feed(
+        &mut index,
+        &mut redactions,
+        message("$o", "@a:x", 10, "secret"),
+    );
+    assert!(keys(&mut index, "secret").is_empty());
+}
+
+#[test]
+fn redacting_the_latest_edit_asks_for_the_original_again() {
+    let mut index = KeywordIndex::new(false);
+    let mut redactions = Redactions::default();
+    feed(
+        &mut index,
+        &mut redactions,
+        message("$o", "@a:x", 10, "first"),
+    );
+    feed(
+        &mut index,
+        &mut redactions,
+        edit("$e", "@a:x", 20, "$o", "second"),
+    );
+    assert_eq!(
+        feed(&mut index, &mut redactions, redaction("$r", "$e")),
+        Effect::Refetch("$o".to_owned())
+    );
+    let mut original = message("$o", "@a:x", 10, "first");
+    original["unsigned"] =
+        json!({ "m.relations": { "m.replace": edit("$e", "@a:x", 20, "$o", "second") } });
+    feed(&mut index, &mut redactions, original);
+    assert_eq!(keys(&mut index, "first"), vec!["$o"]);
+    assert!(keys(&mut index, "second").is_empty());
+}
+
+#[test]
+fn reply_fallbacks_are_not_indexed() {
+    let mut index = KeywordIndex::new(false);
+    let mut redactions = Redactions::default();
+    let mut reply = message("$r", "@a:x", 10, "> <@b:x> quoted stuff\n\nactual answer");
+    reply["content"]["m.relates_to"] = json!({ "m.in_reply_to": { "event_id": "$q" } });
+    feed(&mut index, &mut redactions, reply);
+    assert_eq!(keys(&mut index, "answer"), vec!["$r"]);
+    assert!(keys(&mut index, "quoted").is_empty());
+}

Added file: src/search/index.rs+826 additions@@ -0,0 +1,826 @@
+use std::{collections::HashMap, path::PathBuf, sync::Arc};
+
+use sha2::{Digest, Sha256};
+use tantivy::{
+    DocAddress, DocId, DocSet, Index, IndexSettings, ReloadPolicy, Score, Searcher, SegmentReader,
+    TantivyDocument, Term,
+    collector::{Collector, SegmentCollector},
+    directory::{Directory, OwnedBytes, RamDirectory},
+    fastfield::{AliveBitSet, write_alive_bitset},
+    indexer::{SingleSegmentIndexWriter, merge_filtered_segments},
+    query::{BooleanQuery, Occur, Query, QueryParser, TermSetQuery},
+    schema::{
+        FAST, Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions,
+        Value,
+    },
+    tokenizer::{
+        AsciiFoldingFilter, LowerCaser, RemoveLongFilter, SimpleTokenizer, TextAnalyzer,
+        TokenStream,
+    },
+};
+use tantivy_common::BitSet;
+
+use super::MessageIndex;
+use crate::{Error, ErrorKind};
+
+const TOKENIZER: &str = "macltok";
+const WRITER_BUDGET: usize = 4_000_000;
+/// segments of a size class merge once there are this many of them
+const MERGE_FACTOR: usize = 6;
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct Doc {
+    pub(crate) key: String,
+    pub(crate) room: String,
+    pub(crate) sender: String,
+    /// the original's, or the edit's while the original hasn't turned up
+    pub(crate) ts: u64,
+    /// the event whose text is indexed, the original or its latest edit
+    pub(crate) content: String,
+    pub(crate) content_ts: u64,
+    /// only kept for provisional docs, read back empty otherwise
+    pub(crate) text: String,
+    /// an edit seen before its original
+    pub(crate) provisional: bool,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) struct Hit {
+    pub(crate) key: String,
+    pub(crate) room: String,
+    pub(crate) sender: String,
+    pub(crate) content: String,
+    pub(crate) ts: u64,
+    pub(crate) tie: u64,
+}
+
+struct SegmentRecord {
+    id: u64,
+    del_gen: u64,
+}
+
+#[derive(Default)]
+pub(crate) struct Changes {
+    pub(crate) puts: Vec<(String, Vec<u8>)>,
+    pub(crate) deletes: Vec<String>,
+    files: Vec<u64>,
+    dels: Vec<(u64, u64)>,
+    removed: Vec<u64>,
+}
+
+#[derive(Clone, Copy)]
+struct Fields {
+    key: Field,
+    room: Field,
+    sender: Field,
+    content: Field,
+    text: Field,
+    ts: Field,
+    tie: Field,
+    content_ts: Field,
+    pending: Field,
+}
+
+struct Segment {
+    id: u64,
+    raw: RamDirectory,
+    index: Index,
+    searcher: Searcher,
+    alive: Arc<BitSet>,
+    max_doc: u32,
+    del_gen: u64,
+    persisted: bool,
+    deletes_dirty: bool,
+}
+
+impl Segment {
+    fn live(&self) -> usize {
+        self.alive.len()
+    }
+}
+
+pub(crate) struct KeywordIndex {
+    schema: Schema,
+    fields: Fields,
+    settings: IndexSettings,
+    parser: QueryParser,
+    segments: Vec<Segment>,
+    buffer: HashMap<String, Doc>,
+    next_id: u64,
+    persistent: bool,
+    /// merged away since the last commit, with the delete generation they were persisted at
+    removed: Vec<(u64, bool, u64)>,
+}
+
+pub(crate) fn analyzer() -> TextAnalyzer {
+    TextAnalyzer::builder(SimpleTokenizer::default())
+        .filter(RemoveLongFilter::limit(40))
+        .filter(LowerCaser)
+        .filter(AsciiFoldingFilter)
+        .build()
+}
+
+/// the byte ranges in `text` whose analysed token is one of `terms`
+pub(crate) fn match_ranges(text: &str, terms: &[String]) -> Vec<(usize, usize)> {
+    if terms.is_empty() {
+        return Vec::new();
+    }
+    let mut analyzer = analyzer();
+    let mut stream = analyzer.token_stream(text);
+    let mut ranges = Vec::new();
+    while stream.advance() {
+        let token = stream.token();
+        if terms.iter().any(|term| *term == token.text) {
+            ranges.push((token.offset_from, token.offset_to));
+        }
+    }
+    ranges
+}
+
+fn schema() -> (Schema, Fields) {
+    let mut builder = Schema::builder();
+    let text_options = TextOptions::default().set_indexing_options(
+        TextFieldIndexing::default()
+            .set_tokenizer(TOKENIZER)
+            .set_index_option(IndexRecordOption::WithFreqsAndPositions),
+    );
+    let fields = Fields {
+        key: builder.add_text_field("key", STRING | STORED),
+        room: builder.add_text_field("room", STRING | STORED),
+        sender: builder.add_text_field("sender", STRING | STORED),
+        content: builder.add_text_field("content", STRING | STORED),
+        text: builder.add_text_field("text", text_options),
+        ts: builder.add_u64_field("ts", FAST | STORED),
+        tie: builder.add_u64_field("tie", FAST),
+        content_ts: builder.add_u64_field("content_ts", STORED),
+        pending: builder.add_text_field("pending", STORED),
+    };
+    (builder.build(), fields)
+}
+
+fn tie(key: &str) -> u64 {
+    let digest = Sha256::digest(key.as_bytes());
+    u64::from_le_bytes(digest[..8].try_into().expect("a sha256 digest is 32 bytes"))
+}
+
+fn settings() -> IndexSettings {
+    // the dedicated compressor is a thread, and wasm32-unknown-unknown has none
+    IndexSettings {
+        docstore_compress_dedicated_thread: false,
+        ..IndexSettings::default()
+    }
+}
+
+fn tantivy_error(
+    context: &'static str,
+    error: impl std::error::Error + Send + Sync + 'static,
+) -> Error {
+    Error::with_source(ErrorKind::Store, context, error)
+}
+
+fn open_segment(id: u64, raw: RamDirectory, max_doc_hint: Option<u32>) -> Result<Segment, Error> {
+    let index = Index::open(raw.clone())
+        .map_err(|error| tantivy_error("opening a search segment", error))?;
+    index.tokenizers().register(TOKENIZER, analyzer());
+    let reader = index
+        .reader_builder()
+        .reload_policy(ReloadPolicy::Manual)
+        .try_into()
+        .map_err(|error| tantivy_error("reading a search segment", error))?;
+    let searcher = reader.searcher();
+    let max_doc = searcher
+        .segment_readers()
+        .iter()
+        .map(SegmentReader::max_doc)
+        .sum();
+    if searcher.segment_readers().len() > 1 || max_doc_hint.is_some_and(|hint| hint != max_doc) {
+        return Err(Error::new(
+            ErrorKind::Store,
+            "a search segment has an unexpected shape",
+        ));
+    }
+    Ok(Segment {
+        id,
+        raw,
+        index,
+        searcher,
+        alive: Arc::new(BitSet::with_max_value_and_full(max_doc)),
+        max_doc,
+        del_gen: 0,
+        persisted: false,
+        deletes_dirty: false,
+    })
+}
+
+fn pack(files: &[(String, Vec<u8>)]) -> Vec<u8> {
+    let mut packed = Vec::with_capacity(
+        files
+            .iter()
+            .map(|(name, bytes)| name.len() + bytes.len() + 12)
+            .sum(),
+    );
+    packed.extend_from_slice(&(files.len() as u32).to_le_bytes());
+    for (name, bytes) in files {
+        packed.extend_from_slice(&(name.len() as u32).to_le_bytes());
+        packed.extend_from_slice(name.as_bytes());
+        packed.extend_from_slice(&(bytes.len() as u64).to_le_bytes());
+        packed.extend_from_slice(bytes);
+    }
+    packed
+}
+
+fn unpack(mut bytes: &[u8]) -> Option<Vec<(String, Vec<u8>)>> {
+    fn take<'a>(bytes: &mut &'a [u8], len: usize) -> Option<&'a [u8]> {
+        if bytes.len() < len {
+            return None;
+        }
+        let (head, tail) = bytes.split_at(len);
+        *bytes = tail;
+        Some(head)
+    }
+    let count = u32::from_le_bytes(take(&mut bytes, 4)?.try_into().ok()?);
+    let mut files = Vec::new();
+    for _ in 0..count {
+        let name_len = u32::from_le_bytes(take(&mut bytes, 4)?.try_into().ok()?) as usize;
+        let name = std::str::from_utf8(take(&mut bytes, name_len)?)
+            .ok()?
+            .to_owned();
+        let len =
+            usize::try_from(u64::from_le_bytes(take(&mut bytes, 8)?.try_into().ok()?)).ok()?;
+        files.push((name, take(&mut bytes, len)?.to_vec()));
+    }
+    bytes.is_empty().then_some(files)
+}
+
+pub(crate) fn segment_key(id: u64) -> String {
+    format!("seg/{id}")
+}
+
+pub(crate) fn deletes_key(id: u64, generation: u64) -> String {
+    format!("del/{id}/{generation}")
+}
+
+impl KeywordIndex {
+    pub(crate) fn new(persistent: bool) -> Self {
+        let (schema, fields) = schema();
+        let parser_index = Index::create_in_ram(schema.clone());
+        parser_index.tokenizers().register(TOKENIZER, analyzer());
+        let mut parser = QueryParser::for_index(&parser_index, vec![fields.text]);
+        parser.set_conjunction_by_default();
+        Self {
+            schema,
+            fields,
+            settings: settings(),
+            parser,
+            segments: Vec::new(),
+            buffer: HashMap::new(),
+            next_id: 1,
+            persistent,
+            removed: Vec::new(),
+        }
+    }
+
+    /// `blob` hands back what an earlier commit put, any of it missing or unreadable fails the load
+    pub(crate) fn load(
+        manifest: &serde_json::Value,
+        mut blob: impl FnMut(&str) -> Option<Vec<u8>>,
+    ) -> Result<Self, Error> {
+        let invalid = || Error::new(ErrorKind::Store, "the search manifest is unreadable");
+        let records = manifest["segments"]
+            .as_array()
+            .ok_or_else(invalid)?
+            .iter()
+            .map(|record| {
+                Some(SegmentRecord {
+                    id: record.get(0)?.as_u64()?,
+                    del_gen: record.get(1)?.as_u64()?,
+                })
+            })
+            .collect::<Option<Vec<_>>>()
+            .ok_or_else(invalid)?;
+        let next_id = manifest["next"].as_u64().ok_or_else(invalid)?;
+        let mut index = Self::new(true);
+        let missing = || {
+            Error::new(
+                ErrorKind::Store,
+                "a search segment is missing or unreadable",
+            )
+        };
+        for record in &records {
+            let files = blob(&segment_key(record.id))
+                .and_then(|bytes| unpack(&bytes))
+                .ok_or_else(missing)?;
+            let raw = RamDirectory::create();
+            for (name, bytes) in files {
+                raw.atomic_write(&PathBuf::from(name), &bytes)
+                    .map_err(|error| tantivy_error("loading a search segment", error))?;
+            }
+            let mut segment = open_segment(record.id, raw, None)?;
+            if segment.index.schema() != index.schema {
+                return Err(Error::new(
+                    ErrorKind::Store,
+                    "a search segment has another schema",
+                ));
+            }
+            if record.del_gen > 0 {
+                let bytes = blob(&deletes_key(record.id, record.del_gen)).ok_or_else(missing)?;
+                let alive = AliveBitSet::open(OwnedBytes::new(bytes));
+                let mut bits = BitSet::with_max_value(segment.max_doc);
+                for doc in alive.iter_alive().filter(|doc| *doc < segment.max_doc) {
+                    bits.insert(doc);
+                }
+                segment.alive = Arc::new(bits);
+            }
+            segment.del_gen = record.del_gen;
+            segment.persisted = true;
+            index.segments.push(segment);
+        }
+        index.next_id = next_id.max(
+            records
+                .iter()
+                .map(|record| record.id + 1)
+                .max()
+                .unwrap_or(1),
+        );
+        Ok(index)
+    }
+
+    fn records(&self) -> Vec<SegmentRecord> {
+        self.segments
+            .iter()
+            .filter(|segment| segment.persisted)
+            .map(|segment| SegmentRecord {
+                id: segment.id,
+                del_gen: segment.del_gen,
+            })
+            .collect()
+    }
+
+    fn document(&self, doc: &Doc) -> TantivyDocument {
+        let fields = self.fields;
+        let mut document = TantivyDocument::default();
+        document.add_text(fields.key, &doc.key);
+        document.add_text(fields.room, &doc.room);
+        document.add_text(fields.sender, &doc.sender);
+        document.add_text(fields.content, &doc.content);
+        document.add_text(fields.text, &doc.text);
+        document.add_u64(fields.ts, doc.ts);
+        document.add_u64(fields.tie, tie(&doc.key));
+        document.add_u64(fields.content_ts, doc.content_ts);
+        if doc.provisional {
+            document.add_text(fields.pending, &doc.text);
+        }
+        document
+    }
+
+    fn stored(&self, document: &TantivyDocument) -> Option<Doc> {
+        let fields = self.fields;
+        let text = |field| {
+            document
+                .get_first(field)
+                .and_then(|value| value.as_str())
+                .map(str::to_owned)
+        };
+        let number = |field| document.get_first(field).and_then(|value| value.as_u64());
+        let pending = text(fields.pending);
+        Some(Doc {
+            key: text(fields.key)?,
+            room: text(fields.room)?,
+            sender: text(fields.sender)?,
+            ts: number(fields.ts)?,
+            content: text(fields.content)?,
+            content_ts: number(fields.content_ts)?,
+            provisional: pending.is_some(),
+            text: pending.unwrap_or_default(),
+        })
+    }
+
+    /// the live docs in `segment` whose `field` holds `value`
+    fn find(&self, segment: &Segment, field: Field, value: &str) -> Vec<DocId> {
+        let term = Term::from_field_text(field, value);
+        let mut found = Vec::new();
+        for reader in segment.searcher.segment_readers() {
+            let Ok(inverted) = reader.inverted_index(field) else {
+                continue;
+            };
+            let Ok(Some(mut postings)) = inverted.read_postings(&term, IndexRecordOption::Basic)
+            else {
+                continue;
+            };
+            let mut doc = postings.doc();
+            while doc != tantivy::TERMINATED {
+                if segment.alive.contains(doc) {
+                    found.push(doc);
+                }
+                doc = postings.advance();
+            }
+        }
+        found
+    }
+
+    fn lookup(&self, field: Field, value: &str) -> Option<Doc> {
+        self.segments.iter().rev().find_map(|segment| {
+            let doc = *self.find(segment, field, value).first()?;
+            let document: TantivyDocument = segment.searcher.doc(DocAddress::new(0, doc)).ok()?;
+            self.stored(&document)
+        })
+    }
+
+    fn flush_buffer(&mut self) -> Result<(), Error> {
+        if self.buffer.is_empty() {
+            return Ok(());
+        }
+        let raw = RamDirectory::create();
+        let index = Index::create(raw.clone(), self.schema.clone(), self.settings.clone())
+            .map_err(|error| tantivy_error("creating a search segment", error))?;
+        index.tokenizers().register(TOKENIZER, analyzer());
+        let mut writer = SingleSegmentIndexWriter::new(index, WRITER_BUDGET)
+            .map_err(|error| tantivy_error("writing a search segment", error))?;
+        let mut docs: Vec<_> = self.buffer.values().collect();
+        docs.sort_by_key(|doc| doc.ts);
+        for doc in docs {
+            writer
+                .add_document(self.document(doc))
+                .map_err(|error| tantivy_error("writing a search segment", error))?;
+        }
+        writer
+            .finalize()
+            .map_err(|error| tantivy_error("finishing a search segment", error))?;
+        let segment = open_segment(self.next_id, raw, Some(self.buffer.len() as u32))?;
+        self.next_id += 1;
+        self.segments.push(segment);
+        self.buffer.clear();
+        Ok(())
+    }
+
+    fn class(live: usize) -> u32 {
+        live.max(1).ilog(MERGE_FACTOR)
+    }
+
+    /// merges a size class once it fills up, and rewrites a segment that's mostly deleted
+    fn compact(&mut self) -> Result<(), Error> {
+        self.segments.retain(|segment| {
+            let empty = segment.live() == 0;
+            if empty && self.persistent {
+                self.removed
+                    .push((segment.id, segment.persisted, segment.del_gen));
+            }
+            !empty
+        });
+        loop {
+            let mut classes: HashMap<u32, Vec<usize>> = HashMap::new();
+            for (position, segment) in self.segments.iter().enumerate() {
+                classes
+                    .entry(Self::class(segment.live()))
+                    .or_default()
+                    .push(position);
+            }
+            let full = classes
+                .into_values()
+                .find(|positions| positions.len() >= MERGE_FACTOR);
+            let sparse = self
+                .segments
+                .iter()
+                .position(|segment| {
+                    segment.max_doc >= 1_000 && segment.live() * 2 < segment.max_doc as usize
+                })
+                .map(|position| vec![position]);
+            let Some(positions) = full.or(sparse) else {
+                return Ok(());
+            };
+            self.merge(&positions)?;
+        }
+    }
+
+    fn merge(&mut self, positions: &[usize]) -> Result<(), Error> {
+        let mut inputs = Vec::new();
+        let mut alive = Vec::new();
+        for position in positions {
+            let segment = &self.segments[*position];
+            let searchable = segment
+                .index
+                .searchable_segments()
+                .map_err(|error| tantivy_error("merging search segments", error))?;
+            let mut bytes = Vec::new();
+            write_alive_bitset(&segment.alive, &mut bytes)
+                .map_err(|error| tantivy_error("merging search segments", error))?;
+            for tantivy_segment in searchable {
+                inputs.push(tantivy_segment);
+                alive.push(Some(AliveBitSet::open(OwnedBytes::new(bytes.clone()))));
+            }
+        }
+        let raw = RamDirectory::create();
+        merge_filtered_segments(&inputs, self.settings.clone(), alive, raw.clone())
+            .map_err(|error| tantivy_error("merging search segments", error))?;
+        let merged = open_segment(self.next_id, raw, None)?;
+        self.next_id += 1;
+        let mut position = 0;
+        self.segments.retain(|segment| {
+            let merging = positions.contains(&position);
+            position += 1;
+            if merging && self.persistent {
+                self.removed
+                    .push((segment.id, segment.persisted, segment.del_gen));
+            }
+            !merging
+        });
+        self.segments.push(merged);
+        Ok(())
+    }
+
+    fn segment_files(segment: &Segment) -> Result<Vec<u8>, Error> {
+        let mut names: Vec<PathBuf> = segment
+            .index
+            .searchable_segment_metas()
+            .map_err(|error| tantivy_error("reading a search segment", error))?
+            .iter()
+            .flat_map(|meta| meta.list_files())
+            .collect();
+        names.push(PathBuf::from("meta.json"));
+        let mut files = Vec::new();
+        for name in names {
+            if !segment.raw.exists(&name).unwrap_or(false) {
+                continue;
+            }
+            let bytes = segment
+                .raw
+                .open_read(&name)
+                .map_err(|error| tantivy_error("reading a search segment", error))?
+                .read_bytes()
+                .map_err(|error| tantivy_error("reading a search segment", error))?;
+            files.push((
+                name.to_string_lossy().into_owned(),
+                bytes.as_slice().to_vec(),
+            ));
+        }
+        Ok(pack(&files))
+    }
+}
+
+impl MessageIndex for KeywordIndex {
+    fn get(&self, key: &str) -> Option<Doc> {
+        if let Some(doc) = self.buffer.get(key) {
+            return Some(doc.clone());
+        }
+        self.lookup(self.fields.key, key)
+    }
+
+    fn by_content(&self, content: &str) -> Option<Doc> {
+        if let Some(doc) = self.buffer.values().find(|doc| doc.content == content) {
+            return Some(doc.clone());
+        }
+        self.lookup(self.fields.content, content)
+    }
+
+    fn put(&mut self, doc: Doc) {
+        self.remove(&doc.key);
+        self.buffer.insert(doc.key.clone(), doc);
+    }
+
+    fn remove(&mut self, key: &str) -> bool {
+        let mut removed = self.buffer.remove(key).is_some();
+        let key_field = self.fields.key;
+        for position in 0..self.segments.len() {
+            let docs = self.find(&self.segments[position], key_field, key);
+            if docs.is_empty() {
+                continue;
+            }
+            let segment = &mut self.segments[position];
+            let alive = Arc::make_mut(&mut segment.alive);
+            for doc in docs {
+                alive.remove(doc);
+            }
+            segment.deletes_dirty = true;
+            removed = true;
+        }
+        removed
+    }
+
+    fn buffered(&self) -> usize {
+        self.buffer.len()
+    }
+
+    fn search(
+        &mut self,
+        query: &str,
+        rooms: Option<&[String]>,
+        before: Option<(u64, u64)>,
+        limit: usize,
+    ) -> Result<(Vec<Hit>, Vec<String>), Error> {
+        self.flush_buffer()?;
+        let (text_query, _) = self.parser.parse_query_lenient(query);
+        let mut terms = Vec::new();
+        text_query.query_terms(&mut |term, _| {
+            if term.field() == self.fields.text
+                && let Some(text) = term.value().as_str()
+            {
+                terms.push(text.to_owned());
+            }
+        });
+        let query: Box<dyn Query> = match rooms {
+            None => text_query,
+            Some(rooms) => Box::new(BooleanQuery::new(vec![
+                (Occur::Must, text_query),
+                (
+                    Occur::Must,
+                    Box::new(TermSetQuery::new(
+                        rooms
+                            .iter()
+                            .map(|room| Term::from_field_text(self.fields.room, room)),
+                    )),
+                ),
+            ])),
+        };
+        let mut found = Vec::new();
+        for segment in &self.segments {
+            let collector = Newest {
+                alive: segment.alive.clone(),
+                before,
+                limit,
+            };
+            let newest = segment
+                .searcher
+                .search(&*query, &collector)
+                .map_err(|error| tantivy_error("searching messages", error))?;
+            for (ts, tie, doc) in newest {
+                found.push((ts, tie, segment.id, doc));
+            }
+        }
+        found.sort_by(|a, b| (b.0, b.1).cmp(&(a.0, a.1)));
+        found.truncate(limit);
+        let mut hits = Vec::new();
+        for (ts, tie, id, doc) in found {
+            let segment = self
+                .segments
+                .iter()
+                .find(|segment| segment.id == id)
+                .expect("the segment was just searched");
+            let document: TantivyDocument = segment
+                .searcher
+                .doc(doc)
+                .map_err(|error| tantivy_error("reading a search hit", error))?;
+            let Some(stored) = self.stored(&document) else {
+                continue;
+            };
+            hits.push(Hit {
+                key: stored.key,
+                room: stored.room,
+                sender: stored.sender,
+                content: stored.content,
+                ts,
+                tie,
+            });
+        }
+        terms.sort();
+        terms.dedup();
+        Ok((hits, terms))
+    }
+
+    fn maintain(&mut self) -> Result<(), Error> {
+        self.flush_buffer()?;
+        self.compact()
+    }
+
+    fn changes(&mut self) -> Result<Changes, Error> {
+        self.maintain()?;
+        let mut changes = Changes::default();
+        for (id, persisted, del_gen) in &self.removed {
+            if *persisted {
+                changes.deletes.push(segment_key(*id));
+            }
+            if *del_gen > 0 {
+                changes.deletes.push(deletes_key(*id, *del_gen));
+            }
+            changes.removed.push(*id);
+        }
+        for segment in &self.segments {
+            if !segment.persisted {
+                changes
+                    .puts
+                    .push((segment_key(segment.id), Self::segment_files(segment)?));
+                changes.files.push(segment.id);
+            }
+            if segment.deletes_dirty && segment.live() < segment.max_doc as usize {
+                let generation = segment.del_gen + 1;
+                let mut bytes = Vec::new();
+                write_alive_bitset(&segment.alive, &mut bytes)
+                    .map_err(|error| tantivy_error("saving search deletes", error))?;
+                changes
+                    .puts
+                    .push((deletes_key(segment.id, generation), bytes));
+                if segment.del_gen > 0 {
+                    changes
+                        .deletes
+                        .push(deletes_key(segment.id, segment.del_gen));
+                }
+                changes.dels.push((segment.id, generation));
+            }
+        }
+        Ok(changes)
+    }
+
+    fn committed(&mut self, changes: &Changes) {
+        self.removed
+            .retain(|(id, ..)| !changes.removed.contains(id));
+        for segment in &mut self.segments {
+            if changes.files.contains(&segment.id) {
+                segment.persisted = true;
+            }
+            if let Some((_, generation)) = changes.dels.iter().find(|(id, _)| *id == segment.id) {
+                segment.del_gen = *generation;
+                segment.deletes_dirty = false;
+            }
+        }
+    }
+
+    fn manifest(&self) -> serde_json::Value {
+        let segments: Vec<_> = self
+            .records()
+            .into_iter()
+            .map(|record| serde_json::json!([record.id, record.del_gen]))
+            .collect();
+        serde_json::json!({ "segments": segments, "next": self.next_id })
+    }
+}
+
+/// the `limit` newest live docs older than `before`, by timestamp and then key hash
+struct Newest {
+    alive: Arc<BitSet>,
+    before: Option<(u64, u64)>,
+    limit: usize,
+}
+
+struct NewestSegment {
+    alive: Arc<BitSet>,
+    before: Option<(u64, u64)>,
+    limit: usize,
+    ts: tantivy::columnar::Column<u64>,
+    tie: tantivy::columnar::Column<u64>,
+    segment: u32,
+    top: std::collections::BinaryHeap<std::cmp::Reverse<(u64, u64, DocAddress)>>,
+}
+
+impl Collector for Newest {
+    type Fruit = Vec<(u64, u64, DocAddress)>;
+    type Child = NewestSegment;
+
+    fn for_segment(&self, segment: u32, reader: &SegmentReader) -> tantivy::Result<NewestSegment> {
+        Ok(NewestSegment {
+            alive: self.alive.clone(),
+            before: self.before,
+            limit: self.limit,
+            ts: reader.fast_fields().u64("ts")?,
+            tie: reader.fast_fields().u64("tie")?,
+            segment,
+            top: std::collections::BinaryHeap::new(),
+        })
+    }
+
+    fn requires_scoring(&self) -> bool {
+        false
+    }
+
+    fn merge_fruits(&self, fruits: Vec<Self::Fruit>) -> tantivy::Result<Self::Fruit> {
+        let mut all: Vec<_> = fruits.into_iter().flatten().collect();
+        all.sort_by(|a, b| (b.0, b.1).cmp(&(a.0, a.1)));
+        all.truncate(self.limit);
+        Ok(all)
+    }
+}
+
+impl SegmentCollector for NewestSegment {
+    type Fruit = Vec<(u64, u64, DocAddress)>;
+
+    fn collect(&mut self, doc: DocId, _: Score) {
+        if !self.alive.contains(doc) {
+            return;
+        }
+        let (Some(ts), Some(tie)) = (self.ts.first(doc), self.tie.first(doc)) else {
+            return;
+        };
+        if self.before.is_some_and(|before| (ts, tie) >= before) {
+            return;
+        }
+        let entry = std::cmp::Reverse((ts, tie, DocAddress::new(self.segment, doc)));
+        if self.top.len() < self.limit {
+            self.top.push(entry);
+        } else if self.top.peek().is_some_and(|smallest| entry < *smallest) {
+            self.top.pop();
+            self.top.push(entry);
+        }
+    }
+
+    fn harvest(self) -> Self::Fruit {
+        let mut top: Vec<_> = self
+            .top
+            .into_iter()
+            .map(|std::cmp::Reverse(entry)| entry)
+            .collect();
+        top.sort_by(|a, b| (b.0, b.1).cmp(&(a.0, a.1)));
+        top
+    }
+}
+
+#[cfg(test)]
+#[path = "index_tests.rs"]
+mod tests;

Added file: src/search/index_tests.rs+132 additions@@ -0,0 +1,132 @@
+use std::collections::BTreeMap;
+
+use super::*;
+
+fn doc(key: &str, room: &str, ts: u64, text: &str) -> Doc {
+    Doc {
+        key: key.to_owned(),
+        room: room.to_owned(),
+        sender: "@a:x".to_owned(),
+        ts,
+        content: key.to_owned(),
+        content_ts: ts,
+        text: text.to_owned(),
+        provisional: false,
+    }
+}
+
+fn keys(hits: &[Hit]) -> Vec<&str> {
+    hits.iter().map(|hit| hit.key.as_str()).collect()
+}
+
+fn commit(index: &mut KeywordIndex, blobs: &mut BTreeMap<String, Vec<u8>>) -> serde_json::Value {
+    let changes = index.changes().unwrap();
+    for key in &changes.deletes {
+        blobs.remove(key);
+    }
+    for (key, value) in &changes.puts {
+        blobs.insert(key.clone(), value.clone());
+    }
+    index.committed(&changes);
+    index.manifest()
+}
+
+#[test]
+fn pages_run_newest_first_without_repeats_across_segments() {
+    let mut index = KeywordIndex::new(false);
+    for ts in 0..30 {
+        index.put(doc(&format!("$e{ts}"), "!r:x", ts, "hello there"));
+        if ts % 7 == 0 {
+            index.maintain().unwrap();
+        }
+    }
+    let mut seen = Vec::new();
+    let mut before = None;
+    loop {
+        let (hits, _) = index.search("hello", None, before, 8).unwrap();
+        let Some(last) = hits.last() else { break };
+        before = Some((last.ts, last.tie));
+        seen.extend(hits.iter().map(|hit| hit.ts));
+    }
+    assert_eq!(seen, (0..30).rev().collect::<Vec<_>>());
+}
+
+#[test]
+fn a_replaced_doc_is_found_once_by_its_new_text() {
+    let mut index = KeywordIndex::new(false);
+    index.put(doc("$a", "!r:x", 1, "before"));
+    index.maintain().unwrap();
+    index.put(doc("$a", "!r:x", 1, "after"));
+    assert!(index.search("before", None, None, 10).unwrap().0.is_empty());
+    assert_eq!(
+        keys(&index.search("after", None, None, 10).unwrap().0),
+        vec!["$a"]
+    );
+}
+
+#[test]
+fn rooms_filter_the_results() {
+    let mut index = KeywordIndex::new(false);
+    index.put(doc("$a", "!one:x", 1, "word"));
+    index.put(doc("$b", "!two:x", 2, "word"));
+    let rooms = vec!["!one:x".to_owned()];
+    assert_eq!(
+        keys(&index.search("word", Some(&rooms), None, 10).unwrap().0),
+        vec!["$a"]
+    );
+    assert!(
+        index
+            .search("word", Some(&[]), None, 10)
+            .unwrap()
+            .0
+            .is_empty()
+    );
+}
+
+/// a lost delete brings a redacted message back, and a lost segment loses messages
+#[test]
+fn deletes_and_merges_survive_a_reload() {
+    let mut blobs = BTreeMap::new();
+    let mut index = KeywordIndex::new(true);
+    for ts in 0..(MERGE_FACTOR as u64 + 2) {
+        index.put(doc(&format!("$e{ts}"), "!r:x", ts, "kept"));
+        commit(&mut index, &mut blobs);
+    }
+    index.put(doc("$gone", "!r:x", 100, "kept"));
+    commit(&mut index, &mut blobs);
+    index.remove("$gone");
+    index.remove("$e0");
+    let manifest = commit(&mut index, &mut blobs);
+    let mut reloaded = KeywordIndex::load(&manifest, |key| blobs.get(key).cloned()).unwrap();
+    let (hits, _) = reloaded.search("kept", None, None, 50).unwrap();
+    let mut found = keys(&hits);
+    found.sort();
+    let mut expected: Vec<String> = (1..(MERGE_FACTOR as u64 + 2))
+        .map(|ts| format!("$e{ts}"))
+        .collect();
+    expected.sort();
+    assert_eq!(
+        found,
+        expected.iter().map(String::as_str).collect::<Vec<_>>()
+    );
+    let referenced: Vec<_> = blobs.keys().filter(|key| key.starts_with("seg/")).collect();
+    assert!(
+        referenced.len() < MERGE_FACTOR,
+        "merged segments were not dropped: {referenced:?}"
+    );
+}
+
+#[test]
+fn match_ranges_follow_the_analyser() {
+    assert_eq!(
+        match_ranges("Café, CAFE and cafés", &["cafe".to_owned()]),
+        vec![(0, 5), (7, 11)]
+    );
+}
+
+#[test]
+fn packed_segments_reject_truncation() {
+    let packed = pack(&[("a".to_owned(), vec![1, 2, 3])]);
+    assert!(unpack(&packed).is_some());
+    assert!(unpack(&packed[..packed.len() - 1]).is_none());
+}

Added file: src/search/mod.rs+1234 additions@@ -0,0 +1,1234 @@
+mod feed;
+#[cfg_attr(not(any(feature = "sqlite", feature = "indexeddb")), allow(dead_code))]
+mod index;
+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]
+pub(crate) mod store;
+
+use std::{
+    collections::{BTreeMap, HashMap, HashSet, VecDeque},
+    sync::Arc,
+    time::Duration,
+};
+
+use futures_util::{
+    StreamExt,
+    future::join_all,
+    stream::{AbortHandle, FuturesUnordered, SelectAll, abortable},
+};
+use matrix_sdk::{
+    Client, Room, RoomState, SendOutsideWasm,
+    deserialized_responses::TimelineEvent as SdkTimelineEvent,
+    event_cache::{RoomEventCacheSubscriber, RoomEventCacheUpdate},
+    room::MessagesOptions,
+};
+use matrix_sdk_base::RoomInfoNotableUpdateReasons;
+use matrix_sdk_ui::eyeball_im::VectorDiff;
+use ruma::{
+    OwnedRoomId, OwnedUserId, UInt,
+    events::{
+        ignored_user_list::IgnoredUserListEventContent,
+        room::encrypted::OriginalSyncRoomEncryptedEvent,
+    },
+    serde::Raw,
+};
+use serde_json::Value;
+use tokio::sync::{Mutex, broadcast, mpsc, oneshot};
+
+use self::{
+    feed::{Effect, Observed, Redactions},
+    index::{Changes, Doc, Hit, KeywordIndex},
+};
+use crate::{
+    CommandOutput, EngineConfig, Error, ErrorKind, RoomSearchCoverage, SearchBackfill,
+    SearchCommand, SearchCoverage, SearchHit, SearchMatch, SearchPage, SearchScope, SearchSession,
+    engine::{MediaService, SearchStore, context, executor::EngineBoxFuture},
+    normalise::{MediaRegistration, MediaScope},
+};
+
+const MAX_LIMIT: u16 = 50;
+/// a consumer that never closes its searches still holds at most this many
+const MAX_SEARCHES: usize = 8;
+const COMMIT_AFTER_MS: u64 = 5_000;
+const COMMIT_BUFFERED: usize = 2_000;
+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]
+const COMMIT_RETRY_MS: u64 = 60_000;
+const IDLE_SUBSCRIPTION_MS: u64 = 5 * 60_000;
+const BACKFILL_RETRY_MS: u64 = 60_000;
+const BACKFILL_RETRY_MAX_MS: u64 = 15 * 60_000;
+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]
+const SCHEMA: u64 = 1;
+
+#[cfg(not(target_family = "wasm"))]
+type BoxStream<T> = futures_util::stream::BoxStream<'static, T>;
+#[cfg(target_family = "wasm")]
+type BoxStream<T> = futures_util::stream::LocalBoxStream<'static, T>;
+
+/// a local index of message text. keyword search over tantivy is the only one today, an
+/// embedding index can stand beside it later without the public api noticing
+#[cfg_attr(not(any(feature = "sqlite", feature = "indexeddb")), allow(dead_code))]
+pub(crate) trait MessageIndex: SendOutsideWasm {
+    fn get(&self, key: &str) -> Option<Doc>;
+    fn by_content(&self, content: &str) -> Option<Doc>;
+    /// replaces whatever `doc.key` held
+    fn put(&mut self, doc: Doc);
+    fn remove(&mut self, key: &str) -> bool;
+    fn buffered(&self) -> usize;
+    /// newest first, strictly older than `before`. also hands back the analysed query terms
+    fn search(
+        &mut self,
+        query: &str,
+        rooms: Option<&[String]>,
+        before: Option<(u64, u64)>,
+        limit: usize,
+    ) -> Result<(Vec<Hit>, Vec<String>), Error>;
+    fn maintain(&mut self) -> Result<(), Error>;
+    fn changes(&mut self) -> Result<Changes, Error>;
+    fn committed(&mut self, changes: &Changes);
+    fn manifest(&self) -> Value;
+}
+
+#[derive(Clone, Debug, Default)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+struct RoomProgress {
+    oldest: Option<u64>,
+    newest: Option<u64>,
+    /// where the next backfill page starts, `None` before the first one
+    tail: Option<String>,
+    started: bool,
+    start_reached: bool,
+}
+
+#[derive(Clone, Debug)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+struct Pending {
+    room: String,
+    session_id: String,
+    ts: u64,
+    raw: String,
+}
+
+#[derive(Clone, Copy)]
+struct Budget {
+    pages_per_room: u16,
+    page_size: u16,
+    interval_ms: u64,
+    max_pending: usize,
+}
+
+impl Budget {
+    fn from_config(config: &EngineConfig) -> Self {
+        Self {
+            pages_per_room: config.search_backfill_pages_per_room,
+            page_size: config.search_backfill_page_size.max(1),
+            interval_ms: config.search_backfill_interval_ms,
+            max_pending: config.max_search_undecrypted,
+        }
+    }
+}
+
+struct RawPage {
+    hits: Vec<Hit>,
+    terms: Vec<String>,
+    more: bool,
+    coverage: SearchCoverage,
+}
+
+enum Request {
+    Search {
+        query: String,
+        rooms: Option<Vec<String>>,
+        before: Option<(u64, u64)>,
+        limit: usize,
+        coverage: Vec<OwnedRoomId>,
+        reply: oneshot::Sender<Result<RawPage, Error>>,
+    },
+    Coverage(Vec<OwnedRoomId>, oneshot::Sender<SearchCoverage>),
+    /// a hit that turned out redacted when its preview was read
+    Forget(String),
+    Stop(oneshot::Sender<()>),
+}
+
+#[derive(Clone)]
+pub(crate) struct Handle {
+    requests: mpsc::Sender<Request>,
+    client: Client,
+    media: Arc<MediaService>,
+    sessions: Arc<Sessions>,
+}
+
+enum Work {
+    Backfill {
+        room: OwnedRoomId,
+        head: bool,
+        result: Result<matrix_sdk::room::Messages, Error>,
+    },
+    Decrypted {
+        key: String,
+        room: OwnedRoomId,
+        value: Option<Value>,
+    },
+    Refetched {
+        room: OwnedRoomId,
+        value: Option<Value>,
+    },
+}
+
+struct Failure {
+    message: String,
+    retry_at_ms: u64,
+}
+
+pub(crate) struct Service {
+    client: Client,
+    index: Box<dyn MessageIndex>,
+    store: Option<Result<SearchStore, Error>>,
+    rooms: BTreeMap<String, RoomProgress>,
+    pending: BTreeMap<String, Pending>,
+    pending_dirty: bool,
+    redactions: Redactions,
+    dirty_since: Option<u64>,
+    commit_after: u64,
+    budget: Budget,
+    requests: mpsc::Receiver<Request>,
+    subscriptions: HashMap<OwnedRoomId, (AbortHandle, u64)>,
+    updates: SelectAll<BoxStream<(OwnedRoomId, Option<Vec<SdkTimelineEvent>>)>>,
+    work: FuturesUnordered<EngineBoxFuture<'static, Work>>,
+    pages: HashMap<OwnedRoomId, u16>,
+    heads: VecDeque<OwnedRoomId>,
+    headed: HashSet<OwnedRoomId>,
+    head_tokens: HashMap<OwnedRoomId, String>,
+    backfilling: Option<OwnedRoomId>,
+    next_backfill: u64,
+    retry_delay: u64,
+    failures: HashMap<OwnedRoomId, Failure>,
+    turn: usize,
+}
+
+fn now() -> u64 {
+    ruma::MilliSecondsSinceUnixEpoch::now().get().into()
+}
+
+fn value_of(event: &SdkTimelineEvent) -> Option<Value> {
+    serde_json::from_str(event.raw().json().get()).ok()
+}
+
+fn encode_cursor((ts, tie): (u64, u64)) -> String {
+    format!("{ts:x}.{tie:x}")
+}
+
+fn decode_cursor(cursor: &str) -> Option<(u64, u64)> {
+    let (ts, tie) = cursor.split_once('.')?;
+    Some((
+        u64::from_str_radix(ts, 16).ok()?,
+        u64::from_str_radix(tie, 16).ok()?,
+    ))
+}
+
+impl Handle {
+    pub(crate) async fn execute(&self, command: SearchCommand) -> Result<CommandOutput, Error> {
+        match command {
+            SearchCommand::Search {
+                query,
+                scope,
+                limit,
+            } => {
+                let (page, media) = self.search(&query, &scope, None, limit).await?;
+                let search_id = self.sessions.open(&self.media, query, scope, media).await;
+                Ok(CommandOutput::SearchSession(SearchSession { search_id, page }))
+            }
+            SearchCommand::More {
+                search_id,
+                cursor,
+                limit,
+            } => {
+                let (query, scope) = self.sessions.get(&search_id).await?;
+                let (page, media) = self.search(&query, &scope, Some(cursor.as_str()), limit).await?;
+                self.sessions.register(&self.media, &search_id, media).await?;
+                Ok(CommandOutput::SearchPage(page))
+            }
+            SearchCommand::Close { search_id } => {
+                self.sessions.close(&self.media, &search_id).await?;
+                Ok(CommandOutput::Done)
+            }
+            SearchCommand::Coverage(scope) => {
+                let (reply, response) = oneshot::channel();
+                self.requests
+                    .send(Request::Coverage(self.scope_rooms(&scope), reply))
+                    .await
+                    .map_err(|_| Error::interrupted())?;
+                Ok(CommandOutput::SearchCoverage(
+                    response.await.map_err(|_| Error::interrupted())?,
+                ))
+            }
+        }
+    }
+
+    /// the media service is cleared beside this, so nothing is released one by one
+    pub(crate) async fn stop(&self) {
+        let (reply, response) = oneshot::channel();
+        if self.requests.send(Request::Stop(reply)).await.is_ok() {
+            let _ = response.await;
+        }
+        self.sessions.open.lock().await.clear();
+    }
+
+    fn scope_rooms(&self, scope: &SearchScope) -> Vec<OwnedRoomId> {
+        match scope {
+            SearchScope::Room(room_id) => vec![room_id.clone()],
+            SearchScope::Rooms(room_ids) => room_ids.clone(),
+            SearchScope::All => self
+                .client
+                .joined_rooms()
+                .into_iter()
+                .map(|room| room.room_id().to_owned())
+                .collect(),
+        }
+    }
+
+    async fn search(
+        &self,
+        query: &str,
+        scope: &SearchScope,
+        cursor: Option<&str>,
+        limit: u16,
+    ) -> Result<(SearchPage, Vec<MediaRegistration>), Error> {
+        if limit == 0 || limit > MAX_LIMIT {
+            return Err(Error::new(
+                ErrorKind::Invalid,
+                "search limit must be between 1 and 50",
+            ));
+        }
+        if query.trim().is_empty() {
+            return Err(Error::new(ErrorKind::Invalid, "search query is empty"));
+        }
+        let before = match cursor {
+            Some(cursor) => Some(
+                decode_cursor(cursor)
+                    .ok_or_else(|| Error::new(ErrorKind::Invalid, "search cursor is invalid"))?,
+            ),
+            None => None,
+        };
+        let rooms = self.scope_rooms(scope);
+        let filter = Some(rooms.iter().map(ToString::to_string).collect());
+        let (reply, response) = oneshot::channel();
+        self.requests
+            .send(Request::Search {
+                query: query.to_owned(),
+                rooms: filter,
+                before,
+                limit: usize::from(limit),
+                coverage: rooms,
+                reply,
+            })
+            .await
+            .map_err(|_| Error::interrupted())?;
+        let raw = response.await.map_err(|_| Error::interrupted())??;
+        let next_cursor = raw
+            .more
+            .then(|| raw.hits.last().map(|hit| encode_cursor((hit.ts, hit.tie))))
+            .flatten();
+        let ignored = self.ignored_users().await;
+        let current_user_id = self.client.user_id().map(ToOwned::to_owned);
+        let previews = join_all(
+            raw.hits
+                .into_iter()
+                .filter(|hit| !ignored.iter().any(|user| user.as_str() == hit.sender))
+                .map(|hit| self.preview(hit, current_user_id.clone(), &raw.terms)),
+        )
+        .await;
+        let mut hits = Vec::new();
+        let mut media = Vec::new();
+        for preview in previews {
+            match preview {
+                Preview::Shown(hit, registrations) => {
+                    hits.push(hit);
+                    media.extend(registrations);
+                }
+                Preview::Redacted(key) => {
+                    let _ = self.requests.send(Request::Forget(key)).await;
+                }
+                Preview::Unavailable => {}
+            }
+        }
+        let page = SearchPage {
+            hits,
+            next_cursor,
+            coverage: raw.coverage,
+        };
+        Ok((page, media))
+    }
+
+    async fn ignored_users(&self) -> Vec<OwnedUserId> {
+        self.client
+            .account()
+            .account_data::<IgnoredUserListEventContent>()
+            .await
+            .ok()
+            .flatten()
+            .and_then(|raw| raw.deserialize().ok())
+            .map(|content| content.ignored_users.into_keys().collect())
+            .unwrap_or_default()
+    }
+
+    async fn preview(
+        &self,
+        hit: Hit,
+        current_user_id: Option<OwnedUserId>,
+        terms: &[String],
+    ) -> Preview {
+        let (Ok(room_id), Ok(event_id)) = (
+            OwnedRoomId::try_from(hit.room.as_str()),
+            ruma::OwnedEventId::try_from(hit.key.as_str()),
+        ) else {
+            return Preview::Unavailable;
+        };
+        let Some(room) = self.client.get_room(&room_id) else {
+            return Preview::Unavailable;
+        };
+        let Some((original, highlight)) = load(&room, &hit.key).await else {
+            return Preview::Unavailable;
+        };
+        if original.pointer("/unsigned/redacted_because").is_some() {
+            return Preview::Redacted(hit.key);
+        }
+        let mut values = vec![original];
+        if hit.content != hit.key
+            && let Some((edit, _)) = load(&room, &hit.content).await
+        {
+            values.push(edit);
+        }
+        let Some((event, registrations)) =
+            context::preview(&room, current_user_id.as_deref(), &values, highlight).await
+        else {
+            return Preview::Unavailable;
+        };
+        let text = match &event.content {
+            crate::TimelineContent::Message(message) => Some(message.body.as_str()),
+            crate::TimelineContent::Sticker { body, .. } => Some(body.as_str()),
+            crate::TimelineContent::Poll { question } => Some(question.as_str()),
+            _ => None,
+        };
+        let matches = text
+            .map(|text| index::match_ranges(text, terms))
+            .unwrap_or_default()
+            .into_iter()
+            .filter_map(|(start, end)| {
+                Some(SearchMatch {
+                    start: u32::try_from(start).ok()?,
+                    end: u32::try_from(end).ok()?,
+                })
+            })
+            .collect();
+        Preview::Shown(
+            SearchHit {
+                room_id,
+                event_id,
+                timestamp_ms: event.timestamp_ms,
+                event,
+                matches,
+            },
+            registrations,
+        )
+    }
+}
+
+enum Preview {
+    Shown(SearchHit, Vec<MediaRegistration>),
+    Redacted(String),
+    Unavailable,
+}
+
+/// open searches, oldest first, each owning the media of the hits it handed out
+#[derive(Default)]
+struct Sessions {
+    open: Mutex<VecDeque<Session>>,
+}
+
+struct Session {
+    id: String,
+    query: String,
+    scope: SearchScope,
+    media: MediaScope,
+}
+
+impl Sessions {
+    async fn open(
+        &self,
+        media: &MediaService,
+        query: String,
+        scope: SearchScope,
+        registrations: Vec<MediaRegistration>,
+    ) -> String {
+        let media_scope = media.context_scope();
+        let id = format!("search-{}", media_scope.nonce());
+        let mut open = self.open.lock().await;
+        media.register(&media_scope, registrations).await;
+        open.push_back(Session {
+            id: id.clone(),
+            query,
+            scope,
+            media: media_scope,
+        });
+        while open.len() > MAX_SEARCHES {
+            if let Some(oldest) = open.pop_front() {
+                media.release(&oldest.media).await;
+            }
+        }
+        id
+    }
+
+    async fn get(&self, id: &str) -> Result<(String, SearchScope), Error> {
+        let open = self.open.lock().await;
+        let session = open.iter().find(|session| session.id == id).ok_or_else(not_open)?;
+        Ok((session.query.clone(), session.scope.clone()))
+    }
+
+    /// under the lock, so a page that lands after its close registers nothing
+    async fn register(
+        &self,
+        media: &MediaService,
+        id: &str,
+        registrations: Vec<MediaRegistration>,
+    ) -> Result<(), Error> {
+        let open = self.open.lock().await;
+        let session = open.iter().find(|session| session.id == id).ok_or_else(not_open)?;
+        media.register(&session.media, registrations).await;
+        Ok(())
+    }
+
+    async fn close(&self, media: &MediaService, id: &str) -> Result<(), Error> {
+        let mut open = self.open.lock().await;
+        let index = open
+            .iter()
+            .position(|session| session.id == id)
+            .ok_or_else(not_open)?;
+        if let Some(session) = open.remove(index) {
+            media.release(&session.media).await;
+        }
+        Ok(())
+    }
+}
+
+fn not_open() -> Error {
+    Error::new(ErrorKind::SearchNotOpen, "search is closed or was never opened")
+}
+
+async fn load(room: &Room, event_id: &str) -> Option<(Value, bool)> {
+    let event_id = ruma::EventId::parse(event_id).ok()?;
+    let event = room
+        .load_or_fetch_event(&event_id, None)
+        .await
+        .inspect_err(|error| tracing::debug!(%event_id, %error, "loading a search hit failed"))
+        .ok()?;
+    let highlight = event
+        .push_actions()
+        .is_some_and(|actions| actions.iter().any(ruma::push::Action::is_highlight));
+    Some((value_of(&event)?, highlight))
+}
+
+#[cfg(any(feature = "sqlite", feature = "indexeddb"))]
+fn decode_meta(
+    store: &SearchStore,
+    blobs: &BTreeMap<String, Vec<u8>>,
+) -> Option<(
+    Value,
+    BTreeMap<String, RoomProgress>,
+    BTreeMap<String, Pending>,
+)> {
+    let meta: Value = serde_json::from_slice(&store.open_blob("meta", blobs.get("meta")?)?).ok()?;
+    if meta["schema"].as_u64() != Some(SCHEMA) {
+        return None;
+    }
+    let rooms = serde_json::from_value(meta["rooms"].clone()).ok()?;
+    let pending = match blobs.get("utd") {
+        Some(sealed) => serde_json::from_slice(&store.open_blob("utd", sealed)?).ok()?,
+        None => BTreeMap::new(),
+    };
+    Some((meta["index"].clone(), rooms, pending))
+}
+
+impl Service {
+    pub(crate) fn new(
+        client: Client,
+        media: Arc<MediaService>,
+        store: Option<Result<SearchStore, Error>>,
+        config: &EngineConfig,
+    ) -> (Handle, Self) {
+        let (sender, requests) = mpsc::channel(config.command_capacity.max(1));
+        let service = Self {
+            client: client.clone(),
+            index: Box::new(KeywordIndex::new(store.as_ref().is_some_and(Result::is_ok))),
+            store,
+            rooms: BTreeMap::new(),
+            pending: BTreeMap::new(),
+            pending_dirty: false,
+            redactions: Redactions::default(),
+            dirty_since: None,
+            commit_after: 0,
+            budget: Budget::from_config(config),
+            requests,
+            subscriptions: HashMap::new(),
+            updates: SelectAll::new(),
+            work: FuturesUnordered::new(),
+            pages: HashMap::new(),
+            heads: VecDeque::new(),
+            headed: HashSet::new(),
+            head_tokens: HashMap::new(),
+            backfilling: None,
+            next_backfill: 0,
+            retry_delay: BACKFILL_RETRY_MS,
+            failures: HashMap::new(),
+            turn: 0,
+        };
+        (
+            Handle {
+                requests: sender,
+                client,
+                media,
+                sessions: Arc::default(),
+            },
+            service,
+        )
+    }
+
+    fn persistent(&self) -> bool {
+        matches!(self.store, Some(Ok(_)))
+    }
+
+    #[cfg(any(feature = "sqlite", feature = "indexeddb"))]
+    async fn load(&mut self) {
+        let Some(Ok(store)) = &self.store else {
+            if let Some(Err(error)) = &self.store {
+                tracing::error!(%error, "search store unavailable, the index lives in memory");
+            }
+            return;
+        };
+        let blobs = match store.load().await {
+            Ok(blobs) => blobs,
+            Err(error) => {
+                tracing::error!(%error, "reading the search store failed, the index lives in memory");
+                self.store = Some(Err(error));
+                self.index = Box::new(KeywordIndex::new(false));
+                return;
+            }
+        };
+        if blobs.is_empty() {
+            return;
+        }
+        let restored = decode_meta(store, &blobs).and_then(|(manifest, rooms, pending)| {
+            let index = KeywordIndex::load(&manifest, |key| {
+                blobs
+                    .get(key)
+                    .and_then(|sealed| store.open_blob(key, sealed))
+            })
+            .inspect_err(|error| tracing::warn!(%error, "search segments are unreadable"))
+            .ok()?;
+            Some((index, rooms, pending))
+        });
+        drop(blobs);
+        match restored {
+            Some((index, rooms, pending)) => {
+                self.index = Box::new(index);
+                self.rooms = rooms;
+                self.pending = pending;
+            }
+            None => {
+                tracing::warn!("resetting an unreadable search index");
+                if let Err(error) = store.reset().await {
+                    tracing::error!(%error, "search store unavailable, the index lives in memory");
+                    self.store = Some(Err(error));
+                    self.index = Box::new(KeywordIndex::new(false));
+                }
+            }
+        }
+    }
+
+    #[cfg(not(any(feature = "sqlite", feature = "indexeddb")))]
+    async fn load(&mut self) {
+        if let Some(Err(error)) = &self.store {
+            tracing::error!(%error, "search store unavailable, the index lives in memory");
+        }
+    }
+
+    #[cfg(any(feature = "sqlite", feature = "indexeddb"))]
+    async fn commit(&mut self) {
+        if !self.persistent() {
+            if let Err(error) = self.index.maintain() {
+                tracing::warn!(%error, "maintaining the search index failed");
+            }
+            self.dirty_since = None;
+            return;
+        }
+        let changes = match self.index.changes() {
+            Ok(changes) => changes,
+            Err(error) => {
+                tracing::warn!(%error, "preparing a search commit failed");
+                self.commit_after = now() + COMMIT_RETRY_MS;
+                return;
+            }
+        };
+        let mut puts = changes.puts.clone();
+        let meta = serde_json::json!({
+            "schema": SCHEMA,
+            "index": self.index.manifest(),
+            "rooms": self.rooms,
+        });
+        puts.push(("meta".to_owned(), meta.to_string().into_bytes()));
+        if self.pending_dirty {
+            puts.push((
+                "utd".to_owned(),
+                serde_json::to_vec(&self.pending).unwrap_or_default(),
+            ));
+        }
+        let Some(Ok(store)) = &self.store else {
+            return;
+        };
+        match store.commit(puts, changes.deletes.clone()).await {
+            Ok(()) => {
+                self.index.committed(&changes);
+                self.pending_dirty = false;
+                self.dirty_since = None;
+            }
+            Err(error) => {
+                tracing::warn!(%error, "saving the search index failed, retrying later");
+                self.commit_after = now() + COMMIT_RETRY_MS;
+            }
+        }
+    }
+
+    #[cfg(not(any(feature = "sqlite", feature = "indexeddb")))]
+    async fn commit(&mut self) {
+        if let Err(error) = self.index.maintain() {
+            tracing::warn!(%error, "maintaining the search index failed");
+        }
+        self.dirty_since = None;
+    }
+
+    fn touch(&mut self) {
+        self.dirty_since.get_or_insert_with(now);
+    }
+
+    fn observe(&mut self, room: &OwnedRoomId, value: &Value) {
+        let observed = feed::observe(value);
+        let key = match &observed {
+            Observed::Message { key, .. }
+            | Observed::Redacted { key }
+            | Observed::Encrypted { key, .. } => Some(key.clone()),
+            Observed::Edit(edit) => Some(edit.id.clone()),
+            Observed::Redaction { .. } | Observed::Other => None,
+        };
+        if let Observed::Encrypted {
+            key,
+            session_id,
+            ts,
+        } = &observed
+        {
+            if self.index.get(key).is_none() && !self.pending.contains_key(key) {
+                self.pending.insert(
+                    key.clone(),
+                    Pending {
+                        room: room.to_string(),
+                        session_id: session_id.clone(),
+                        ts: *ts,
+                        raw: value.to_string(),
+                    },
+                );
+                while self.pending.len() > self.budget.max_pending {
+                    let Some(oldest) = self
+                        .pending
+                        .iter()
+                        .min_by_key(|(_, pending)| pending.ts)
+                        .map(|(key, _)| key.clone())
+                    else {
+                        break;
+                    };
+                    self.pending.remove(&oldest);
+                }
+                self.pending_dirty = true;
+                self.touch();
+            }
+            return;
+        }
+        if let Some(key) = key
+            && self.pending.remove(&key).is_some()
+        {
+            self.pending_dirty = true;
+            self.touch();
+        }
+        match feed::apply(
+            self.index.as_mut(),
+            &mut self.redactions,
+            room.as_str(),
+            observed,
+        ) {
+            Effect::Indexed(ts) => {
+                let progress = self.rooms.entry(room.to_string()).or_default();
+                progress.oldest = Some(progress.oldest.map_or(ts, |oldest| oldest.min(ts)));
+                progress.newest = Some(progress.newest.map_or(ts, |newest| newest.max(ts)));
+                self.touch();
+            }
+            Effect::Removed => self.touch(),
+            Effect::Refetch(key) => {
+                self.touch();
+                let room_id = room.clone();
+                let client = self.client.clone();
+                self.work.push(Box::pin(async move {
+                    let value = match client.get_room(&room_id) {
+                        Some(room) => load(&room, &key).await.map(|(value, _)| value),
+                        None => None,
+                    };
+                    Work::Refetched {
+                        room: room_id,
+                        value,
+                    }
+                }));
+            }
+            Effect::Nothing => {}
+        }
+    }
+
+    fn observe_all(&mut self, room: &OwnedRoomId, events: &[SdkTimelineEvent]) {
+        for event in events {
+            if let Some(value) = value_of(event) {
+                self.observe(room, &value);
+            }
+        }
+    }
+
+    async fn subscribe(&mut self, room_id: OwnedRoomId) {
+        if let Some((_, seen)) = self.subscriptions.get_mut(&room_id) {
+            *seen = now();
+            return;
+        }
+        let Some(room) = self.client.get_room(&room_id) else {
+            return;
+        };
+        if room.state() != RoomState::Joined {
+            return;
+        }
+        let Ok((cache, drop_handles)) = room.event_cache().await else {
+            return;
+        };
+        let Ok((initial, subscriber)) = cache.subscribe().await else {
+            return;
+        };
+        self.observe_all(&room_id, &initial);
+        let stream = updates(room_id.clone(), subscriber, drop_handles);
+        let (stream, abort) = abortable(stream);
+        self.updates.push(Box::pin(stream));
+        self.subscriptions.insert(room_id.clone(), (abort, now()));
+        if self.headed.insert(room_id.clone()) {
+            self.heads.push_back(room_id);
+        }
+    }
+
+    fn drop_idle(&mut self) {
+        let cutoff = now().saturating_sub(IDLE_SUBSCRIPTION_MS);
+        self.subscriptions.retain(|_, (abort, seen)| {
+            let keep = *seen >= cutoff;
+            if !keep {
+                abort.abort();
+            }
+            keep
+        });
+    }
+
+    fn coverage(&self, rooms: &[OwnedRoomId]) -> SearchCoverage {
+        let rooms = rooms
+            .iter()
+            .map(|room_id| {
+                let progress = self
+                    .rooms
+                    .get(room_id.as_str())
+                    .cloned()
+                    .unwrap_or_default();
+                let backfill = if progress.start_reached {
+                    SearchBackfill::Complete
+                } else if self.budget.pages_per_room == 0 {
+                    SearchBackfill::Disabled
+                } else if self.backfilling.as_ref() == Some(room_id) {
+                    SearchBackfill::Running
+                } else if let Some(Failure {
+                    message,
+                    retry_at_ms,
+                }) = self.failures.get(room_id)
+                {
+                    SearchBackfill::Failed {
+                        message: message.clone(),
+                        retry_at_ms: *retry_at_ms,
+                    }
+                } else if self.pages.get(room_id).copied().unwrap_or(0)
+                    >= self.budget.pages_per_room
+                {
+                    SearchBackfill::Paused
+                } else {
+                    SearchBackfill::Waiting
+                };
+                RoomSearchCoverage {
+                    room_id: room_id.clone(),
+                    oldest_indexed_ms: progress.oldest,
+                    newest_indexed_ms: progress.newest,
+                    start_reached: progress.start_reached,
+                    backfill,
+                }
+            })
+            .collect();
+        SearchCoverage {
+            rooms,
+            persistent: self.persistent(),
+            undecrypted: self.pending.len(),
+        }
+    }
+
+    /// head passes first, so what arrived while we were away is found before anything older
+    fn schedule_backfill(&mut self) {
+        if self.backfilling.is_some()
+            || self.budget.pages_per_room == 0
+            || now() < self.next_backfill
+        {
+            return;
+        }
+        let budget = self.budget.pages_per_room;
+        let spent = |pages: &HashMap<OwnedRoomId, u16>, room: &OwnedRoomId| {
+            pages.get(room).copied().unwrap_or(0) >= budget
+        };
+        let blocked = |failures: &HashMap<OwnedRoomId, Failure>, room: &OwnedRoomId| {
+            failures
+                .get(room)
+                .is_some_and(|failure| failure.retry_at_ms > now())
+        };
+        let mut chosen = None;
+        while let Some(room_id) = self.heads.pop_front() {
+            let started = self
+                .rooms
+                .get(room_id.as_str())
+                .is_some_and(|progress| progress.started);
+            if started && !spent(&self.pages, &room_id) && !blocked(&self.failures, &room_id) {
+                chosen = Some((room_id, true));
+                break;
+            }
+        }
+        if chosen.is_none() {
+            let mut joined: Vec<_> = self
+                .client
+                .joined_rooms()
+                .into_iter()
+                .map(|room| room.room_id().to_owned())
+                .collect();
+            joined.sort();
+            let count = joined.len();
+            for offset in 0..count {
+                let room_id = &joined[(self.turn + offset) % count];
+                let done = self
+                    .rooms
+                    .get(room_id.as_str())
+                    .is_some_and(|progress| progress.start_reached);
+                if !done && !spent(&self.pages, room_id) && !blocked(&self.failures, room_id) {
+                    self.turn = (self.turn + offset + 1) % count;
+                    chosen = Some((room_id.clone(), false));
+                    break;
+                }
+            }
+        }
+        let Some((room_id, head)) = chosen else {
+            return;
+        };
+        let Some(room) = self.client.get_room(&room_id) else {
+            return;
+        };
+        let from = if head {
+            self.head_tokens.get(&room_id).cloned()
+        } else {
+            self.rooms
+                .get(room_id.as_str())
+                .and_then(|progress| progress.tail.clone())
+        };
+        *self.pages.entry(room_id.clone()).or_default() += 1;
+        self.backfilling = Some(room_id.clone());
+        let mut options = MessagesOptions::backward();
+        options.from = from;
+        options.limit = UInt::from(self.budget.page_size);
+        self.work.push(Box::pin(async move {
+            let result = room
+                .messages(options)
+                .await
+                .map_err(|error| Error::from_sdk("backfilling search history failed", error));
+            Work::Backfill {
+                room: room_id,
+                head,
+                result,
+            }
+        }));
+    }
+
+    fn backfilled(
+        &mut self,
+        room_id: OwnedRoomId,
+        head: bool,
+        result: Result<matrix_sdk::room::Messages, Error>,
+    ) {
+        self.backfilling = None;
+        let messages = match result {
+            Ok(messages) => messages,
+            Err(error) => {
+                let retry_at_ms = now() + self.retry_delay;
+                self.next_backfill = retry_at_ms;
+                self.retry_delay = (self.retry_delay * 2).min(BACKFILL_RETRY_MAX_MS);
+                self.failures.insert(
+                    room_id,
+                    Failure {
+                        message: error.message.clone(),
+                        retry_at_ms,
+                    },
+                );
+                return;
+            }
+        };
+        self.retry_delay = BACKFILL_RETRY_MS;
+        self.failures.remove(&room_id);
+        self.next_backfill = now() + self.budget.interval_ms;
+        let mut overlapped = false;
+        for event in &messages.chunk {
+            let Some(value) = value_of(event) else {
+                continue;
+            };
+            if head
+                && let Some(key) = value.get("event_id").and_then(Value::as_str)
+                && self.index.get(key).is_some_and(|doc| !doc.provisional)
+            {
+                overlapped = true;
+            }
+            self.observe(&room_id, &value);
+        }
+        let ended = messages.end.is_none() || messages.chunk.is_empty();
+        if head {
+            match messages.end {
+                Some(end) if !overlapped && !ended => {
+                    self.head_tokens.insert(room_id.clone(), end);
+                    self.heads.push_front(room_id);
+                }
+                _ => {
+                    self.head_tokens.remove(&room_id);
+                }
+            }
+            return;
+        }
+        let progress = self.rooms.entry(room_id.to_string()).or_default();
+        progress.started = true;
+        progress.tail = messages.end;
+        if ended {
+            progress.start_reached = true;
+            progress.tail = None;
+        }
+        self.touch();
+    }
+
+    fn retry_pending(&mut self, room_id: Option<&OwnedRoomId>, session_id: Option<&str>) {
+        let due: Vec<_> = self
+            .pending
+            .iter()
+            .filter(|(_, pending)| {
+                room_id.is_none_or(|room_id| pending.room == room_id.as_str())
+                    && session_id.is_none_or(|session_id| pending.session_id == session_id)
+            })
+            .map(|(key, pending)| (key.clone(), pending.clone()))
+            .collect();
+        for (key, pending) in due {
+            let Ok(room_id) = OwnedRoomId::try_from(pending.room.as_str()) else {
+                continue;
+            };
+            let Some(room) = self.client.get_room(&room_id) else {
+                continue;
+            };
+            self.work.push(Box::pin(async move {
+                let value = async {
+                    let raw: Raw<OriginalSyncRoomEncryptedEvent> =
+                        Raw::from_json_string(pending.raw).ok()?;
+                    let event = room.decrypt_event(&raw, None).await.ok()?;
+                    value_of(&event).filter(|value| value["type"] != "m.room.encrypted")
+                }
+                .await;
+                Work::Decrypted {
+                    key,
+                    room: room_id,
+                    value,
+                }
+            }));
+        }
+    }
+
+    fn complete(&mut self, work: Work) {
+        match work {
+            Work::Backfill { room, head, result } => self.backfilled(room, head, result),
+            Work::Decrypted { key, room, value } => {
+                if let Some(value) = value {
+                    self.pending.remove(&key);
+                    self.pending_dirty = true;
+                    self.observe(&room, &value);
+                    self.touch();
+                }
+            }
+            Work::Refetched { room, value } => {
+                if let Some(value) = value {
+                    self.observe(&room, &value);
+                }
+            }
+        }
+    }
+
+    fn search(
+        &mut self,
+        query: &str,
+        rooms: Option<&[String]>,
+        before: Option<(u64, u64)>,
+        limit: usize,
+        coverage: &[OwnedRoomId],
+    ) -> Result<RawPage, Error> {
+        let (mut hits, terms) = self.index.search(query, rooms, before, limit + 1)?;
+        let more = hits.len() > limit;
+        hits.truncate(limit);
+        Ok(RawPage {
+            hits,
+            terms,
+            more,
+            coverage: self.coverage(coverage),
+        })
+    }
+
+    pub(crate) async fn run(mut self) {
+        self.load().await;
+        let client = self.client.clone();
+        let mut generic = client.event_cache().subscribe_to_room_generic_updates();
+        let mut keys = client
+            .encryption()
+            .room_keys_received_stream()
+            .await
+            .map(Box::pin);
+        let mut room_info = client.room_info_notable_update_receiver();
+        self.retry_pending(None, None);
+        let mut tick = Box::pin(matrix_sdk::sleep::sleep(Duration::from_secs(1)));
+        loop {
+            self.schedule_backfill();
+            tokio::select! {
+                biased;
+                request = self.requests.recv() => match request {
+                    Some(Request::Stop(reply)) => {
+                        if self.dirty_since.is_some() || self.index.buffered() > 0 {
+                            self.commit().await;
+                        }
+                        for (abort, _) in self.subscriptions.values() {
+                            abort.abort();
+                        }
+                        let _ = reply.send(());
+                        return;
+                    }
+                    Some(Request::Search { query, rooms, before, limit, coverage, reply }) => {
+                        let _ = reply.send(self.search(&query, rooms.as_deref(), before, limit, &coverage));
+                    }
+                    Some(Request::Coverage(rooms, reply)) => {
+                        let _ = reply.send(self.coverage(&rooms));
+                    }
+                    Some(Request::Forget(key)) => {
+                        let observed = Observed::Redacted { key };
+                        if feed::apply(self.index.as_mut(), &mut self.redactions, "", observed) == Effect::Removed {
+                            self.touch();
+                        }
+                    }
+                    None => return,
+                },
+                Some(work) = self.work.next(), if !self.work.is_empty() => self.complete(work),
+                Some((room_id, events)) = self.updates.next(), if !self.updates.is_empty() => {
+                    if let Some((_, seen)) = self.subscriptions.get_mut(&room_id) {
+                        *seen = now();
+                    }
+                    match events {
+                        Some(events) => self.observe_all(&room_id, &events),
+                        None => {
+                            if let Some((abort, _)) = self.subscriptions.remove(&room_id) {
+                                abort.abort();
+                            }
+                            self.subscribe(room_id).await;
+                        }
+                    }
+                }
+                update = generic.recv() => match update {
+                    Ok(update) => self.subscribe(update.room_id).await,
+                    Err(broadcast::error::RecvError::Lagged(_)) => {}
+                    Err(broadcast::error::RecvError::Closed) => return,
+                },
+                Some(update) = async { keys.as_mut()?.next().await }, if keys.is_some() => match update {
+                    Ok(infos) => {
+                        for info in infos {
+                            self.retry_pending(Some(&info.room_id), Some(&info.session_id));
+                        }
+                    }
+                    Err(_) => self.retry_pending(None, None),
+                },
+                update = room_info.recv() => match update {
+                    Ok(update) if update.reasons.contains(RoomInfoNotableUpdateReasons::MEMBERSHIP) => {
+                        let left = self.client.get_room(&update.room_id).is_none_or(|room| room.state() != RoomState::Joined);
+                        if left && let Some((abort, _)) = self.subscriptions.remove(&update.room_id) {
+                            abort.abort();
+                        }
+                    }
+                    Ok(_) | Err(broadcast::error::RecvError::Lagged(_)) => {}
+                    Err(broadcast::error::RecvError::Closed) => return,
+                },
+                () = &mut tick => {
+                    tick = Box::pin(matrix_sdk::sleep::sleep(Duration::from_secs(1)));
+                    self.drop_idle();
+                    let due = self.dirty_since.is_some_and(|since| now() >= since + COMMIT_AFTER_MS)
+                        || self.index.buffered() >= COMMIT_BUFFERED;
+                    if due && now() >= self.commit_after {
+                        self.commit().await;
+                    }
+                }
+            }
+        }
+    }
+}
+
+fn updates(
+    room_id: OwnedRoomId,
+    subscriber: RoomEventCacheSubscriber,
+    drop_handles: Arc<matrix_sdk::event_cache::EventCacheDropHandles>,
+) -> BoxStream<(OwnedRoomId, Option<Vec<SdkTimelineEvent>>)> {
+    Box::pin(futures_util::stream::unfold(
+        (subscriber, drop_handles),
+        move |(mut subscriber, drop_handles)| {
+            let room_id = room_id.clone();
+            async move {
+                loop {
+                    match subscriber.recv().await {
+                        Ok(RoomEventCacheUpdate::UpdateTimelineEvents(update)) => {
+                            let events = update.diffs.into_iter().flat_map(added).collect();
+                            return Some(((room_id, Some(events)), (subscriber, drop_handles)));
+                        }
+                        Ok(_) => {}
+                        Err(broadcast::error::RecvError::Lagged(_)) => {
+                            return Some(((room_id, None), (subscriber, drop_handles)));
+                        }
+                        Err(broadcast::error::RecvError::Closed) => return None,
+                    }
+                }
+            }
+        },
+    ))
+}
+
+fn added(diff: VectorDiff<SdkTimelineEvent>) -> Vec<SdkTimelineEvent> {
+    match diff {
+        VectorDiff::Append { values } | VectorDiff::Reset { values } => {
+            values.into_iter().collect()
+        }
+        VectorDiff::PushFront { value }
+        | VectorDiff::PushBack { value }
+        | VectorDiff::Insert { value, .. }
+        | VectorDiff::Set { value, .. } => vec![value],
+        VectorDiff::Clear
+        | VectorDiff::PopFront
+        | VectorDiff::PopBack
+        | VectorDiff::Remove { .. }
+        | VectorDiff::Truncate { .. } => Vec::new(),
+    }
+}

Added file: src/search/store/indexeddb.rs+167 additions@@ -0,0 +1,167 @@
+use std::{collections::BTreeMap, ops::Deref};
+
+use indexed_db_futures::{
+    Build,
+    database::Database,
+    error::{Error as IndexedDbError, OpenDbError},
+    prelude::*,
+    transaction::{Transaction, TransactionDurability, TransactionMode, TransactionOptions},
+    typed_array::{Uint8Array, Uint8ArraySlice},
+};
+
+use super::storage_error;
+use crate::Error;
+
+const BLOBS: &str = "blobs";
+
+struct DatabaseHandle(Database);
+
+impl Deref for DatabaseHandle {
+    type Target = Database;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl Drop for DatabaseHandle {
+    fn drop(&mut self) {
+        self.0.clone().close();
+    }
+}
+
+pub(super) struct Backend {
+    name: String,
+}
+
+impl Backend {
+    pub(super) async fn open(name: String) -> Result<Self, Error> {
+        open_database(&name)
+            .await
+            .map_err(|error| storage_error("opening the search database", error))?;
+        Ok(Self { name })
+    }
+
+    async fn database(&self) -> Result<DatabaseHandle, Error> {
+        open_database(&self.name)
+            .await
+            .map_err(|error| storage_error("opening the search database", error))
+    }
+
+    pub(super) async fn load(&self, keys: &[&str]) -> Result<BTreeMap<String, Vec<u8>>, Error> {
+        let database = self.database().await?;
+        let read = async {
+            let transaction = database
+                .transaction(BLOBS)
+                .with_mode(TransactionMode::Readonly)
+                .build()?;
+            let store = transaction.object_store(BLOBS)?;
+            let mut found = BTreeMap::new();
+            for key in keys {
+                if let Some(value) = store.get::<Uint8Array, _, _>(*key).await? {
+                    found.insert((*key).to_owned(), Vec::<u8>::from(value));
+                }
+            }
+            Ok::<_, IndexedDbError>(found)
+        };
+        read.await
+            .map_err(|error| storage_error("reading the search database", error))
+    }
+
+    pub(super) async fn load_all(&self) -> Result<BTreeMap<String, Vec<u8>>, Error> {
+        let database = self.database().await?;
+        let read = async {
+            let transaction = database
+                .transaction(BLOBS)
+                .with_mode(TransactionMode::Readonly)
+                .build()?;
+            let store = transaction.object_store(BLOBS)?;
+            let keys = store.get_all_keys::<String>().build()?;
+            let values = store.get_all::<Uint8Array>().build()?;
+            let keys = keys.await?.collect::<Result<Vec<_>, _>>()?;
+            let values = values.await?.collect::<Result<Vec<_>, _>>()?;
+            Ok::<_, IndexedDbError>((keys, values))
+        };
+        let (keys, values) = read
+            .await
+            .map_err(|error| storage_error("reading the search database", error))?;
+        if keys.len() != values.len() {
+            return Err(Error::new(
+                crate::ErrorKind::Store,
+                "search database keys and values disagree",
+            ));
+        }
+        Ok(keys
+            .into_iter()
+            .zip(values)
+            .map(|(key, value)| (key, Vec::<u8>::from(value)))
+            .collect())
+    }
+
+    pub(super) async fn commit(
+        &self,
+        puts: Vec<(String, Vec<u8>)>,
+        deletes: Vec<String>,
+    ) -> Result<(), Error> {
+        let database = self.database().await?;
+        let write = async {
+            let mut options = TransactionOptions::new();
+            options.set_durability(TransactionDurability::Relaxed);
+            let transaction = database
+                .transaction(BLOBS)
+                .with_mode(TransactionMode::Readwrite)
+                .with_options(options)
+                .build()?;
+            let requests = (|| -> Result<(), IndexedDbError> {
+                let store = transaction.object_store(BLOBS)?;
+                for key in deletes {
+                    store.delete(key).build()?;
+                }
+                for (key, value) in &puts {
+                    store
+                        .put(Uint8ArraySlice::new(value))
+                        .with_key(key.as_str())
+                        .build()?;
+                }
+                Ok(())
+            })();
+            if let Err(error) = requests {
+                let _ = transaction.abort().await;
+                return Err(error);
+            }
+            transaction.commit().await
+        };
+        write
+            .await
+            .map_err(|error| storage_error("writing the search database", error))
+    }
+
+    pub(super) async fn clear(&self) -> Result<(), Error> {
+        let database = self.database().await?;
+        let clear = async {
+            let transaction = database
+                .transaction(BLOBS)
+                .with_mode(TransactionMode::Readwrite)
+                .build()?;
+            transaction.object_store(BLOBS)?.clear()?.await?;
+            transaction.commit().await
+        };
+        clear
+            .await
+            .map_err(|error| storage_error("clearing the search database", error))
+    }
+}
+
+async fn open_database(name: &str) -> Result<DatabaseHandle, OpenDbError> {
+    Database::open(name)
+        .with_version(1u32)
+        .with_on_upgrade_needed(
+            |_, transaction: &Transaction<'_>| -> Result<(), IndexedDbError> {
+                transaction.db().create_object_store(BLOBS).build()?;
+                Ok(())
+            },
+        )
+        .build()?
+        .await
+        .map(DatabaseHandle)
+}

Added file: src/search/store/mod.rs+145 additions@@ -0,0 +1,145 @@
+use std::collections::BTreeMap;
+
+use matrix_sdk::{SendOutsideWasm, SyncOutsideWasm};
+use matrix_sdk_store_encryption::{EncryptedValueBase64, StoreCipher};
+use serde::{Deserialize, Serialize};
+
+use crate::{Error, ErrorKind, StoreConfig, mentions::store::Account};
+
+#[cfg(feature = "indexeddb")]
+mod indexeddb;
+#[cfg(feature = "indexeddb")]
+use indexeddb::Backend;
+#[cfg(feature = "sqlite")]
+mod sqlite;
+#[cfg(feature = "sqlite")]
+use sqlite::Backend;
+
+const CIPHER: &str = "!cipher";
+const ACCOUNT: &str = "!account";
+
+#[derive(Serialize, Deserialize)]
+struct Owner {
+    homeserver: String,
+    user_id: String,
+}
+
+/// blobs sealed with a `StoreCipher` under the store's passphrase. every blob carries its
+/// own key inside the seal, so one can't be swapped for another
+pub(crate) struct SearchStore {
+    backend: Backend,
+    cipher: StoreCipher,
+}
+
+impl std::fmt::Debug for SearchStore {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter
+            .debug_struct("SearchStore")
+            .finish_non_exhaustive()
+    }
+}
+
+impl SearchStore {
+    /// an unreadable or foreign store is emptied rather than refused
+    pub(crate) async fn open(config: &StoreConfig, account: Account) -> Result<Self, Error> {
+        let backend = Backend::open(config.search_location()).await?;
+        let owner = Owner {
+            homeserver: account.homeserver,
+            user_id: account.user_id.to_string(),
+        };
+        let mut blobs = backend.load(&[CIPHER, ACCOUNT]).await?;
+        let opened = blobs.remove(CIPHER).and_then(|exported| {
+            let cipher = StoreCipher::import(config.passphrase(), &exported).ok()?;
+            let stored: Owner =
+                serde_json::from_slice(&unseal(&cipher, ACCOUNT, &blobs.remove(ACCOUNT)?)?).ok()?;
+            (stored.homeserver == owner.homeserver && stored.user_id == owner.user_id)
+                .then_some(cipher)
+        });
+        if let Some(cipher) = opened {
+            return Ok(Self { backend, cipher });
+        }
+        let cipher = StoreCipher::new()
+            .map_err(|error| storage_error("creating the search cipher", error))?;
+        let exported = cipher
+            .export(config.passphrase())
+            .map_err(|error| storage_error("sealing the search cipher", error))?;
+        let account = serde_json::to_vec(&owner)
+            .map_err(|error| storage_error("encoding the search owner", error))?;
+        let store = Self { backend, cipher };
+        store.backend.clear().await?;
+        let sealed = store.seal(ACCOUNT, account)?;
+        store
+            .backend
+            .commit(
+                vec![(CIPHER.to_owned(), exported), (ACCOUNT.to_owned(), sealed)],
+                Vec::new(),
+            )
+            .await?;
+        Ok(store)
+    }
+
+    /// every blob but the store's own, still sealed
+    pub(crate) async fn load(&self) -> Result<BTreeMap<String, Vec<u8>>, Error> {
+        let mut blobs = self.backend.load_all().await?;
+        blobs.remove(CIPHER);
+        blobs.remove(ACCOUNT);
+        Ok(blobs)
+    }
+
+    pub(crate) fn open_blob(&self, key: &str, sealed: &[u8]) -> Option<Vec<u8>> {
+        unseal(&self.cipher, key, sealed)
+    }
+
+    pub(crate) async fn commit(
+        &self,
+        puts: Vec<(String, Vec<u8>)>,
+        deletes: Vec<String>,
+    ) -> Result<(), Error> {
+        let mut sealed = Vec::with_capacity(puts.len());
+        for (key, value) in puts {
+            let value = self.seal(&key, value)?;
+            sealed.push((key, value));
+        }
+        self.backend.commit(sealed, deletes).await
+    }
+
+    /// keeps the cipher and owner, drops everything else
+    pub(crate) async fn reset(&self) -> Result<(), Error> {
+        let keep = self.backend.load(&[CIPHER, ACCOUNT]).await?;
+        self.backend.clear().await?;
+        self.backend
+            .commit(keep.into_iter().collect(), Vec::new())
+            .await
+    }
+
+    fn seal(&self, key: &str, value: Vec<u8>) -> Result<Vec<u8>, Error> {
+        let mut plain = Vec::with_capacity(key.len() + 1 + value.len());
+        plain.extend_from_slice(key.as_bytes());
+        plain.push(0);
+        plain.extend_from_slice(&value);
+        drop(value);
+        let encrypted = self
+            .cipher
+            .encrypt_value_base64_data(plain)
+            .map_err(|error| storage_error("sealing a search blob", error))?;
+        serde_json::to_vec(&encrypted)
+            .map_err(|error| storage_error("encoding a search blob", error))
+    }
+}
+
+fn unseal(cipher: &StoreCipher, key: &str, sealed: &[u8]) -> Option<Vec<u8>> {
+    let encrypted: EncryptedValueBase64 = serde_json::from_slice(sealed).ok()?;
+    let mut plain = cipher.decrypt_value_base64_data(encrypted).ok()?;
+    let prefix = key.len() + 1;
+    if plain.len() < prefix || &plain[..key.len()] != key.as_bytes() || plain[key.len()] != 0 {
+        return None;
+    }
+    Some(plain.split_off(prefix))
+}
+
+pub(super) fn storage_error(
+    context: &'static str,
+    source: impl std::error::Error + SendOutsideWasm + SyncOutsideWasm + 'static,
+) -> Error {
+    Error::with_source(ErrorKind::Store, context, source)
+}

Added file: src/search/store/sqlite.rs+149 additions@@ -0,0 +1,149 @@
+use std::{
+    collections::BTreeMap,
+    fs,
+    path::{Path, PathBuf},
+    time::Duration,
+};
+
+use rusqlite::{Connection, TransactionBehavior, params};
+
+use super::storage_error;
+use crate::Error;
+
+pub(super) struct Backend {
+    path: PathBuf,
+}
+
+impl Backend {
+    pub(super) async fn open(path: PathBuf) -> Result<Self, Error> {
+        let setup = path.clone();
+        blocking("opening the search store", move || initialise(&setup)).await?;
+        Ok(Self { path })
+    }
+
+    pub(super) async fn load(&self, keys: &[&str]) -> Result<BTreeMap<String, Vec<u8>>, Error> {
+        let path = self.path.clone();
+        let keys: Vec<String> = keys.iter().map(|key| (*key).to_owned()).collect();
+        blocking("reading the search store", move || {
+            let connection = connect(&path)?;
+            let mut statement = connection
+                .prepare("SELECT value FROM blobs WHERE key = ?1")
+                .map_err(|error| storage_error("reading the search store", error))?;
+            let mut found = BTreeMap::new();
+            for key in keys {
+                let mut rows = statement
+                    .query([&key])
+                    .map_err(|error| storage_error("reading the search store", error))?;
+                if let Some(row) = rows
+                    .next()
+                    .map_err(|error| storage_error("reading the search store", error))?
+                {
+                    let value: Vec<u8> = row
+                        .get(0)
+                        .map_err(|error| storage_error("reading the search store", error))?;
+                    found.insert(key, value);
+                }
+            }
+            Ok(found)
+        })
+        .await
+    }
+
+    pub(super) async fn load_all(&self) -> Result<BTreeMap<String, Vec<u8>>, Error> {
+        let path = self.path.clone();
+        blocking("reading the search store", move || {
+            let connection = connect(&path)?;
+            let mut statement = connection
+                .prepare("SELECT key, value FROM blobs")
+                .map_err(|error| storage_error("reading the search store", error))?;
+            let rows = statement
+                .query_map([], |row| {
+                    Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?))
+                })
+                .map_err(|error| storage_error("reading the search store", error))?;
+            rows.collect::<Result<_, _>>()
+                .map_err(|error| storage_error("reading the search store", error))
+        })
+        .await
+    }
+
+    pub(super) async fn commit(
+        &self,
+        puts: Vec<(String, Vec<u8>)>,
+        deletes: Vec<String>,
+    ) -> Result<(), Error> {
+        let path = self.path.clone();
+        blocking("writing the search store", move || {
+            let mut connection = connect(&path)?;
+            let transaction = connection
+                .transaction_with_behavior(TransactionBehavior::Immediate)
+                .map_err(|error| storage_error("writing the search store", error))?;
+            for key in deletes {
+                transaction
+                    .execute("DELETE FROM blobs WHERE key = ?1", [key])
+                    .map_err(|error| storage_error("writing the search store", error))?;
+            }
+            for (key, value) in puts {
+                transaction
+                    .execute(
+                        "INSERT INTO blobs (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+                        params![key, value],
+                    )
+                    .map_err(|error| storage_error("writing the search store", error))?;
+            }
+            transaction
+                .commit()
+                .map_err(|error| storage_error("writing the search store", error))
+        })
+        .await
+    }
+
+    pub(super) async fn clear(&self) -> Result<(), Error> {
+        let path = self.path.clone();
+        blocking("clearing the search store", move || {
+            connect(&path)?
+                .execute("DELETE FROM blobs", [])
+                .map(drop)
+                .map_err(|error| storage_error("clearing the search store", error))
+        })
+        .await
+    }
+}
+
+async fn blocking<T: Send + 'static>(
+    context: &'static str,
+    work: impl FnOnce() -> Result<T, Error> + Send + 'static,
+) -> Result<T, Error> {
+    tokio::task::spawn_blocking(work)
+        .await
+        .map_err(|error| storage_error(context, error))?
+}
+
+fn initialise(path: &Path) -> Result<(), Error> {
+    if let Some(parent) = path
+        .parent()
+        .filter(|parent| !parent.as_os_str().is_empty())
+    {
+        fs::create_dir_all(parent)
+            .map_err(|error| storage_error("creating the search store directory", error))?;
+    }
+    connect(path)?
+        .execute(
+            "CREATE TABLE IF NOT EXISTS blobs (key TEXT PRIMARY KEY, value BLOB NOT NULL)",
+            [],
+        )
+        .map(drop)
+        .map_err(|error| storage_error("initialising the search store", error))
+}
+
+fn connect(path: &Path) -> Result<Connection, Error> {
+    let connection =
+        Connection::open(path).map_err(|error| storage_error("opening the search store", error))?;
+    connection
+        .busy_timeout(Duration::from_secs(5))
+        .map_err(|error| storage_error("configuring the search store", error))?;
+    connection
+        .execute_batch("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL")
+        .map_err(|error| storage_error("configuring the search store", error))?;
+    Ok(connection)
+}

Added file: src/search_model.rs+113 additions@@ -0,0 +1,113 @@
+use ruma::{OwnedEventId, OwnedRoomId};
+#[cfg(feature = "serde")]
+use serde::{Deserialize, Serialize};
+
+use crate::TimelineEvent;
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[non_exhaustive]
+pub enum SearchScope {
+    Room(OwnedRoomId),
+    /// a space's rooms, resolved by the host
+    Rooms(Vec<OwnedRoomId>),
+    /// every joined room
+    All,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[non_exhaustive]
+pub enum SearchCommand {
+    /// opens a search that owns the media of every hit it hands out. at most 8 stay open,
+    /// opening another closes the oldest. words must all match, `"quoted phrases"` match
+    /// in order and `-word` excludes, newest first
+    Search {
+        query: String,
+        scope: SearchScope,
+        limit: u16,
+    },
+    /// `cursor` is the previous page's `next_cursor`
+    More {
+        search_id: String,
+        cursor: String,
+        limit: u16,
+    },
+    /// frees the search's media. its hits' media ids stop resolving
+    Close { search_id: String },
+    Coverage(SearchScope),
+}
+
+/// a search's media stay resolvable until it closes, and past `EngineConfig::max_context_media`
+/// its least recently used are dropped
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct SearchSession {
+    pub search_id: String,
+    pub page: SearchPage,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct SearchPage {
+    pub hits: Vec<SearchHit>,
+    /// `None` once there's nothing older that matches
+    pub next_cursor: Option<String>,
+    pub coverage: SearchCoverage,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct SearchHit {
+    pub room_id: OwnedRoomId,
+    pub event_id: OwnedEventId,
+    pub timestamp_ms: u64,
+    /// as event context shows it, with the latest edit applied
+    pub event: TimelineEvent,
+    /// utf-8 byte ranges of matched words in the content's text, the message body, a
+    /// sticker's body or a poll's question
+    pub matches: Vec<SearchMatch>,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct SearchMatch {
+    pub start: u32,
+    pub end: u32,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct SearchCoverage {
+    pub rooms: Vec<RoomSearchCoverage>,
+    /// false when the index lives in memory and starts over with the next engine
+    pub persistent: bool,
+    /// messages that couldn't be decrypted yet, so can't be found yet
+    pub undecrypted: usize,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub struct RoomSearchCoverage {
+    pub room_id: OwnedRoomId,
+    pub oldest_indexed_ms: Option<u64>,
+    pub newest_indexed_ms: Option<u64>,
+    /// backfill walked back to the oldest event the server would hand over
+    pub start_reached: bool,
+    pub backfill: SearchBackfill,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub enum SearchBackfill {
+    Waiting,
+    Running,
+    /// this engine's page budget for the room is spent, the next one carries on
+    Paused,
+    Complete,
+    Disabled,
+    Failed {
+        message: String,
+        retry_at_ms: u64,
+    },
+}

Modified file: src/session_store.rs+3 additions−1 deletions@@ -473,12 +473,13 @@ pub(crate) fn delete_sqlite_store(path: &Path) -> std::io::Result<()> {
     use std::{fs, io::ErrorKind};
 
     // matrix-sdk-sqlite 0.18's database names, which it keeps private
-    const DATABASES: [&str; 5] = [
+    const DATABASES: [&str; 6] = [
         "matrix-sdk-state.sqlite3",
         "matrix-sdk-crypto.sqlite3",
         "matrix-sdk-event-cache.sqlite3",
         "matrix-sdk-media.sqlite3",
         "macltok-mentions.sqlite3",
+        "macltok-search.sqlite3",
     ];
     for database in DATABASES {
         for suffix in ["", "-wal", "-shm", "-journal"] {
@@ -509,6 +510,7 @@ pub(crate) async fn delete_indexeddb_store(name: &str) -> Result<(), DeleteError
         "::event_cache",
         "::media",
         "::macltok-mentions",
+        "::macltok-search",
     ] {
         delete_database(&format!("{name}{suffix}")).await?;
     }

Modified file: src/transport.rs+59 additions−1 deletions@@ -36,7 +36,8 @@ use crate::{
     NotificationCommand, OwnDevice, PackCommand, PackImageInfo, PackUsage, Patch,
     PowerLevelChanges, Profile, PushNotification, PushNotificationEvent, PushNotificationStatus,
     RoomCommand, RoomListFilter, RoomNotificationMode, RoomNotificationSettings, RoomPowerLevels,
-    RoomSummary, SavedMentionContext, SpaceChild, ThreadListCommand, TimelineCommand, TimelineId,
+    RoomSummary, SavedMentionContext, SearchCommand, SearchCoverage, SearchPage, SearchScope,
+    SearchSession, SpaceChild, ThreadListCommand, TimelineCommand, TimelineId,
     VerificationAction, VerificationTarget,
     encryption::encryption_state,
     engine::{MediaService, StoreLock},
@@ -517,6 +518,63 @@ impl ClientHandle {
         self.run(Command::Mentions(MentionsCommand::Recover)).await
     }
 
+    /// newest first, at most 50 a page. close the search when its results go, or its media
+    /// stay registered
+    pub async fn search_messages(
+        &self,
+        query: &str,
+        scope: SearchScope,
+        limit: u16,
+    ) -> Result<SearchSession, Error> {
+        match self
+            .execute(Command::Search(SearchCommand::Search {
+                query: query.to_owned(),
+                scope,
+                limit,
+            }))
+            .await?
+        {
+            CommandOutput::SearchSession(session) => Ok(session),
+            output => Err(unexpected("search session", &output)),
+        }
+    }
+
+    pub async fn search_more(
+        &self,
+        search_id: &str,
+        cursor: &str,
+        limit: u16,
+    ) -> Result<SearchPage, Error> {
+        match self
+            .execute(Command::Search(SearchCommand::More {
+                search_id: search_id.to_owned(),
+                cursor: cursor.to_owned(),
+                limit,
+            }))
+            .await?
+        {
+            CommandOutput::SearchPage(page) => Ok(page),
+            output => Err(unexpected("search page", &output)),
+        }
+    }
+
+    pub async fn close_search(&self, search_id: &str) -> Result<(), Error> {
+        self.run(Command::Search(SearchCommand::Close {
+            search_id: search_id.to_owned(),
+        }))
+        .await
+    }
+
+    pub async fn search_coverage(&self, scope: SearchScope) -> Result<SearchCoverage, Error> {
+        match self
+            .execute(Command::Search(SearchCommand::Coverage(scope)))
+            .await?
+        {
+            CommandOutput::SearchCoverage(coverage) => Ok(coverage),
+            output => Err(unexpected("search coverage", &output)),
+        }
+    }
+
     pub async fn send_message(
         &self,
         timeline_id: TimelineId,

macltok

Commit c9effeac326a

Commit `c9effeac326a`