c9effeac326a | atagen | 2026-10-25 10:32 UTC | 1 | use std::collections::BTreeMap; |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 2 | |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 3 | use super::*; |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 4 | |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 5 | fn doc(key: &str, room: &str, ts: u64, text: &str) -> Doc { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 6 | Doc { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 7 | key: key.to_owned(), |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 8 | room: room.to_owned(), |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 9 | sender: "@a:x".to_owned(), |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 10 | ts, |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 11 | content: key.to_owned(), |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 12 | content_ts: ts, |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 13 | text: text.to_owned(), |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 14 | provisional: false, |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 15 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 16 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 17 | |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 18 | fn keys(hits: &[Hit]) -> Vec<&str> { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 19 | hits.iter().map(|hit| hit.key.as_str()).collect() |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 20 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 21 | |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 22 | fn commit(index: &mut KeywordIndex, blobs: &mut BTreeMap<String, Vec<u8>>) -> serde_json::Value { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 23 | let changes = index.changes().unwrap(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 24 | for key in &changes.deletes { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 25 | blobs.remove(key); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 26 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 27 | for (key, value) in &changes.puts { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 28 | blobs.insert(key.clone(), value.clone()); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 29 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 30 | index.committed(&changes); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 31 | index.manifest() |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 32 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 33 | |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 34 | #[test] |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 35 | fn pages_run_newest_first_without_repeats_across_segments() { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 36 | let mut index = KeywordIndex::new(false); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 37 | for ts in 0..30 { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 38 | index.put(doc(&format!("$e{ts}"), "!r:x", ts, "hello there")); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 39 | if ts % 7 == 0 { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 40 | index.maintain().unwrap(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 41 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 42 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 43 | let mut seen = Vec::new(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 44 | let mut before = None; |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 45 | loop { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 46 | let (hits, _) = index.search("hello", None, before, 8).unwrap(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 47 | let Some(last) = hits.last() else { break }; |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 48 | before = Some((last.ts, last.tie)); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 49 | seen.extend(hits.iter().map(|hit| hit.ts)); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 50 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 51 | assert_eq!(seen, (0..30).rev().collect::<Vec<_>>()); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 52 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 53 | |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 54 | #[test] |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 55 | fn a_replaced_doc_is_found_once_by_its_new_text() { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 56 | let mut index = KeywordIndex::new(false); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 57 | index.put(doc("$a", "!r:x", 1, "before")); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 58 | index.maintain().unwrap(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 59 | index.put(doc("$a", "!r:x", 1, "after")); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 60 | assert!(index.search("before", None, None, 10).unwrap().0.is_empty()); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 61 | assert_eq!( |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 62 | keys(&index.search("after", None, None, 10).unwrap().0), |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 63 | vec!["$a"] |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 64 | ); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 65 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 66 | |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 67 | #[test] |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 68 | fn rooms_filter_the_results() { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 69 | let mut index = KeywordIndex::new(false); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 70 | index.put(doc("$a", "!one:x", 1, "word")); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 71 | index.put(doc("$b", "!two:x", 2, "word")); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 72 | let rooms = vec!["!one:x".to_owned()]; |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 73 | assert_eq!( |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 74 | keys(&index.search("word", Some(&rooms), None, 10).unwrap().0), |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 75 | vec!["$a"] |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 76 | ); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 77 | assert!( |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 78 | index |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 79 | .search("word", Some(&[]), None, 10) |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 80 | .unwrap() |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 81 | .0 |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 82 | .is_empty() |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 83 | ); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 84 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 85 | |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 86 | /// a lost delete brings a redacted message back, and a lost segment loses messages |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 87 | #[test] |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 88 | fn deletes_and_merges_survive_a_reload() { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 89 | let mut blobs = BTreeMap::new(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 90 | let mut index = KeywordIndex::new(true); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 91 | for ts in 0..(MERGE_FACTOR as u64 + 2) { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 92 | index.put(doc(&format!("$e{ts}"), "!r:x", ts, "kept")); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 93 | commit(&mut index, &mut blobs); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 94 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 95 | index.put(doc("$gone", "!r:x", 100, "kept")); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 96 | commit(&mut index, &mut blobs); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 97 | index.remove("$gone"); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 98 | index.remove("$e0"); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 99 | let manifest = commit(&mut index, &mut blobs); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 100 | let mut reloaded = KeywordIndex::load(&manifest, |key| blobs.get(key).cloned()).unwrap(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 101 | let (hits, _) = reloaded.search("kept", None, None, 50).unwrap(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 102 | let mut found = keys(&hits); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 103 | found.sort(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 104 | let mut expected: Vec<String> = (1..(MERGE_FACTOR as u64 + 2)) |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 105 | .map(|ts| format!("$e{ts}")) |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 106 | .collect(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 107 | expected.sort(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 108 | assert_eq!( |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 109 | found, |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 110 | expected.iter().map(String::as_str).collect::<Vec<_>>() |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 111 | ); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 112 | let referenced: Vec<_> = blobs.keys().filter(|key| key.starts_with("seg/")).collect(); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 113 | assert!( |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 114 | referenced.len() < MERGE_FACTOR, |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 115 | "merged segments were not dropped: {referenced:?}" |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 116 | ); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 117 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 118 | |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 119 | #[test] |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 120 | fn match_ranges_follow_the_analyser() { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 121 | assert_eq!( |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 122 | match_ranges("Café, CAFE and cafés", &["cafe".to_owned()]), |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 123 | vec![(0, 5), (7, 11)] |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 124 | ); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 125 | } |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 126 | |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 127 | #[test] |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 128 | fn packed_segments_reject_truncation() { |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 129 | let packed = pack(&[("a".to_owned(), vec![1, 2, 3])]); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 130 | assert!(unpack(&packed).is_some()); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 131 | assert!(unpack(&packed[..packed.len() - 1]).is_none()); |
c9effeac326a | atagen | 2026-10-25 10:32 UTC | 132 | } |