| 1 | use std::collections::BTreeMap; |
| 2 | |
| 3 | use super::*; |
| 4 | |
| 5 | fn doc(key: &str, room: &str, ts: u64, text: &str) -> Doc { |
| 6 | Doc { |
| 7 | key: key.to_owned(), |
| 8 | room: room.to_owned(), |
| 9 | sender: "@a:x".to_owned(), |
| 10 | ts, |
| 11 | content: key.to_owned(), |
| 12 | content_ts: ts, |
| 13 | text: text.to_owned(), |
| 14 | provisional: false, |
| 15 | } |
| 16 | } |
| 17 | |
| 18 | fn keys(hits: &[Hit]) -> Vec<&str> { |
| 19 | hits.iter().map(|hit| hit.key.as_str()).collect() |
| 20 | } |
| 21 | |
| 22 | fn commit(index: &mut KeywordIndex, blobs: &mut BTreeMap<String, Vec<u8>>) -> serde_json::Value { |
| 23 | let changes = index.changes().unwrap(); |
| 24 | for key in &changes.deletes { |
| 25 | blobs.remove(key); |
| 26 | } |
| 27 | for (key, value) in &changes.puts { |
| 28 | blobs.insert(key.clone(), value.clone()); |
| 29 | } |
| 30 | index.committed(&changes); |
| 31 | index.manifest() |
| 32 | } |
| 33 | |
| 34 | #[test] |
| 35 | fn pages_run_newest_first_without_repeats_across_segments() { |
| 36 | let mut index = KeywordIndex::new(false); |
| 37 | for ts in 0..30 { |
| 38 | index.put(doc(&format!("$e{ts}"), "!r:x", ts, "hello there")); |
| 39 | if ts % 7 == 0 { |
| 40 | index.maintain().unwrap(); |
| 41 | } |
| 42 | } |
| 43 | let mut seen = Vec::new(); |
| 44 | let mut before = None; |
| 45 | loop { |
| 46 | let (hits, _) = index.search("hello", None, before, 8).unwrap(); |
| 47 | let Some(last) = hits.last() else { break }; |
| 48 | before = Some((last.ts, last.tie)); |
| 49 | seen.extend(hits.iter().map(|hit| hit.ts)); |
| 50 | } |
| 51 | assert_eq!(seen, (0..30).rev().collect::<Vec<_>>()); |
| 52 | } |
| 53 | |
| 54 | #[test] |
| 55 | fn a_replaced_doc_is_found_once_by_its_new_text() { |
| 56 | let mut index = KeywordIndex::new(false); |
| 57 | index.put(doc("$a", "!r:x", 1, "before")); |
| 58 | index.maintain().unwrap(); |
| 59 | index.put(doc("$a", "!r:x", 1, "after")); |
| 60 | assert!(index.search("before", None, None, 10).unwrap().0.is_empty()); |
| 61 | assert_eq!( |
| 62 | keys(&index.search("after", None, None, 10).unwrap().0), |
| 63 | vec!["$a"] |
| 64 | ); |
| 65 | } |
| 66 | |
| 67 | #[test] |
| 68 | fn rooms_filter_the_results() { |
| 69 | let mut index = KeywordIndex::new(false); |
| 70 | index.put(doc("$a", "!one:x", 1, "word")); |
| 71 | index.put(doc("$b", "!two:x", 2, "word")); |
| 72 | let rooms = vec!["!one:x".to_owned()]; |
| 73 | assert_eq!( |
| 74 | keys(&index.search("word", Some(&rooms), None, 10).unwrap().0), |
| 75 | vec!["$a"] |
| 76 | ); |
| 77 | assert!( |
| 78 | index |
| 79 | .search("word", Some(&[]), None, 10) |
| 80 | .unwrap() |
| 81 | .0 |
| 82 | .is_empty() |
| 83 | ); |
| 84 | } |
| 85 | |
| 86 | /// a lost delete brings a redacted message back, and a lost segment loses messages |
| 87 | #[test] |
| 88 | fn deletes_and_merges_survive_a_reload() { |
| 89 | let mut blobs = BTreeMap::new(); |
| 90 | let mut index = KeywordIndex::new(true); |
| 91 | for ts in 0..(MERGE_FACTOR as u64 + 2) { |
| 92 | index.put(doc(&format!("$e{ts}"), "!r:x", ts, "kept")); |
| 93 | commit(&mut index, &mut blobs); |
| 94 | } |
| 95 | index.put(doc("$gone", "!r:x", 100, "kept")); |
| 96 | commit(&mut index, &mut blobs); |
| 97 | index.remove("$gone"); |
| 98 | index.remove("$e0"); |
| 99 | let manifest = commit(&mut index, &mut blobs); |
| 100 | let mut reloaded = KeywordIndex::load(&manifest, |key| blobs.get(key).cloned()).unwrap(); |
| 101 | let (hits, _) = reloaded.search("kept", None, None, 50).unwrap(); |
| 102 | let mut found = keys(&hits); |
| 103 | found.sort(); |
| 104 | let mut expected: Vec<String> = (1..(MERGE_FACTOR as u64 + 2)) |
| 105 | .map(|ts| format!("$e{ts}")) |
| 106 | .collect(); |
| 107 | expected.sort(); |
| 108 | assert_eq!( |
| 109 | found, |
| 110 | expected.iter().map(String::as_str).collect::<Vec<_>>() |
| 111 | ); |
| 112 | let referenced: Vec<_> = blobs.keys().filter(|key| key.starts_with("seg/")).collect(); |
| 113 | assert!( |
| 114 | referenced.len() < MERGE_FACTOR, |
| 115 | "merged segments were not dropped: {referenced:?}" |
| 116 | ); |
| 117 | } |
| 118 | |
| 119 | #[test] |
| 120 | fn match_ranges_follow_the_analyser() { |
| 121 | assert_eq!( |
| 122 | match_ranges("Café, CAFE and cafés", &["cafe".to_owned()]), |
| 123 | vec![(0, 5), (7, 11)] |
| 124 | ); |
| 125 | } |
| 126 | |
| 127 | #[test] |
| 128 | fn packed_segments_reject_truncation() { |
| 129 | let packed = pack(&[("a".to_owned(), vec![1, 2, 3])]); |
| 130 | assert!(unpack(&packed).is_some()); |
| 131 | assert!(unpack(&packed[..packed.len() - 1]).is_none()); |
| 132 | } |