atagen

macltok

public
Star0
Watch0
c9effeac326a64a991e3de0049f2ffa1c7a2d18a
3
ssh://brewery@pls.lobotomise.me:33/atagen/macltok.git

Blame: src/search/index_tests.rs

c9effeac326aatagen2026-10-25 10:32 UTC1
use std::collections::BTreeMap;
c9effeac326aatagen2026-10-25 10:32 UTC2
c9effeac326aatagen2026-10-25 10:32 UTC3
use super::*;
c9effeac326aatagen2026-10-25 10:32 UTC4
c9effeac326aatagen2026-10-25 10:32 UTC5
fn doc(key: &str, room: &str, ts: u64, text: &str) -> Doc {
c9effeac326aatagen2026-10-25 10:32 UTC6
    Doc {
c9effeac326aatagen2026-10-25 10:32 UTC7
        key: key.to_owned(),
c9effeac326aatagen2026-10-25 10:32 UTC8
        room: room.to_owned(),
c9effeac326aatagen2026-10-25 10:32 UTC9
        sender: "@a:x".to_owned(),
c9effeac326aatagen2026-10-25 10:32 UTC10
        ts,
c9effeac326aatagen2026-10-25 10:32 UTC11
        content: key.to_owned(),
c9effeac326aatagen2026-10-25 10:32 UTC12
        content_ts: ts,
c9effeac326aatagen2026-10-25 10:32 UTC13
        text: text.to_owned(),
c9effeac326aatagen2026-10-25 10:32 UTC14
        provisional: false,
c9effeac326aatagen2026-10-25 10:32 UTC15
    }
c9effeac326aatagen2026-10-25 10:32 UTC16
}
c9effeac326aatagen2026-10-25 10:32 UTC17
c9effeac326aatagen2026-10-25 10:32 UTC18
fn keys(hits: &[Hit]) -> Vec<&str> {
c9effeac326aatagen2026-10-25 10:32 UTC19
    hits.iter().map(|hit| hit.key.as_str()).collect()
c9effeac326aatagen2026-10-25 10:32 UTC20
}
c9effeac326aatagen2026-10-25 10:32 UTC21
c9effeac326aatagen2026-10-25 10:32 UTC22
fn commit(index: &mut KeywordIndex, blobs: &mut BTreeMap<String, Vec<u8>>) -> serde_json::Value {
c9effeac326aatagen2026-10-25 10:32 UTC23
    let changes = index.changes().unwrap();
c9effeac326aatagen2026-10-25 10:32 UTC24
    for key in &changes.deletes {
c9effeac326aatagen2026-10-25 10:32 UTC25
        blobs.remove(key);
c9effeac326aatagen2026-10-25 10:32 UTC26
    }
c9effeac326aatagen2026-10-25 10:32 UTC27
    for (key, value) in &changes.puts {
c9effeac326aatagen2026-10-25 10:32 UTC28
        blobs.insert(key.clone(), value.clone());
c9effeac326aatagen2026-10-25 10:32 UTC29
    }
c9effeac326aatagen2026-10-25 10:32 UTC30
    index.committed(&changes);
c9effeac326aatagen2026-10-25 10:32 UTC31
    index.manifest()
c9effeac326aatagen2026-10-25 10:32 UTC32
}
c9effeac326aatagen2026-10-25 10:32 UTC33
c9effeac326aatagen2026-10-25 10:32 UTC34
#[test]
c9effeac326aatagen2026-10-25 10:32 UTC35
fn pages_run_newest_first_without_repeats_across_segments() {
c9effeac326aatagen2026-10-25 10:32 UTC36
    let mut index = KeywordIndex::new(false);
c9effeac326aatagen2026-10-25 10:32 UTC37
    for ts in 0..30 {
c9effeac326aatagen2026-10-25 10:32 UTC38
        index.put(doc(&format!("$e{ts}"), "!r:x", ts, "hello there"));
c9effeac326aatagen2026-10-25 10:32 UTC39
        if ts % 7 == 0 {
c9effeac326aatagen2026-10-25 10:32 UTC40
            index.maintain().unwrap();
c9effeac326aatagen2026-10-25 10:32 UTC41
        }
c9effeac326aatagen2026-10-25 10:32 UTC42
    }
c9effeac326aatagen2026-10-25 10:32 UTC43
    let mut seen = Vec::new();
c9effeac326aatagen2026-10-25 10:32 UTC44
    let mut before = None;
c9effeac326aatagen2026-10-25 10:32 UTC45
    loop {
c9effeac326aatagen2026-10-25 10:32 UTC46
        let (hits, _) = index.search("hello", None, before, 8).unwrap();
c9effeac326aatagen2026-10-25 10:32 UTC47
        let Some(last) = hits.last() else { break };
c9effeac326aatagen2026-10-25 10:32 UTC48
        before = Some((last.ts, last.tie));
c9effeac326aatagen2026-10-25 10:32 UTC49
        seen.extend(hits.iter().map(|hit| hit.ts));
c9effeac326aatagen2026-10-25 10:32 UTC50
    }
c9effeac326aatagen2026-10-25 10:32 UTC51
    assert_eq!(seen, (0..30).rev().collect::<Vec<_>>());
c9effeac326aatagen2026-10-25 10:32 UTC52
}
c9effeac326aatagen2026-10-25 10:32 UTC53
c9effeac326aatagen2026-10-25 10:32 UTC54
#[test]
c9effeac326aatagen2026-10-25 10:32 UTC55
fn a_replaced_doc_is_found_once_by_its_new_text() {
c9effeac326aatagen2026-10-25 10:32 UTC56
    let mut index = KeywordIndex::new(false);
c9effeac326aatagen2026-10-25 10:32 UTC57
    index.put(doc("$a", "!r:x", 1, "before"));
c9effeac326aatagen2026-10-25 10:32 UTC58
    index.maintain().unwrap();
c9effeac326aatagen2026-10-25 10:32 UTC59
    index.put(doc("$a", "!r:x", 1, "after"));
c9effeac326aatagen2026-10-25 10:32 UTC60
    assert!(index.search("before", None, None, 10).unwrap().0.is_empty());
c9effeac326aatagen2026-10-25 10:32 UTC61
    assert_eq!(
c9effeac326aatagen2026-10-25 10:32 UTC62
        keys(&index.search("after", None, None, 10).unwrap().0),
c9effeac326aatagen2026-10-25 10:32 UTC63
        vec!["$a"]
c9effeac326aatagen2026-10-25 10:32 UTC64
    );
c9effeac326aatagen2026-10-25 10:32 UTC65
}
c9effeac326aatagen2026-10-25 10:32 UTC66
c9effeac326aatagen2026-10-25 10:32 UTC67
#[test]
c9effeac326aatagen2026-10-25 10:32 UTC68
fn rooms_filter_the_results() {
c9effeac326aatagen2026-10-25 10:32 UTC69
    let mut index = KeywordIndex::new(false);
c9effeac326aatagen2026-10-25 10:32 UTC70
    index.put(doc("$a", "!one:x", 1, "word"));
c9effeac326aatagen2026-10-25 10:32 UTC71
    index.put(doc("$b", "!two:x", 2, "word"));
c9effeac326aatagen2026-10-25 10:32 UTC72
    let rooms = vec!["!one:x".to_owned()];
c9effeac326aatagen2026-10-25 10:32 UTC73
    assert_eq!(
c9effeac326aatagen2026-10-25 10:32 UTC74
        keys(&index.search("word", Some(&rooms), None, 10).unwrap().0),
c9effeac326aatagen2026-10-25 10:32 UTC75
        vec!["$a"]
c9effeac326aatagen2026-10-25 10:32 UTC76
    );
c9effeac326aatagen2026-10-25 10:32 UTC77
    assert!(
c9effeac326aatagen2026-10-25 10:32 UTC78
        index
c9effeac326aatagen2026-10-25 10:32 UTC79
            .search("word", Some(&[]), None, 10)
c9effeac326aatagen2026-10-25 10:32 UTC80
            .unwrap()
c9effeac326aatagen2026-10-25 10:32 UTC81
            .0
c9effeac326aatagen2026-10-25 10:32 UTC82
            .is_empty()
c9effeac326aatagen2026-10-25 10:32 UTC83
    );
c9effeac326aatagen2026-10-25 10:32 UTC84
}
c9effeac326aatagen2026-10-25 10:32 UTC85
c9effeac326aatagen2026-10-25 10:32 UTC86
/// a lost delete brings a redacted message back, and a lost segment loses messages
c9effeac326aatagen2026-10-25 10:32 UTC87
#[test]
c9effeac326aatagen2026-10-25 10:32 UTC88
fn deletes_and_merges_survive_a_reload() {
c9effeac326aatagen2026-10-25 10:32 UTC89
    let mut blobs = BTreeMap::new();
c9effeac326aatagen2026-10-25 10:32 UTC90
    let mut index = KeywordIndex::new(true);
c9effeac326aatagen2026-10-25 10:32 UTC91
    for ts in 0..(MERGE_FACTOR as u64 + 2) {
c9effeac326aatagen2026-10-25 10:32 UTC92
        index.put(doc(&format!("$e{ts}"), "!r:x", ts, "kept"));
c9effeac326aatagen2026-10-25 10:32 UTC93
        commit(&mut index, &mut blobs);
c9effeac326aatagen2026-10-25 10:32 UTC94
    }
c9effeac326aatagen2026-10-25 10:32 UTC95
    index.put(doc("$gone", "!r:x", 100, "kept"));
c9effeac326aatagen2026-10-25 10:32 UTC96
    commit(&mut index, &mut blobs);
c9effeac326aatagen2026-10-25 10:32 UTC97
    index.remove("$gone");
c9effeac326aatagen2026-10-25 10:32 UTC98
    index.remove("$e0");
c9effeac326aatagen2026-10-25 10:32 UTC99
    let manifest = commit(&mut index, &mut blobs);
c9effeac326aatagen2026-10-25 10:32 UTC100
    let mut reloaded = KeywordIndex::load(&manifest, |key| blobs.get(key).cloned()).unwrap();
c9effeac326aatagen2026-10-25 10:32 UTC101
    let (hits, _) = reloaded.search("kept", None, None, 50).unwrap();
c9effeac326aatagen2026-10-25 10:32 UTC102
    let mut found = keys(&hits);
c9effeac326aatagen2026-10-25 10:32 UTC103
    found.sort();
c9effeac326aatagen2026-10-25 10:32 UTC104
    let mut expected: Vec<String> = (1..(MERGE_FACTOR as u64 + 2))
c9effeac326aatagen2026-10-25 10:32 UTC105
        .map(|ts| format!("$e{ts}"))
c9effeac326aatagen2026-10-25 10:32 UTC106
        .collect();
c9effeac326aatagen2026-10-25 10:32 UTC107
    expected.sort();
c9effeac326aatagen2026-10-25 10:32 UTC108
    assert_eq!(
c9effeac326aatagen2026-10-25 10:32 UTC109
        found,
c9effeac326aatagen2026-10-25 10:32 UTC110
        expected.iter().map(String::as_str).collect::<Vec<_>>()
c9effeac326aatagen2026-10-25 10:32 UTC111
    );
c9effeac326aatagen2026-10-25 10:32 UTC112
    let referenced: Vec<_> = blobs.keys().filter(|key| key.starts_with("seg/")).collect();
c9effeac326aatagen2026-10-25 10:32 UTC113
    assert!(
c9effeac326aatagen2026-10-25 10:32 UTC114
        referenced.len() < MERGE_FACTOR,
c9effeac326aatagen2026-10-25 10:32 UTC115
        "merged segments were not dropped: {referenced:?}"
c9effeac326aatagen2026-10-25 10:32 UTC116
    );
c9effeac326aatagen2026-10-25 10:32 UTC117
}
c9effeac326aatagen2026-10-25 10:32 UTC118
c9effeac326aatagen2026-10-25 10:32 UTC119
#[test]
c9effeac326aatagen2026-10-25 10:32 UTC120
fn match_ranges_follow_the_analyser() {
c9effeac326aatagen2026-10-25 10:32 UTC121
    assert_eq!(
c9effeac326aatagen2026-10-25 10:32 UTC122
        match_ranges("Café, CAFE and cafés", &["cafe".to_owned()]),
c9effeac326aatagen2026-10-25 10:32 UTC123
        vec![(0, 5), (7, 11)]
c9effeac326aatagen2026-10-25 10:32 UTC124
    );
c9effeac326aatagen2026-10-25 10:32 UTC125
}
c9effeac326aatagen2026-10-25 10:32 UTC126
c9effeac326aatagen2026-10-25 10:32 UTC127
#[test]
c9effeac326aatagen2026-10-25 10:32 UTC128
fn packed_segments_reject_truncation() {
c9effeac326aatagen2026-10-25 10:32 UTC129
    let packed = pack(&[("a".to_owned(), vec![1, 2, 3])]);
c9effeac326aatagen2026-10-25 10:32 UTC130
    assert!(unpack(&packed).is_some());
c9effeac326aatagen2026-10-25 10:32 UTC131
    assert!(unpack(&packed[..packed.len() - 1]).is_none());
c9effeac326aatagen2026-10-25 10:32 UTC132
}