atagen

macltok

public
Star0
Watch0
417da9e719bcfbb8b60e4ff4f1a09f3d6b0d0160
3
ssh://brewery@pls.lobotomise.me:33/atagen/macltok.git

Blame: src/search/index_tests.rs

c9effeac326aatagen2026-10-25 10:32 UTC1
use std::collections::BTreeMap;
c9effeac326aatagen2026-10-25 10:32 UTC2
c9effeac326aatagen2026-10-25 10:32 UTC3
use super::*;
c9effeac326aatagen2026-10-25 10:32 UTC4
c9effeac326aatagen2026-10-25 10:32 UTC5
fn doc(key: &str, room: &str, ts: u64, text: &str) -> Doc {
c9effeac326aatagen2026-10-25 10:32 UTC6
    Doc {
c9effeac326aatagen2026-10-25 10:32 UTC7
        key: key.to_owned(),
c9effeac326aatagen2026-10-25 10:32 UTC8
        room: room.to_owned(),
c9effeac326aatagen2026-10-25 10:32 UTC9
        sender: "@a:x".to_owned(),
c9effeac326aatagen2026-10-25 10:32 UTC10
        ts,
c9effeac326aatagen2026-10-25 10:32 UTC11
        content: key.to_owned(),
c9effeac326aatagen2026-10-25 10:32 UTC12
        content_ts: ts,
c9effeac326aatagen2026-10-25 10:32 UTC13
        text: text.to_owned(),
c9effeac326aatagen2026-10-25 10:32 UTC14
        provisional: false,
c9effeac326aatagen2026-10-25 10:32 UTC15
    }
c9effeac326aatagen2026-10-25 10:32 UTC16
}
c9effeac326aatagen2026-10-25 10:32 UTC17
c9effeac326aatagen2026-10-25 10:32 UTC18
fn keys(hits: &[Hit]) -> Vec<&str> {
c9effeac326aatagen2026-10-25 10:32 UTC19
    hits.iter().map(|hit| hit.key.as_str()).collect()
c9effeac326aatagen2026-10-25 10:32 UTC20
}
c9effeac326aatagen2026-10-25 10:32 UTC21
74befbc72bafatagen2026-10-25 11:18 UTC22
/// the order `Service::commit` uses, the manifest saved with the blobs before `committed`
74befbc72bafatagen2026-10-25 11:18 UTC23
fn commit(index: &mut KeywordIndex, store: &mut BTreeMap<String, Vec<u8>>) {
c9effeac326aatagen2026-10-25 10:32 UTC24
    let changes = index.changes().unwrap();
c9effeac326aatagen2026-10-25 10:32 UTC25
    for key in &changes.deletes {
74befbc72bafatagen2026-10-25 11:18 UTC26
        store.remove(key);
c9effeac326aatagen2026-10-25 10:32 UTC27
    }
c9effeac326aatagen2026-10-25 10:32 UTC28
    for (key, value) in &changes.puts {
74befbc72bafatagen2026-10-25 11:18 UTC29
        store.insert(key.clone(), value.clone());
c9effeac326aatagen2026-10-25 10:32 UTC30
    }
74befbc72bafatagen2026-10-25 11:18 UTC31
    store.insert("meta".to_owned(), changes.manifest.to_string().into_bytes());
c9effeac326aatagen2026-10-25 10:32 UTC32
    index.committed(&changes);
74befbc72bafatagen2026-10-25 11:18 UTC33
}
74befbc72bafatagen2026-10-25 11:18 UTC34
74befbc72bafatagen2026-10-25 11:18 UTC35
fn reload(store: &BTreeMap<String, Vec<u8>>) -> KeywordIndex {
74befbc72bafatagen2026-10-25 11:18 UTC36
    let manifest: serde_json::Value = serde_json::from_slice(&store["meta"]).unwrap();
74befbc72bafatagen2026-10-25 11:18 UTC37
    KeywordIndex::load(&manifest, |key| store.get(key).cloned()).unwrap()
74befbc72bafatagen2026-10-25 11:18 UTC38
}
74befbc72bafatagen2026-10-25 11:18 UTC39
74befbc72bafatagen2026-10-25 11:18 UTC40
fn found(index: &mut KeywordIndex, query: &str) -> Vec<String> {
74befbc72bafatagen2026-10-25 11:18 UTC41
    let (hits, _) = index.search(query, None, None, 100).unwrap();
74befbc72bafatagen2026-10-25 11:18 UTC42
    let mut found: Vec<_> = keys(&hits).into_iter().map(str::to_owned).collect();
74befbc72bafatagen2026-10-25 11:18 UTC43
    found.sort();
74befbc72bafatagen2026-10-25 11:18 UTC44
    found
c9effeac326aatagen2026-10-25 10:32 UTC45
}
c9effeac326aatagen2026-10-25 10:32 UTC46
c9effeac326aatagen2026-10-25 10:32 UTC47
#[test]
c9effeac326aatagen2026-10-25 10:32 UTC48
fn pages_run_newest_first_without_repeats_across_segments() {
c9effeac326aatagen2026-10-25 10:32 UTC49
    let mut index = KeywordIndex::new(false);
c9effeac326aatagen2026-10-25 10:32 UTC50
    for ts in 0..30 {
c9effeac326aatagen2026-10-25 10:32 UTC51
        index.put(doc(&format!("$e{ts}"), "!r:x", ts, "hello there"));
c9effeac326aatagen2026-10-25 10:32 UTC52
        if ts % 7 == 0 {
c9effeac326aatagen2026-10-25 10:32 UTC53
            index.maintain().unwrap();
c9effeac326aatagen2026-10-25 10:32 UTC54
        }
c9effeac326aatagen2026-10-25 10:32 UTC55
    }
c9effeac326aatagen2026-10-25 10:32 UTC56
    let mut seen = Vec::new();
c9effeac326aatagen2026-10-25 10:32 UTC57
    let mut before = None;
c9effeac326aatagen2026-10-25 10:32 UTC58
    loop {
c9effeac326aatagen2026-10-25 10:32 UTC59
        let (hits, _) = index.search("hello", None, before, 8).unwrap();
c9effeac326aatagen2026-10-25 10:32 UTC60
        let Some(last) = hits.last() else { break };
c9effeac326aatagen2026-10-25 10:32 UTC61
        before = Some((last.ts, last.tie));
c9effeac326aatagen2026-10-25 10:32 UTC62
        seen.extend(hits.iter().map(|hit| hit.ts));
c9effeac326aatagen2026-10-25 10:32 UTC63
    }
c9effeac326aatagen2026-10-25 10:32 UTC64
    assert_eq!(seen, (0..30).rev().collect::<Vec<_>>());
c9effeac326aatagen2026-10-25 10:32 UTC65
}
c9effeac326aatagen2026-10-25 10:32 UTC66
c9effeac326aatagen2026-10-25 10:32 UTC67
#[test]
c9effeac326aatagen2026-10-25 10:32 UTC68
fn a_replaced_doc_is_found_once_by_its_new_text() {
c9effeac326aatagen2026-10-25 10:32 UTC69
    let mut index = KeywordIndex::new(false);
c9effeac326aatagen2026-10-25 10:32 UTC70
    index.put(doc("$a", "!r:x", 1, "before"));
c9effeac326aatagen2026-10-25 10:32 UTC71
    index.maintain().unwrap();
c9effeac326aatagen2026-10-25 10:32 UTC72
    index.put(doc("$a", "!r:x", 1, "after"));
74befbc72bafatagen2026-10-25 11:18 UTC73
    assert!(found(&mut index, "before").is_empty());
74befbc72bafatagen2026-10-25 11:18 UTC74
    assert_eq!(found(&mut index, "after"), vec!["$a"]);
c9effeac326aatagen2026-10-25 10:32 UTC75
}
c9effeac326aatagen2026-10-25 10:32 UTC76
c9effeac326aatagen2026-10-25 10:32 UTC77
#[test]
c9effeac326aatagen2026-10-25 10:32 UTC78
fn rooms_filter_the_results() {
c9effeac326aatagen2026-10-25 10:32 UTC79
    let mut index = KeywordIndex::new(false);
c9effeac326aatagen2026-10-25 10:32 UTC80
    index.put(doc("$a", "!one:x", 1, "word"));
c9effeac326aatagen2026-10-25 10:32 UTC81
    index.put(doc("$b", "!two:x", 2, "word"));
c9effeac326aatagen2026-10-25 10:32 UTC82
    let rooms = vec!["!one:x".to_owned()];
c9effeac326aatagen2026-10-25 10:32 UTC83
    assert_eq!(
c9effeac326aatagen2026-10-25 10:32 UTC84
        keys(&index.search("word", Some(&rooms), None, 10).unwrap().0),
c9effeac326aatagen2026-10-25 10:32 UTC85
        vec!["$a"]
c9effeac326aatagen2026-10-25 10:32 UTC86
    );
c9effeac326aatagen2026-10-25 10:32 UTC87
    assert!(
c9effeac326aatagen2026-10-25 10:32 UTC88
        index
c9effeac326aatagen2026-10-25 10:32 UTC89
            .search("word", Some(&[]), None, 10)
c9effeac326aatagen2026-10-25 10:32 UTC90
            .unwrap()
c9effeac326aatagen2026-10-25 10:32 UTC91
            .0
c9effeac326aatagen2026-10-25 10:32 UTC92
            .is_empty()
c9effeac326aatagen2026-10-25 10:32 UTC93
    );
c9effeac326aatagen2026-10-25 10:32 UTC94
}
c9effeac326aatagen2026-10-25 10:32 UTC95
c9effeac326aatagen2026-10-25 10:32 UTC96
#[test]
74befbc72bafatagen2026-10-25 11:18 UTC97
fn provisional_docs_stay_out_of_results() {
74befbc72bafatagen2026-10-25 11:18 UTC98
    let mut index = KeywordIndex::new(false);
74befbc72bafatagen2026-10-25 11:18 UTC99
    let mut pending = doc(&provisional_key("$o", "@a:x"), "!r:x", 5, "early edit");
74befbc72bafatagen2026-10-25 11:18 UTC100
    pending.provisional = true;
74befbc72bafatagen2026-10-25 11:18 UTC101
    index.put(pending);
74befbc72bafatagen2026-10-25 11:18 UTC102
    assert!(found(&mut index, "early").is_empty());
74befbc72bafatagen2026-10-25 11:18 UTC103
    assert!(index.remove_provisional("$o"));
74befbc72bafatagen2026-10-25 11:18 UTC104
    assert!(index.get(&provisional_key("$o", "@a:x")).is_none());
74befbc72bafatagen2026-10-25 11:18 UTC105
}
74befbc72bafatagen2026-10-25 11:18 UTC106
74befbc72bafatagen2026-10-25 11:18 UTC107
/// the manifest used to be read before `committed`, one commit stale, so a restart lost
74befbc72bafatagen2026-10-25 11:18 UTC108
/// the last flush and merge, and a bumped delete generation failed the whole load
74befbc72bafatagen2026-10-25 11:18 UTC109
#[test]
74befbc72bafatagen2026-10-25 11:18 UTC110
fn every_commit_reloads_to_what_was_committed() {
74befbc72bafatagen2026-10-25 11:18 UTC111
    let mut store = BTreeMap::new();
c9effeac326aatagen2026-10-25 10:32 UTC112
    let mut index = KeywordIndex::new(true);
74befbc72bafatagen2026-10-25 11:18 UTC113
    let count = MERGE_FACTOR as u64 + 2;
74befbc72bafatagen2026-10-25 11:18 UTC114
    for ts in 0..count {
c9effeac326aatagen2026-10-25 10:32 UTC115
        index.put(doc(&format!("$e{ts}"), "!r:x", ts, "kept"));
74befbc72bafatagen2026-10-25 11:18 UTC116
        commit(&mut index, &mut store);
74befbc72bafatagen2026-10-25 11:18 UTC117
        assert_eq!(found(&mut reload(&store), "kept").len(), ts as usize + 1);
c9effeac326aatagen2026-10-25 10:32 UTC118
    }
c9effeac326aatagen2026-10-25 10:32 UTC119
    index.remove("$e0");
74befbc72bafatagen2026-10-25 11:18 UTC120
    commit(&mut index, &mut store);
74befbc72bafatagen2026-10-25 11:18 UTC121
    index.remove("$e1");
74befbc72bafatagen2026-10-25 11:18 UTC122
    commit(&mut index, &mut store);
74befbc72bafatagen2026-10-25 11:18 UTC123
    index.put(doc("$e2", "!r:x", 2, "edited"));
74befbc72bafatagen2026-10-25 11:18 UTC124
    commit(&mut index, &mut store);
74befbc72bafatagen2026-10-25 11:18 UTC125
    let mut reloaded = reload(&store);
74befbc72bafatagen2026-10-25 11:18 UTC126
    let expected: Vec<String> = (3..count).map(|ts| format!("$e{ts}")).collect();
74befbc72bafatagen2026-10-25 11:18 UTC127
    assert_eq!(found(&mut reloaded, "kept"), expected);
74befbc72bafatagen2026-10-25 11:18 UTC128
    assert_eq!(found(&mut reloaded, "edited"), vec!["$e2"]);
74befbc72bafatagen2026-10-25 11:18 UTC129
    let mut referenced = reloaded.referenced();
74befbc72bafatagen2026-10-25 11:18 UTC130
    referenced.push("meta".to_owned());
74befbc72bafatagen2026-10-25 11:18 UTC131
    referenced.sort();
74befbc72bafatagen2026-10-25 11:18 UTC132
    let stored: Vec<_> = store.keys().cloned().collect();
74befbc72bafatagen2026-10-25 11:18 UTC133
    assert_eq!(stored, referenced, "a commit left blobs nothing points at");
c9effeac326aatagen2026-10-25 10:32 UTC134
}
c9effeac326aatagen2026-10-25 10:32 UTC135
c9effeac326aatagen2026-10-25 10:32 UTC136
#[test]
74befbc72bafatagen2026-10-25 11:18 UTC137
fn unspaced_text_matches_inside_a_run() {
74befbc72bafatagen2026-10-25 11:18 UTC138
    let mut index = KeywordIndex::new(false);
74befbc72bafatagen2026-10-25 11:18 UTC139
    index.put(doc("$a", "!r:x", 1, "明日東京都庁に行きます"));
74befbc72bafatagen2026-10-25 11:18 UTC140
    assert_eq!(found(&mut index, "東京"), vec!["$a"]);
74befbc72bafatagen2026-10-25 11:18 UTC141
    assert_eq!(found(&mut index, "\"都庁\""), vec!["$a"]);
74befbc72bafatagen2026-10-25 11:18 UTC142
    assert!(found(&mut index, "大阪").is_empty());
c9effeac326aatagen2026-10-25 10:32 UTC143
}
c9effeac326aatagen2026-10-25 10:32 UTC144
c9effeac326aatagen2026-10-25 10:32 UTC145
#[test]
c9effeac326aatagen2026-10-25 10:32 UTC146
fn packed_segments_reject_truncation() {
74befbc72bafatagen2026-10-25 11:18 UTC147
    let packed = pack(&[("a".to_owned(), OwnedBytes::new(vec![1, 2, 3]))]);
c9effeac326aatagen2026-10-25 10:32 UTC148
    assert!(unpack(&packed).is_some());
c9effeac326aatagen2026-10-25 10:32 UTC149
    assert!(unpack(&packed[..packed.len() - 1]).is_none());
c9effeac326aatagen2026-10-25 10:32 UTC150
}