macltok

use super::*;

fn tokens(text: &str) -> Vec<String> {

    let mut analyzer = analyzer();

    let mut stream = analyzer.token_stream(text);

    let mut found = Vec::new();

    while stream.advance() {

        found.push(stream.token().text.clone());

    }

    found

}

fn highlights(query: &str) -> Result<Vec<String>, ErrorKind> {

    build_query(Field::from_field_id(0), query)

        .map(|(_, terms)| terms)

        .map_err(|error| error.kind)

}

#[test]

fn unspaced_scripts_become_singles_and_pairs() {

    assert_eq!(tokens("東京都"), vec!["東", "東京", "京", "京都", "都"]);

    assert_eq!(tokens("ok 日本"), vec!["ok", "日", "日本", "本"]);

}

#[test]

fn the_word_limit_counts_chars_not_bytes() {

    let sha = "a".repeat(40);

    let cyrillic = "щ".repeat(40);

    assert_eq!(tokens(&format!("{sha} {cyrillic}")).len(), 2);

    assert!(tokens(&"x".repeat(65)).is_empty());

}

/// a colon used to mean a field to the query parser, so "re: deploy" lost both words

#[test]

fn queries_have_no_field_syntax() {

    assert_eq!(highlights("re: deploy").unwrap(), vec!["deploy", "re"]);

    assert_eq!(

        highlights("https://example.org/x").unwrap(),

        vec!["example", "https", "org", "x"]

    );

    assert_eq!(highlights("sender:@a").unwrap(), vec!["a", "sender"]);

}

#[test]

fn nothing_but_exclusions_or_wildcards_is_invalid() {

    assert_eq!(highlights("-spam"), Err(ErrorKind::Invalid));

    assert_eq!(highlights("*"), Err(ErrorKind::Invalid));

    assert_eq!(highlights("   "), Err(ErrorKind::Invalid));

    assert_eq!(highlights("ham -spam").unwrap(), vec!["ham"]);

}

#[test]

fn a_cjk_word_query_needs_its_pairs() {

    assert_eq!(highlights("東京").unwrap(), vec!["東京"]);

    assert_eq!(highlights("東").unwrap(), vec!["東"]);

}

#[test]

fn match_ranges_join_overlapping_pairs() {

    assert_eq!(

        match_ranges("Café, CAFE and cafés", &["cafe".to_owned()]),

        vec![(0, 5), (7, 11)]

    );

    assert_eq!(

        match_ranges("東京都", &["東京".to_owned(), "京都".to_owned()]),

        vec![(0, 9)]

    );

}