| 1 | use super::*; |
| 2 | |
| 3 | fn tokens(text: &str) -> Vec<String> { |
| 4 | let mut analyzer = analyzer(); |
| 5 | let mut stream = analyzer.token_stream(text); |
| 6 | let mut found = Vec::new(); |
| 7 | while stream.advance() { |
| 8 | found.push(stream.token().text.clone()); |
| 9 | } |
| 10 | found |
| 11 | } |
| 12 | |
| 13 | fn highlights(query: &str) -> Result<Vec<String>, ErrorKind> { |
| 14 | build_query(Field::from_field_id(0), query) |
| 15 | .map(|(_, terms)| terms) |
| 16 | .map_err(|error| error.kind) |
| 17 | } |
| 18 | |
| 19 | #[test] |
| 20 | fn unspaced_scripts_become_singles_and_pairs() { |
| 21 | assert_eq!(tokens("東京都"), vec!["東", "東京", "京", "京都", "都"]); |
| 22 | assert_eq!(tokens("ok 日本"), vec!["ok", "日", "日本", "本"]); |
| 23 | } |
| 24 | |
| 25 | #[test] |
| 26 | fn the_word_limit_counts_chars_not_bytes() { |
| 27 | let sha = "a".repeat(40); |
| 28 | let cyrillic = "щ".repeat(40); |
| 29 | assert_eq!(tokens(&format!("{sha} {cyrillic}")).len(), 2); |
| 30 | assert!(tokens(&"x".repeat(65)).is_empty()); |
| 31 | } |
| 32 | |
| 33 | /// a colon used to mean a field to the query parser, so "re: deploy" lost both words |
| 34 | #[test] |
| 35 | fn queries_have_no_field_syntax() { |
| 36 | assert_eq!(highlights("re: deploy").unwrap(), vec!["deploy", "re"]); |
| 37 | assert_eq!( |
| 38 | highlights("https://example.org/x").unwrap(), |
| 39 | vec!["example", "https", "org", "x"] |
| 40 | ); |
| 41 | assert_eq!(highlights("sender:@a").unwrap(), vec!["a", "sender"]); |
| 42 | } |
| 43 | |
| 44 | #[test] |
| 45 | fn nothing_but_exclusions_or_wildcards_is_invalid() { |
| 46 | assert_eq!(highlights("-spam"), Err(ErrorKind::Invalid)); |
| 47 | assert_eq!(highlights("*"), Err(ErrorKind::Invalid)); |
| 48 | assert_eq!(highlights(" "), Err(ErrorKind::Invalid)); |
| 49 | assert_eq!(highlights("ham -spam").unwrap(), vec!["ham"]); |
| 50 | } |
| 51 | |
| 52 | #[test] |
| 53 | fn a_cjk_word_query_needs_its_pairs() { |
| 54 | assert_eq!(highlights("東京").unwrap(), vec!["東京"]); |
| 55 | assert_eq!(highlights("東").unwrap(), vec!["東"]); |
| 56 | } |
| 57 | |
| 58 | #[test] |
| 59 | fn match_ranges_join_overlapping_pairs() { |
| 60 | assert_eq!( |
| 61 | match_ranges("Café, CAFE and cafés", &["cafe".to_owned()]), |
| 62 | vec![(0, 5), (7, 11)] |
| 63 | ); |
| 64 | assert_eq!( |
| 65 | match_ranges("東京都", &["東京".to_owned(), "京都".to_owned()]), |
| 66 | vec![(0, 9)] |
| 67 | ); |
| 68 | } |