atagen

matrix-rust-sdk

public
Star0
Watch0
ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk.git
xtask/src/log/overview.rsRust496 lines14.9 KB
1
use std::{
2
    collections::{BTreeMap, HashMap},
3
    fmt, fs,
4
    io::{self, BufRead, Write},
5
    ops::Not,
6
    path,
7
    str::FromStr,
8
};
9
10
use chrono::{DateTime, FixedOffset};
11
use lazy_static::lazy_static;
12
use regex::{Regex, RegexBuilder};
13
14
use crate::Result;
15
16
const OUTPUT_TEMPLATE: &str = include_str!("overview.template.html");
17
18
lazy_static! {
19
    static ref LINE_PARSER: Regex = {
20
        RegexBuilder::new(
21
            r#"
22
            # Let's start.
23
            ^
24
25
            # Datetime of the log line.
26
            (?<datetime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)
27
28
            # Log level.
29
            \s+(?<level>\S+)
30
31
            # Target.
32
            \s(?<target>matrix_[\w_]+(::[\w_]+)*):
33
34
            # The log message. We don't care about it.
35
            (?<message>.*)
36
37
            # The source file and line.
38
            \|\scrates/
39
            (?<source_file>[^:]+)
40
            :(?<source_line>\d+)
41
42
            # The spans.
43
            \s\|\sspans:\s
44
            (?<spans>.+)
45
        "#,
46
        )
47
        .ignore_whitespace(true)
48
        .build()
49
        .expect("Failed to build the `line_parser` regex")
50
    };
51
    static ref MESSAGE_PARSER: Regex = {
52
        RegexBuilder::new(
53
            r#"
54
            # Let's start.
55
            ^
56
57
            # Anything (trimmed)…
58
            \s*(?<message>.*?)
59
60
            # … until optional fields!
61
            (\s(?<fields>[\w\d_]+=.*))?$
62
        "#,
63
        )
64
        .ignore_whitespace(true)
65
        .build()
66
        .expect("Failed to build the `message_parser` regex")
67
    };
68
    static ref FIELDS_PARSER: Regex = {
69
        RegexBuilder::new(
70
            r#"
71
            # Let's start.
72
            ^
73
74
            # A name.
75
            \s*(?<name>[\w\d_]+)
76
            # Equal
77
            =
78
            # A value, which can be anything: it stops when a new field is found.
79
            (?<value>.*?)
80
81
            # The new field (and everything else).
82
            (?<next_fields>\s[\w\d_]+=.+)?$
83
        "#,
84
        )
85
        .ignore_whitespace(true)
86
        .build()
87
        .expect("Failed to build the `fields_parser` regex")
88
    };
89
}
90
91
pub(super) fn run(log_path: path::PathBuf, output_path: path::PathBuf) -> Result<()> {
92
    let line_parser = &*LINE_PARSER;
93
94
    let log_file = fs::File::open(&log_path)?;
95
    let mut output_file = io::BufWriter::new(fs::File::create(&output_path)?);
96
97
    let reader = io::BufReader::new(log_file);
98
    let mut number_of_analysed_lines = 0;
99
    let mut number_of_matched_lines = 0;
100
101
    let mut tree = TargetTree::default();
102
103
    for (line_nth, line) in reader.lines().enumerate().map(|(nth, line)| {
104
        (
105
            nth + 1,
106
            line.unwrap_or_else(|error| {
107
                panic!("Failed to read line #{nth}\n{error}");
108
            }),
109
        )
110
    }) {
111
        number_of_analysed_lines += 1;
112
113
        if let Some(captures) = line_parser.captures(&line) {
114
            number_of_matched_lines += 1;
115
116
            let date_time = DateTime::parse_from_rfc3339(
117
                captures.name("datetime").expect("Failed to capture `datetime`").as_str(),
118
            )
119
            .expect("Failed to parse `datetime`");
120
            let full_target = captures.name("target").expect("Failed to capture `target`").as_str();
121
            let level = captures
122
                .name("level")
123
                .expect("Failed to capture `level`")
124
                .as_str()
125
                .parse()
126
                .expect("Failed to parse the `level`");
127
            let message = captures.name("message").expect("Failed to capture `message`").as_str();
128
            let source_file =
129
                captures.name("source_file").expect("Failed to capture `source_file`").as_str();
130
            let source_line = captures
131
                .name("source_line")
132
                .expect("Failed to capture `source_line`")
133
                .as_str()
134
                .parse()
135
                .expect("Failed to parse the `source_line`");
136
            let spans = captures.name("spans").expect("Failed to capture `spans`").as_str();
137
138
            let mut tree_node = &mut tree;
139
140
            for target in full_target.split("::") {
141
                tree_node =
142
                    match tree_node.entry(target.to_owned()).or_insert_with(|| Target::Node {
143
                        number_of_errors: 0,
144
                        number_of_warnings: 0,
145
                        node: TargetTree::default(),
146
                    }) {
147
                        Target::Leaf(_) => panic!("Expect a `Node`, not a `Leaf`"),
148
                        Target::Node { number_of_errors, number_of_warnings, node } => {
149
                            // Adjust number of specific log levels all the targets.
150
                            match level {
151
                                Level::Error => *number_of_errors += 1,
152
                                Level::Warn => *number_of_warnings += 1,
153
                                _ => (),
154
                            }
155
156
                            node
157
                        }
158
                    };
159
            }
160
161
            let span =
162
                Span { message: message.to_owned(), log_line: line_nth, raw: spans.to_owned() };
163
164
            tree_node
165
                // `LogLevels`
166
                .entry("".to_owned())
167
                .or_insert_with(|| Target::Leaf(LogLevels::default()))
168
                .as_leaf_mut()
169
                .expect("Expect a `Leaf`, not a `Node`")
170
                // `LogLocations`
171
                .entry(level)
172
                .or_insert_with(LogLocations::default)
173
                // `SourceFile`
174
                .entry(source_file.to_owned())
175
                .or_insert_with(Spans::default)
176
                // `Spans`
177
                .entry(source_line)
178
                .or_insert_with(BTreeMap::default)
179
                // `Span`!
180
                .insert(date_time, span);
181
        }
182
    }
183
184
    fn render_target_tree(tree: TargetTree, buffer: &mut io::BufWriter<fs::File>) {
185
        if tree.is_empty() {
186
            return;
187
        }
188
189
        writeln!(buffer, "<ul>").unwrap();
190
191
        for (target_name, target) in tree {
192
            match target {
193
                Target::Node { node, number_of_errors, number_of_warnings } => {
194
                    render_node(target_name, node, number_of_errors, number_of_warnings, buffer);
195
                }
196
197
                Target::Leaf(log_levels) => {
198
                    render_leaf(log_levels, buffer);
199
                }
200
            }
201
        }
202
203
        writeln!(buffer, "</ul>").unwrap();
204
    }
205
206
    fn render_node(
207
        target_name: String,
208
        node: TargetTree,
209
        number_of_errors: usize,
210
        number_of_warnings: usize,
211
        buffer: &mut io::BufWriter<fs::File>,
212
    ) {
213
        writeln!(
214
            buffer,
215
            "<li>\n\
216
            <details><summary>{target_name}\
217
            <span class=\"aside\">\
218
            <span class=\"highlight\"><input type=\"checkbox\" title=\"Highlight\" /></span>\
219
            <span class=\"error\" title=\"Number of errors\">{number_of_errors}</span>\
220
            <span class=\"warn\" title=\"Number of warnings\">{number_of_warnings}</span>\
221
            </span>\
222
            </summary>"
223
        )
224
        .unwrap();
225
        render_target_tree(node, buffer);
226
        writeln!(buffer, "</details></li>").unwrap();
227
    }
228
229
    fn render_leaf(log_levels: LogLevels, buffer: &mut io::BufWriter<fs::File>) {
230
        writeln!(
231
            buffer,
232
            "<li>\n\
233
            <details open><summary>(root)\
234
            <span class=\"aside\">\
235
            <span class=\"highlight\"><input type=\"checkbox\" title=\"Highlight\" /></span>\
236
            </span>\
237
            </summary>"
238
        )
239
        .unwrap();
240
241
        if log_levels.is_empty().not() {
242
            writeln!(buffer, "<ul>").unwrap();
243
244
            for (level, locations) in log_levels {
245
                render_locations(level, locations, buffer);
246
            }
247
248
            writeln!(buffer, "</ul>").unwrap();
249
        }
250
251
        writeln!(buffer, "</details></li>").unwrap();
252
    }
253
254
    fn render_locations(
255
        level: Level,
256
        locations: LogLocations,
257
        buffer: &mut io::BufWriter<fs::File>,
258
    ) {
259
        for (source_file, spans) in locations {
260
            writeln!(
261
                buffer,
262
                "<li>\n\
263
                <details><summary><span class=\"{level_class}\">{level}</span> <code>{source_file}</code>\
264
                <span class=\"aside\">\
265
                <span class=\"highlight\"><input type=\"checkbox\" title=\"Highlight\" /></span>\
266
                </span>\
267
                </summary>\n\
268
                <ul>",
269
                level_class = level.css(),
270
            )
271
            .unwrap();
272
            render_spans(spans, buffer);
273
            writeln!(
274
                buffer,
275
                "</ul>\n\
276
                </details></li>"
277
            )
278
            .unwrap();
279
        }
280
    }
281
282
    fn render_spans(spans: Spans, buffer: &mut io::BufWriter<fs::File>) {
283
        for (source_line, all_spans) in spans {
284
            writeln!(
285
                buffer,
286
                "<li>\n\
287
                <details><summary>{n} ocurrrence{s} at line <code>{source_line}</code>\
288
                <span class=\"aside\">\
289
                <span class=\"highlight\"><input type=\"checkbox\" title=\"Highlight\" /></span>\
290
                </span>\
291
                </summary>\n\
292
                <ol class=\"spans\">",
293
                n = all_spans.len(),
294
                s = if all_spans.len() > 1 { "s" } else { "" }
295
            )
296
            .unwrap();
297
298
            for (date_time, span) in all_spans {
299
                render_span(date_time, span, buffer);
300
            }
301
302
            writeln!(
303
                buffer,
304
                "</ol>\n\
305
                </details></li>"
306
            )
307
            .unwrap();
308
        }
309
    }
310
311
    fn render_span(
312
        date_time: DateTime<FixedOffset>,
313
        span: Span,
314
        buffer: &mut io::BufWriter<fs::File>,
315
    ) {
316
        let Span { message, log_line, .. } = span;
317
318
        let message_parser = &*MESSAGE_PARSER;
319
        let fields_parser = &*FIELDS_PARSER;
320
321
        if let Some(captures) = message_parser.captures(&message) {
322
            // Poor man HTML sanitizer.
323
            let formatted_message = captures
324
                .name("message")
325
                .expect("Failed to parse the `message`")
326
                .as_str()
327
                .replace("&", "&amp;")
328
                .replace("<", "&lt;")
329
                .replace(">", "&gt;");
330
331
            writeln!(
332
                buffer,
333
                "<li><time datetime=\"{date_time}\">{time}</time> {formatted_message} <span title=\"Log line\">[#{log_line}]</span>\
334
                <span class=\"aside\">\
335
                <span class=\"highlight\"><input type=\"checkbox\" title=\"Highlight\" /></span>\
336
                </span>\
337
                ",
338
                date_time = date_time.format("%+"),
339
                time = date_time.format("%H:%M:%S%.3f"),
340
            )
341
            .unwrap();
342
343
            // The message can contain HTML data. It happens notably when the
344
            // homeserver is broken, and an HTML document is returned in some
345
            // errors. We don't want to parse the fields in this case, because
346
            // HTML breaks everything.
347
            if !formatted_message.contains("!DOCTYPE html")
348
                && let Some(fields) = captures.name("fields")
349
            {
350
                writeln!(buffer, "<ul class=\"fields\">").unwrap();
351
352
                let mut fields = &message[fields.start()..fields.end()];
353
354
                while let Some(captures) = fields_parser.captures(fields) {
355
                    let name = captures
356
                        .name("name")
357
                        .expect("Failed to parse the `name` of the field")
358
                        .as_str();
359
                    let value = captures
360
                        .name("value")
361
                        .expect("Failed to parse the `value` of the field")
362
                        .as_str();
363
364
                    writeln!(buffer, "<li><span>{name}</span><span>{value}</span></li>").unwrap();
365
366
                    if let Some(next_fields) = captures.name("next_fields") {
367
                        fields = &fields[next_fields.start()..];
368
                    } else {
369
                        break;
370
                    }
371
                }
372
373
                writeln!(buffer, "</ul>").unwrap();
374
            }
375
376
            writeln!(buffer, "</li>").unwrap();
377
        } else {
378
            writeln!(buffer, "<li>{message}</li>").unwrap();
379
        }
380
    }
381
382
    let output = OUTPUT_TEMPLATE.replace("{log_file}", &log_path.to_string_lossy());
383
384
    output_file.write_all(output.as_bytes()).expect(
385
        "Failed to write the
386
    output",
387
    );
388
389
    render_target_tree(tree, &mut output_file);
390
391
    output_file.write_all("</main></body></html>".as_bytes()).expect("Failed to write the output");
392
393
    println!(
394
        "\nNumber of analysed log lines: {number_of_analysed_lines}\n\
395
        Number of matched lines: {number_of_matched_lines}\n\
396
        Output file: {output_path:?}\n\
397
        Done!"
398
    );
399
400
    Ok(())
401
}
402
403
type TargetName = String;
404
type TargetTree = BTreeMap<TargetName, Target>;
405
406
#[derive(Debug)]
407
enum Target {
408
    Node {
409
        /// Number of errors for this node entirely (including sub-trees).
410
        number_of_errors: usize,
411
412
        /// Number of warnings for this node entirely (including sub-trees).
413
        number_of_warnings: usize,
414
415
        /// The Node.
416
        node: TargetTree,
417
    },
418
    Leaf(LogLevels),
419
}
420
421
#[derive(Debug, Eq, PartialEq, Ord, PartialOrd)]
422
enum Level {
423
    Error,
424
    Warn,
425
    Info,
426
    Debug,
427
    Trace,
428
}
429
430
type LogLevels = BTreeMap<Level, LogLocations>;
431
432
type SourceFile = String;
433
type SourceLine = usize;
434
type LogLocations = HashMap<SourceFile, Spans>;
435
436
type Spans = BTreeMap<SourceLine, BTreeMap<DateTime<FixedOffset>, Span>>;
437
438
#[derive(Debug)]
439
struct Span {
440
    message: String,
441
    log_line: usize,
442
    #[allow(dead_code)]
443
    raw: String,
444
}
445
446
impl Target {
447
    fn as_leaf_mut(&mut self) -> Option<&mut LogLevels> {
448
        match self {
449
            Self::Leaf(log_levels) => Some(log_levels),
450
            Self::Node { .. } => None,
451
        }
452
    }
453
}
454
455
impl Level {
456
    fn css(&self) -> &str {
457
        match self {
458
            Self::Error => "error",
459
            Self::Warn => "warn",
460
            Self::Info => "info",
461
            Self::Debug => "debug",
462
            Self::Trace => "trace",
463
        }
464
    }
465
}
466
467
impl FromStr for Level {
468
    type Err = ();
469
470
    fn from_str(str: &str) -> Result<Self, Self::Err> {
471
        Ok(match str {
472
            "ERROR" => Self::Error,
473
            "WARN" => Self::Warn,
474
            "INFO" => Self::Info,
475
            "DEBUG" => Self::Debug,
476
            "TRACE" => Self::Trace,
477
            _ => return Err(()),
478
        })
479
    }
480
}
481
482
impl fmt::Display for Level {
483
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
484
        write!(
485
            formatter,
486
            "{}",
487
            match self {
488
                Self::Error => "ERROR",
489
                Self::Warn => "WARNING",
490
                Self::Info => "INFO",
491
                Self::Debug => "DEBUG",
492
                Self::Trace => "TRACE",
493
            }
494
        )
495
    }
496
}