1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
//! Cassiopeia file lexer
use logos::{Lexer, Logos};
use std::iter::Iterator;
/// A basic line lexer type
///
/// This lexer distinguishes between comments, and keyword lines. It
/// does not attempt to parse the line specifics. This is what the
/// content lexer is for.
#[derive(Logos, Debug, PartialEq)]
pub(crate) enum Token {
#[token("HEADER")]
Header,
#[token("START")]
Start,
#[token("STOP")]
Stop,
#[token("INVOICE")]
Invoice,
#[regex(r"\w+=[^,$]+[,$]")]
HeaderData,
// FIXME: this will have a leading whitespace that we could remove
// with ^\w, but logos does not support this at the moment
#[regex(r"[0-9-:+ ]+")]
Date,
#[token(" ", logos::skip)]
Space,
#[regex(";;.*")]
Comment,
#[error]
Error,
}
/// A single token type on a line
#[derive(Debug)]
pub(crate) struct LineToken<'l> {
pub(crate) tt: Token,
pub(crate) slice: &'l str,
}
/// A lexer wrapped for a single line
pub(crate) struct LineLexer<'l> {
lexer: Lexer<'l, Token>,
}
impl<'l> LineLexer<'l> {
pub(crate) fn get_all(self) -> Vec<LineToken<'l>> {
let mut acc = vec![];
for l in self {
acc.push(l);
}
acc
}
}
impl<'l> Iterator for LineLexer<'l> {
type Item = LineToken<'l>;
fn next(&mut self) -> Option<Self::Item> {
self.lexer.next().map(|tt| Self::Item {
tt,
slice: self.lexer.slice(),
})
}
}
/// Take a line of input and lex it into a stream of tokens
pub(crate) fn lex<'l>(line: &'l mut String) -> LineLexer<'l> {
LineLexer {
lexer: Token::lexer(line),
}
}
#[test]
fn basic_header() {
let mut lex = Token::lexer("HEADER version=0.0.0,location=Berlin Lichtenberg,");
assert_eq!(lex.next(), Some(Token::Header));
assert_eq!(lex.span(), 0..6);
assert_eq!(lex.slice(), "HEADER");
assert_eq!(lex.next(), Some(Token::HeaderData));
assert_eq!(lex.span(), 7..21);
assert_eq!(lex.slice(), "version=0.0.0,");
assert_eq!(lex.next(), Some(Token::HeaderData));
assert_eq!(lex.span(), 21..49);
assert_eq!(lex.slice(), "location=Berlin Lichtenberg,");
assert_eq!(lex.next(), None);
}
#[test]
fn basic_start() {
let mut lex = Token::lexer("START 2020-11-11 13:00:00+01:00");
assert_eq!(lex.next(), Some(Token::Start));
assert_eq!(lex.span(), 0..5);
assert_eq!(lex.slice(), "START");
assert_eq!(lex.next(), Some(Token::Date));
assert_eq!(lex.span(), 5..31);
assert_eq!(lex.slice(), " 2020-11-11 13:00:00+01:00");
assert_eq!(lex.next(), None);
}
#[test]
fn basic_stop() {
let mut lex = Token::lexer("STOP 2020-11-11 13:00:00+01:00");
assert_eq!(lex.next(), Some(Token::Stop));
assert_eq!(lex.span(), 0..4);
assert_eq!(lex.slice(), "STOP");
assert_eq!(lex.next(), Some(Token::Date));
assert_eq!(lex.span(), 4..30);
assert_eq!(lex.slice(), " 2020-11-11 13:00:00+01:00");
assert_eq!(lex.next(), None);
}
#[test]
fn basic_invoice() {
let mut lex = Token::lexer("INVOICE 2020-11-11 13:00:00+01:00");
assert_eq!(lex.next(), Some(Token::Invoice));
assert_eq!(lex.span(), 0..7);
assert_eq!(lex.slice(), "INVOICE");
assert_eq!(lex.next(), Some(Token::Date));
assert_eq!(lex.span(), 7..33);
assert_eq!(lex.slice(), " 2020-11-11 13:00:00+01:00");
assert_eq!(lex.next(), None);
}
#[test]
fn basic_comment() {
let mut lex = Token::lexer(";; This file is auto generated!");
assert_eq!(lex.next(), Some(Token::Comment));
}
|
