1 files changed, 63 insertions, 28 deletions

diff --git a/apps/cassiopeia/src/format/lexer.rs b/apps/cassiopeia/src/format/lexer.rs
index f062ca4238c1..bdb89f5180e5 100644
--- a/apps/cassiopeia/src/format/lexer.rs
+++ b/apps/cassiopeia/src/format/lexer.rs
@@ -1,6 +1,7 @@
 //! Cassiopeia file lexer
 
-use logos::Logos;
+use logos::{Lexer, Logos};
+use std::iter::Iterator;
 
 /// A basic line lexer type
 ///
@@ -8,8 +9,7 @@ use logos::Logos;
 /// does not attempt to parse the line specifics.  This is what the
 /// content lexer is for.
 #[derive(Logos, Debug, PartialEq)]
-enum Line {
-
+pub(crate) enum Token {
     #[token("HEADER")]
     Header,
 
@@ -21,7 +21,7 @@ enum Line {
 
     #[token("INVOICE")]
     Invoice,
-    
+
     #[regex(r"\w+=[^,$]+[,$]")]
     HeaderData,
 
@@ -32,85 +32,120 @@ enum Line {
 
     #[token(" ", logos::skip)]
     Space,
-    
+
+    #[regex(";;.*")]
+    Comment,
+
     #[error]
     Error,
 }
 
+/// A single token type on a line
+#[derive(Debug)]
+pub(crate) struct LineToken<'l> {
+    pub(crate) tt: Token,
+    pub(crate) slice: &'l str,
+}
 
-// pub fn test_this() {
-//     // let mut lex = Line::lexer("HEADER version=0.0.0,location=Berlin,");
-//     let mut lex = Line::lexer("START 2020-11-11 13:00:00+01:00");
+/// A lexer wrapped for a single line
+pub(crate) struct LineLexer<'l> {
+    lexer: Lexer<'l, Token>,
+}
+
+impl<'l> LineLexer<'l> {
+    pub(crate) fn get_all(self) -> Vec<LineToken<'l>> {
+        let mut acc = vec![];
+        for l in self {
+            acc.push(l);
+        }
+        acc
+    }
+}
 
-//     while let Some(t) = lex.next() {
-//         println!("{:?}: {}", t, lex.slice());
-//     }
-// }
+impl<'l> Iterator for LineLexer<'l> {
+    type Item = LineToken<'l>;
 
+    fn next(&mut self) -> Option<Self::Item> {
+        self.lexer.next().map(|tt| Self::Item {
+            tt,
+            slice: self.lexer.slice(),
+        })
+    }
+}
+
+/// Take a line of input and lex it into a stream of tokens
+pub(crate) fn lex<'l>(line: &'l mut String) -> LineLexer<'l> {
+    LineLexer {
+        lexer: Token::lexer(line),
+    }
+}
 
 #[test]
 fn basic_header() {
-    let mut lex = Line::lexer("HEADER version=0.0.0,location=Berlin Lichtenberg,");
+    let mut lex = Token::lexer("HEADER version=0.0.0,location=Berlin Lichtenberg,");
 
-    assert_eq!(lex.next(), Some(Line::Header));
+    assert_eq!(lex.next(), Some(Token::Header));
     assert_eq!(lex.span(), 0..6);
     assert_eq!(lex.slice(), "HEADER");
 
-    assert_eq!(lex.next(), Some(Line::HeaderData));
+    assert_eq!(lex.next(), Some(Token::HeaderData));
     assert_eq!(lex.span(), 7..21);
     assert_eq!(lex.slice(), "version=0.0.0,");
 
-    assert_eq!(lex.next(), Some(Line::HeaderData));
+    assert_eq!(lex.next(), Some(Token::HeaderData));
     assert_eq!(lex.span(), 21..49);
     assert_eq!(lex.slice(), "location=Berlin Lichtenberg,");
 
     assert_eq!(lex.next(), None);
 }
 
-
 #[test]
 fn basic_start() {
-    let mut lex = Line::lexer("START 2020-11-11 13:00:00+01:00");
+    let mut lex = Token::lexer("START 2020-11-11 13:00:00+01:00");
 
-    assert_eq!(lex.next(), Some(Line::Start));
+    assert_eq!(lex.next(), Some(Token::Start));
     assert_eq!(lex.span(), 0..5);
     assert_eq!(lex.slice(), "START");
 
-    assert_eq!(lex.next(), Some(Line::Date));
+    assert_eq!(lex.next(), Some(Token::Date));
     assert_eq!(lex.span(), 5..31);
     assert_eq!(lex.slice(), " 2020-11-11 13:00:00+01:00");
 
     assert_eq!(lex.next(), None);
 }
 
-
 #[test]
 fn basic_stop() {
-    let mut lex = Line::lexer("STOP 2020-11-11 13:00:00+01:00");
+    let mut lex = Token::lexer("STOP 2020-11-11 13:00:00+01:00");
 
-    assert_eq!(lex.next(), Some(Line::Stop));
+    assert_eq!(lex.next(), Some(Token::Stop));
     assert_eq!(lex.span(), 0..4);
     assert_eq!(lex.slice(), "STOP");
 
-    assert_eq!(lex.next(), Some(Line::Date));
+    assert_eq!(lex.next(), Some(Token::Date));
     assert_eq!(lex.span(), 4..30);
     assert_eq!(lex.slice(), " 2020-11-11 13:00:00+01:00");
 
     assert_eq!(lex.next(), None);
 }
 
-
 #[test]
 fn basic_invoice() {
-    let mut lex = Line::lexer("INVOICE 2020-11-11 13:00:00+01:00");
+    let mut lex = Token::lexer("INVOICE 2020-11-11 13:00:00+01:00");
 
-    assert_eq!(lex.next(), Some(Line::Invoice));
+    assert_eq!(lex.next(), Some(Token::Invoice));
     assert_eq!(lex.span(), 0..7);
     assert_eq!(lex.slice(), "INVOICE");
 
-    assert_eq!(lex.next(), Some(Line::Date));
+    assert_eq!(lex.next(), Some(Token::Date));
     assert_eq!(lex.span(), 7..33);
     assert_eq!(lex.slice(), " 2020-11-11 13:00:00+01:00");
 
     assert_eq!(lex.next(), None);
 }
+
+#[test]
+fn basic_comment() {
+    let mut lex = Token::lexer(";; This file is auto generated!");
+    assert_eq!(lex.next(), Some(Token::Comment));
+}