review

src/main.rs

@@ -1,7 +1,6 @@
 use std::io;
 use std::io::Write;
 
-use parse::token::Token;
 use symbols::SymbolsTable;
 
 mod cli;
@@ -12,20 +11,14 @@ fn main() -> anyhow::Result<()> {
     let args = cli::Args::parse();
 
     let input = std::fs::read_to_string(&args.input).unwrap();
-    let tokens = parse::lexer::make_tokenizer(&input).collect::<Result<Vec<_>, _>>();
+    let mut symbols = SymbolsTable::default();
+    let tokens = parse::lexer::make_tokenizer(&input, &mut symbols).collect::<Result<Vec<_>, _>>();
     match tokens {
         Ok(tokens) => {
-            let symbols = SymbolsTable::from(tokens.iter().map(|(_, token, _)| token));
             let mut writer_tokens = io::BufWriter::new(std::fs::File::create(&args.output_tokens)?);
 
             for (_, token, _) in tokens {
-                match token {
-                    Token::Name(ref name) => match symbols.get(name) {
-                        Some(id) => writeln!(writer_tokens, "{token} ({id})")?,
-                        None => writeln!(writer_tokens, "{token} (undefined)")?,
-                    },
-                    _ => writeln!(writer_tokens, "{token}")?,
-                }
+                writeln!(writer_tokens, "{token:>6} - {}", token.as_str())?;
             }
 
             let mut writer_symbols =

src/parse/lexer.rs

@@ -1,28 +1,35 @@
 use itertools::PeekNth;
 
+use crate::symbols::SymbolsTable;
+
 use super::{error::LexicalError, token::Token};
 
 pub type SpannedToken = (usize, Token, usize);
 pub type LexerResult = Result<SpannedToken, LexicalError>;
 
 #[derive(Debug)]
-pub struct Lexer<T: Iterator<Item = (usize, char)>> {
+pub struct Lexer<'s, T: Iterator<Item = (usize, char)>> {
     chars: PeekNth<T>,
     pos: usize,
     pending: Vec<SpannedToken>,
+    symbols: &'s mut SymbolsTable<String>,
 }
 
-pub fn make_tokenizer(input: &str) -> impl Iterator<Item = LexerResult> + '_ {
+pub fn make_tokenizer<'s>(
+    input: &'s str,
+    symbols: &'s mut SymbolsTable<String>,
+) -> impl Iterator<Item = LexerResult> + 's {
     let chars = input.char_indices();
-    Lexer::new(chars)
+    Lexer::new(chars, symbols)
 }
 
-impl<T: Iterator<Item = (usize, char)>> Lexer<T> {
-    pub fn new(chars: T) -> Self {
+impl<'s, T: Iterator<Item = (usize, char)>> Lexer<'s, T> {
+    pub fn new(chars: T, symbols: &'s mut SymbolsTable<String>) -> Self {
         Self {
             chars: itertools::peek_nth(chars),
             pos: 0,
             pending: vec![],
+            symbols,
         }
     }
 
@@ -68,7 +75,13 @@ impl<T: Iterator<Item = (usize, char)>> Lexer<T> {
 
         let end = self.get_pos();
 
-        Ok((start, Token::Name(name), end))
+        if let Some(id) = self.symbols.get(&name) {
+            Ok((start, Token::Name(id), end))
+        } else {
+            self.symbols.add(name.clone());
+            let id = self.symbols.get(&name).unwrap();
+            Ok((start, Token::Name(id), end))
+        }
     }
 
     fn lex_number(&mut self) -> LexerResult {
@@ -137,8 +150,8 @@ impl<T: Iterator<Item = (usize, char)>> Lexer<T> {
         self.emit((start, token, end));
     }
 
-    fn is_number_start(&self, c: char, c1: Option<char>) -> bool {
-        c.is_ascii_digit() || (c == '.' && c1.map_or(false, |c| c.is_ascii_digit()))
+    fn is_number_start(&self, c: char, _c1: Option<char>) -> bool {
+        c.is_ascii_digit()
     }
 
     fn is_digit(&self, c: char) -> bool {
@@ -173,7 +186,7 @@ impl<T: Iterator<Item = (usize, char)>> Lexer<T> {
     }
 }
 
-impl<T: Iterator<Item = (usize, char)>> Iterator for Lexer<T> {
+impl<'s, T: Iterator<Item = (usize, char)>> Iterator for Lexer<'s, T> {
     type Item = LexerResult;
 
     fn next(&mut self) -> Option<Self::Item> {

src/parse/token.rs

@@ -2,7 +2,7 @@ use std::fmt;
 
 #[derive(Debug, Clone, PartialEq)]
 pub enum Token {
-    Name(String),
+    Name(usize),
     Float(f64),
     Int(i64),
     // operators
@@ -18,18 +18,36 @@ pub enum Token {
     EndOfFile, // end of file
 }
 
+impl Token {
+    pub fn as_str(&self) -> &str {
+        match self {
+            Token::Name(_) => "name",
+            Token::Float(_) => "float",
+            Token::Int(_) => "int",
+            Token::Plus => "plus",
+            Token::Minus => "minus",
+            Token::Star => "mul",
+            Token::Slash => "div",
+            Token::LParen => "lparen",
+            Token::RParen => "rparen",
+            Token::NewLine => "new line",
+            Token::EndOfFile => "end of file",
+        }
+    }
+}
+
 impl fmt::Display for Token {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            Token::Name(name) => write!(f, "{name} - name"),
-            Token::Float(number) => write!(f, "{number} - float"),
-            Token::Int(number) => write!(f, "{number} - int"),
-            Token::Plus => write!(f, "+"),
-            Token::Minus => write!(f, "-"),
-            Token::Star => write!(f, "*"),
-            Token::Slash => write!(f, "/"),
-            Token::LParen => write!(f, "("),
-            Token::RParen => write!(f, ")"),
+            Token::Name(id) => write!(f, "<id,{id}>"),
+            Token::Float(number) => write!(f, "<{number}>"),
+            Token::Int(number) => write!(f, "<{number}>"),
+            Token::Plus => write!(f, "<+>"),
+            Token::Minus => write!(f, "<->"),
+            Token::Star => write!(f, "<*>"),
+            Token::Slash => write!(f, "</>"),
+            Token::LParen => write!(f, "<(>"),
+            Token::RParen => write!(f, "<)>"),
             Token::NewLine => write!(f, "<new line>"),
             Token::EndOfFile => write!(f, "<end of file>"),
         }

src/symbols/mod.rs

@@ -1,7 +1,6 @@
 use std::collections::{hash_map, HashMap};
 
-use crate::parse::token::Token;
-
+#[derive(Debug)]
 pub struct SymbolsTable<S: std::hash::Hash + Eq> {
     symbols: HashMap<S, usize>,
     next_id: usize,
@@ -50,17 +49,3 @@ impl<'a, S: std::hash::Hash + Eq> IntoIterator for &'a SymbolsTable<S> {
         self.symbols.iter()
     }
 }
-
-impl<'a, I: Iterator<Item = &'a Token>> From<I> for SymbolsTable<String> {
-    fn from(value: I) -> Self {
-        let mut symbols = Self::new();
-
-        for token in value {
-            if let Token::Name(name) = token {
-                symbols.add(name.clone())
-            }
-        }
-
-        symbols
-    }
-}