From 4827bdb4c0d68c2702ed8a73cc5e3304d215253c Mon Sep 17 00:00:00 2001 From: lionarius Date: Fri, 4 Oct 2024 16:17:45 +0300 Subject: [PATCH] review --- src/main.rs | 13 +++---------- src/parse/lexer.rs | 31 ++++++++++++++++++++++--------- src/parse/token.rs | 38 ++++++++++++++++++++++++++++---------- src/symbols/mod.rs | 17 +---------------- 4 files changed, 54 insertions(+), 45 deletions(-) diff --git a/src/main.rs b/src/main.rs index 44b857d..de57129 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,6 @@ use std::io; use std::io::Write; -use parse::token::Token; use symbols::SymbolsTable; mod cli; @@ -12,20 +11,14 @@ fn main() -> anyhow::Result<()> { let args = cli::Args::parse(); let input = std::fs::read_to_string(&args.input).unwrap(); - let tokens = parse::lexer::make_tokenizer(&input).collect::, _>>(); + let mut symbols = SymbolsTable::default(); + let tokens = parse::lexer::make_tokenizer(&input, &mut symbols).collect::, _>>(); match tokens { Ok(tokens) => { - let symbols = SymbolsTable::from(tokens.iter().map(|(_, token, _)| token)); let mut writer_tokens = io::BufWriter::new(std::fs::File::create(&args.output_tokens)?); for (_, token, _) in tokens { - match token { - Token::Name(ref name) => match symbols.get(name) { - Some(id) => writeln!(writer_tokens, "{token} ({id})")?, - None => writeln!(writer_tokens, "{token} (undefined)")?, - }, - _ => writeln!(writer_tokens, "{token}")?, - } + writeln!(writer_tokens, "{token:>6} - {}", token.as_str())?; } let mut writer_symbols = diff --git a/src/parse/lexer.rs b/src/parse/lexer.rs index f619ad2..d607191 100644 --- a/src/parse/lexer.rs +++ b/src/parse/lexer.rs @@ -1,28 +1,35 @@ use itertools::PeekNth; +use crate::symbols::SymbolsTable; + use super::{error::LexicalError, token::Token}; pub type SpannedToken = (usize, Token, usize); pub type LexerResult = Result; #[derive(Debug)] -pub struct Lexer> { +pub struct Lexer<'s, T: Iterator> { chars: PeekNth, pos: usize, pending: Vec, + symbols: &'s mut SymbolsTable, } -pub fn make_tokenizer(input: &str) -> impl Iterator + '_ { +pub fn make_tokenizer<'s>( + input: &'s str, + symbols: &'s mut SymbolsTable, +) -> impl Iterator + 's { let chars = input.char_indices(); - Lexer::new(chars) + Lexer::new(chars, symbols) } -impl> Lexer { - pub fn new(chars: T) -> Self { +impl<'s, T: Iterator> Lexer<'s, T> { + pub fn new(chars: T, symbols: &'s mut SymbolsTable) -> Self { Self { chars: itertools::peek_nth(chars), pos: 0, pending: vec![], + symbols, } } @@ -68,7 +75,13 @@ impl> Lexer { let end = self.get_pos(); - Ok((start, Token::Name(name), end)) + if let Some(id) = self.symbols.get(&name) { + Ok((start, Token::Name(id), end)) + } else { + self.symbols.add(name.clone()); + let id = self.symbols.get(&name).unwrap(); + Ok((start, Token::Name(id), end)) + } } fn lex_number(&mut self) -> LexerResult { @@ -137,8 +150,8 @@ impl> Lexer { self.emit((start, token, end)); } - fn is_number_start(&self, c: char, c1: Option) -> bool { - c.is_ascii_digit() || (c == '.' && c1.map_or(false, |c| c.is_ascii_digit())) + fn is_number_start(&self, c: char, _c1: Option) -> bool { + c.is_ascii_digit() } fn is_digit(&self, c: char) -> bool { @@ -173,7 +186,7 @@ impl> Lexer { } } -impl> Iterator for Lexer { +impl<'s, T: Iterator> Iterator for Lexer<'s, T> { type Item = LexerResult; fn next(&mut self) -> Option { diff --git a/src/parse/token.rs b/src/parse/token.rs index 2aef946..2a910c3 100644 --- a/src/parse/token.rs +++ b/src/parse/token.rs @@ -2,7 +2,7 @@ use std::fmt; #[derive(Debug, Clone, PartialEq)] pub enum Token { - Name(String), + Name(usize), Float(f64), Int(i64), // operators @@ -18,18 +18,36 @@ pub enum Token { EndOfFile, // end of file } +impl Token { + pub fn as_str(&self) -> &str { + match self { + Token::Name(_) => "name", + Token::Float(_) => "float", + Token::Int(_) => "int", + Token::Plus => "plus", + Token::Minus => "minus", + Token::Star => "mul", + Token::Slash => "div", + Token::LParen => "lparen", + Token::RParen => "rparen", + Token::NewLine => "new line", + Token::EndOfFile => "end of file", + } + } +} + impl fmt::Display for Token { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Token::Name(name) => write!(f, "{name} - name"), - Token::Float(number) => write!(f, "{number} - float"), - Token::Int(number) => write!(f, "{number} - int"), - Token::Plus => write!(f, "+"), - Token::Minus => write!(f, "-"), - Token::Star => write!(f, "*"), - Token::Slash => write!(f, "/"), - Token::LParen => write!(f, "("), - Token::RParen => write!(f, ")"), + Token::Name(id) => write!(f, ""), + Token::Float(number) => write!(f, "<{number}>"), + Token::Int(number) => write!(f, "<{number}>"), + Token::Plus => write!(f, "<+>"), + Token::Minus => write!(f, "<->"), + Token::Star => write!(f, "<*>"), + Token::Slash => write!(f, ""), + Token::LParen => write!(f, "<(>"), + Token::RParen => write!(f, "<)>"), Token::NewLine => write!(f, ""), Token::EndOfFile => write!(f, ""), } diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index 6bb9626..e63eb1d 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -1,7 +1,6 @@ use std::collections::{hash_map, HashMap}; -use crate::parse::token::Token; - +#[derive(Debug)] pub struct SymbolsTable { symbols: HashMap, next_id: usize, @@ -50,17 +49,3 @@ impl<'a, S: std::hash::Hash + Eq> IntoIterator for &'a SymbolsTable { self.symbols.iter() } } - -impl<'a, I: Iterator> From for SymbolsTable { - fn from(value: I) -> Self { - let mut symbols = Self::new(); - - for token in value { - if let Token::Name(name) = token { - symbols.add(name.clone()) - } - } - - symbols - } -}