From 34342344f2d84211886fb7fbf7f53a3973d6c103 Mon Sep 17 00:00:00 2001 From: namibj Date: Tue, 4 Feb 2020 18:36:43 +0000 Subject: [PATCH 01/26] Intermediate state of transition the parser to nom. Broken code. --- Cargo.lock | 102 +++++++ Cargo.toml | 6 +- src/ast.rs | 12 +- src/main.rs | 6 +- src/parser.rs | 831 +++++++++++++++++++++++++------------------------- src/token.rs | 2 +- 6 files changed, 534 insertions(+), 425 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce278e7..d9bd702 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,6 +16,22 @@ dependencies = [ "const-random 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "aho-corasick" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "arrayvec" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "atty" version = "0.2.14" @@ -226,6 +242,7 @@ dependencies = [ "hashlink 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "hmap 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "jlight-vm 0.1.0", + "nom 5.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "regalloc 0.0.1", ] @@ -234,6 +251,18 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "lexical-core" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "static_assertions 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "libc" version = "0.2.66" @@ -264,6 +293,11 @@ dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "memchr" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "memoffset" version = "0.5.3" @@ -281,6 +315,23 @@ dependencies = [ "libmimalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nom" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lexical-core 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num_cpus" version = "1.12.0" @@ -351,6 +402,22 @@ dependencies = [ "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "regex" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.7.7 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "rustc-demangle" version = "0.1.16" @@ -372,6 +439,11 @@ dependencies = [ "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ryu" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "scopeguard" version = "1.0.0" @@ -395,6 +467,11 @@ name = "smallvec" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "static_assertions" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "syn" version = "1.0.14" @@ -413,11 +490,24 @@ dependencies = [ "winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "unicode-xid" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -453,6 +543,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] "checksum ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" "checksum ahash 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0989268a37e128d4d7a8028f1c60099430113fdbc70419010601ce51a228e4fe" +"checksum aho-corasick 0.7.7 (registry+https://github.com/rust-lang/crates.io-index)" = "5f56c476256dc249def911d6f7580b5fc7e875895b5d7ee88f5d602208035744" +"checksum arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" "checksum backtrace 0.3.43 (registry+https://github.com/rust-lang/crates.io-index)" = "7f80256bc78f67e7df7e36d77366f636ed976895d91fe2ab9efa3973e8fe8c4f" @@ -477,12 +569,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772" "checksum hmap 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "92fbd0445bda1ac099b76a3fbad67668f915f1873cc1d9b47c02889211fa1ace" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +"checksum lexical-core 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2304bccb228c4b020f3a4835d247df0a02a7c4686098d4167762cfbbe4c5cb14" "checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" "checksum libmimalloc-sys 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "a0ca9ff47097bebad6319fc4a55ca01d0d7fdfcb930302a181dce3e24b94e7ae" "checksum lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" "checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" "checksum mimalloc 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8057eb95231759bcd92c30c0be9f970d25d5b2f99767f5aa53c9df93264789cb" +"checksum nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +"checksum nom 5.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c433f4d505fe6ce7ff78523d2fa13a0b9f2690e181fc26168bcbe5ccc5d14e07" "checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" "checksum parking_lot 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" "checksum parking_lot_core 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" @@ -490,16 +586,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" +"checksum regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b28dfe3fe9badec5dbf0a79a9cccad2cfc2ab5484bdb3e44cbd1ae8b3ba2be06" "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" "checksum rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7540fc8b0c49f096ee9c961cda096467dce8084bec6bdca2fc83895fd9b28cb8" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +"checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" "checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" +"checksum static_assertions 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7f3eb36b47e512f8f1c9e3d10c2c1965bc992bd9cdb024fa581e2194501c83d3" "checksum syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5" "checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +"checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +"checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" "checksum wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)" = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" diff --git a/Cargo.toml b/Cargo.toml index 9f9e8ff..83afdd6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,4 +10,8 @@ edition = "2018" hmap = "0.1" jlight-vm = {path = "jlight-vm"} regalloc = {path = "jlight-vm/regalloc.rs/lib"} -hashlink = "0.4" \ No newline at end of file +hashlink = "0.4" + +[dependencies.nom] +version = "5.1.0" +features = ["regexp_macros"] diff --git a/src/ast.rs b/src/ast.rs index 595187d..12e3671 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -1,12 +1,12 @@ use crate::token::Position; -#[derive(Clone, PartialEq)] +#[derive(Clone, PartialEq, Default)] pub struct Expr { pub pos: Position, pub expr: ExprKind, } -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Default)] pub enum ExprKind { Assign(Box, Box), BinOp(Box, String, Box), @@ -32,7 +32,7 @@ pub enum ExprKind { Nil, Break, Continue, - Throw(String), + Throw(Box), ConstBool(bool), Array(Vec>), ArrayIndex(Box, Box), @@ -40,6 +40,12 @@ pub enum ExprKind { Import(String), } +impl Default for ExprKind { + fn default() -> Self { + ExprKind::Nil + } +} + use std::fmt; impl Expr { diff --git a/src/main.rs b/src/main.rs index 73630ac..d13814b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,9 +8,9 @@ fn main() { let start_time = std::time::Instant::now(); let mut ast = vec![]; let r = Reader::from_string( - " + " function fac(x) { - + if x < 2 { return 1 } else { @@ -22,7 +22,7 @@ io.writeln(fac(6)) ", ); let mut p = Parser::new(r, &mut ast); - p.parse().unwrap(); + //p.parse().unwrap(); let mut ctx = compile(ast); ctx.finalize(); let state = jlight_vm::util::arc::Arc::new(jlight_vm::runtime::state::State::new()); diff --git a/src/parser.rs b/src/parser.rs index f4df556..0169800 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3,7 +3,18 @@ use crate::lexer::*; use crate::msg::*; use crate::reader::Reader; use crate::token::*; +use nom::{ + branch::alt, + bytes::complete::{escaped, tag}, + character::complete::{alpha1, anychar, none_of, one_of}, + combinator::{map, opt}, + multi::separated_list, + number::complete::double, + sequence::{pair, preceded, terminated, tuple}, + Err, IResult, +}; use std::mem; +//#[macro_use] extern crate nom; pub struct Parser<'a> { lexer: Lexer, @@ -19,497 +30,483 @@ macro_rules! expr { }) }; } +macro_rules! exp { + ($e:expr) => { + Expr { + pos: Position::default(), + expr: $e, + } + }; +} -type EResult = Result, MsgWithPos>; +type EResult<'a> = IResult<&'a str, Expr>; //Result, MsgWithPos>; -impl<'a> Parser<'a> { - pub fn new(reader: Reader, ast: &'a mut Vec>) -> Parser<'a> { - Self { - lexer: Lexer::new(reader), - token: Token::new(TokenKind::End, Position::new(1, 1)), - ast, - } - } +fn map_str<'b>( + i: impl (Fn(&'b str) -> IResult<&'b str, &'_ str>), +) -> impl Fn(&'b str) -> IResult<&'b str, String> { + map(i, |s| s.to_string()) +} - fn init(&mut self) -> Result<(), MsgWithPos> { - self.advance_token()?; +fn expect_identifier<'b>(i: &'b str) -> IResult<&'b str, &str> { + //re_match!(i, r"[a-zA-Z_][a-zA-Z0-9_]*") + alpha1(i) +} - Ok(()) - } +fn parse_nil<'b>(i: &'b str) -> EResult { + /*let tok = self.advance_token()?; + let pos = tok.position; + if let TokenKind::Nil = tok.kind { + Ok(expr!(ExprKind::Nil, pos)) + } else { + unreachable!() + }*/ + unimplemented!() +} - pub fn parse(&mut self) -> Result<(), MsgWithPos> { - self.init()?; - while !self.token.is_eof() { - self.parse_top_level()?; - } - Ok(()) - } +fn parse_bool_literal<'b>(i: &'b str) -> EResult { + /*let tok = self.advance_token()?; + let value = tok.is(TokenKind::True); + Ok(expr!(ExprKind::ConstBool(value), tok.position))*/ + unimplemented!() +} - fn expect_token(&mut self, kind: TokenKind) -> Result { - if self.token.kind == kind { - let token = self.advance_token()?; +fn lit_int<'b>(i: &'b str) -> EResult { + /*let tok = self.advance_token()?; + let pos = tok.position; + if let TokenKind::LitInt(i, _, _) = tok.kind { + Ok(expr!(ExprKind::ConstInt(i.parse().unwrap()), pos)) + } else { + unreachable!() + }*/ + unimplemented!() +} - Ok(token) - } else { - Err(MsgWithPos::new( - self.token.position, - Msg::ExpectedToken(kind.name().into(), self.token.name()), - )) - } - } +fn lit_char<'b>(i: &'b str) -> EResult { + map(preceded(tag("'"), terminated(anychar, tag("'"))), |c| { + exp!(ExprKind::ConstChar(c)) + })(i) +} - fn parse_top_level(&mut self) -> Result<(), MsgWithPos> { - let expr = self.parse_expression()?; +fn lit_float<'b>(i: &'b str) -> EResult { + map(double, |f| exp!(ExprKind::ConstFloat(f)))(i) +} - self.ast.push(expr); - Ok(()) - } +fn lit_str<'b>(i: &'b str) -> EResult { + map( + preceded( + tag("\""), + terminated(escaped(none_of("\\\""), '"', one_of("\\\"")), tag("\"")), + ), + |s: &'b str| exp!(ExprKind::ConstStr(s.to_string())), + )(i) +} - fn parse_function_param(&mut self) -> Result { - let name = self.expect_identifier()?; - Ok(name) +pub fn parse<'b>(i: &'b str) -> Result<(), MsgWithPos> { + /* self.init()?; + while !self.token.is_eof() { + self.parse_top_level()?; } + Ok(())*/ + unimplemented!() +} - fn parse_import(&mut self) -> EResult { - let pos = self.expect_token(TokenKind::Import)?.position; - if let ExprKind::ConstStr(s) = self.lit_str()?.expr { - return Ok(expr!(ExprKind::Import(s.clone()), pos)); - } else { - unreachable!() - } - } +fn expect_token<'b>(i: &'b str, kind: TokenKind) -> Result { + /*if self.token.kind == kind { + let token = self.advance_token()?; - fn parse_function(&mut self) -> EResult { - let pos = self.expect_token(TokenKind::Fun)?.position; - let name = if let TokenKind::Identifier(_) = &self.token.kind { - Some(self.expect_identifier()?) - } else { - None - }; - self.expect_token(TokenKind::LParen)?; - let params = if self.token.kind == TokenKind::RParen { - vec![] - } else { - let mut tmp = vec![]; - while !self.token.is(TokenKind::RParen) { - tmp.push(self.expect_identifier()?); - if !self.token.is(TokenKind::RParen) { - self.expect_token(TokenKind::Comma)?; - } - } - tmp - }; - self.expect_token(TokenKind::RParen)?; - let block = self.parse_block()?; - Ok(expr!(ExprKind::Function(name, params, block), pos)) - } + Ok(token) + } else { + Err(MsgWithPos::new( + self.token.position, + Msg::ExpectedToken(kind.name().into(), self.token.name()), + )) + }*/ + unimplemented!() +} - fn parse_let(&mut self) -> EResult { - let reassignable = self.token.is(TokenKind::Var); +fn parse_top_level<'b>(i: &'b str) -> Result<(), MsgWithPos> { + /*let expr = self.parse_expression()?; - let pos = self.advance_token()?.position; - let ident = self.expect_identifier()?; - let expr = if self.token.is(TokenKind::Eq) { - self.expect_token(TokenKind::Eq)?; - let expr = self.parse_expression()?; - Some(expr) - } else { - None - }; - Ok(expr!(ExprKind::Var(reassignable, ident, expr), pos)) - } + self.ast.push(expr); + Ok(())*/ + unimplemented!() +} + +fn parse_function_param<'b>(i: &'b str) -> Result { + /*let name = self.expect_identifier()?; + Ok(name)*/ + unimplemented!() +} + +fn parse_import<'b>(i: &'b str) -> EResult { + /*let pos = self.expect_token(TokenKind::Import)?.position; + if let ExprKind::ConstStr(s) = self.lit_str()?.expr { + return Ok(expr!(ExprKind::Import(s.clone()), pos)); + } else { + unreachable!() + }*/ + unimplemented!() +} + +fn parse_function<'b>(i: &'b str) -> EResult { + let fn_arg_sep = tag(","); + let fn_arg = expect_identifier; + let tup = tuple(( + opt(map_str(expect_identifier)), + tag("("), + separated_list(fn_arg_sep, map_str(fn_arg)), + tag(")"), + parse_block, + )); + map(tup, |(name, _, params, _, block)| { + exp!(ExprKind::Function(name, params, Box::new(block))) + })(i) + //Ok(expr!(ExprKind::Function(name, params, block), pos)) +} - fn parse_return(&mut self) -> EResult { - let pos = self.expect_token(TokenKind::Return)?.position; +fn parse_let<'b>(i: &'b str) -> EResult { + /*let reassignable = self.token.is(TokenKind::Var); + + let pos = self.advance_token()?.position; + let ident = self.expect_identifier()?; + let expr = if self.token.is(TokenKind::Eq) { + self.expect_token(TokenKind::Eq)?; let expr = self.parse_expression()?; - Ok(expr!(ExprKind::Return(Some(expr)), pos)) - } + Some(expr) + } else { + None + }; + Ok(expr!(ExprKind::Var(reassignable, ident, expr), pos))*/ + unimplemented!() +} - fn parse_expression(&mut self) -> EResult { - match self.token.kind { - TokenKind::New => { - let pos = self.advance_token()?.position; - let calling = self.parse_expression()?; - Ok(expr!(ExprKind::New(calling), pos)) - } - TokenKind::Fun => self.parse_function(), - TokenKind::Match => self.parse_match(), - TokenKind::Let | TokenKind::Var => self.parse_let(), - TokenKind::LBrace => self.parse_block(), - TokenKind::If => self.parse_if(), - TokenKind::While => self.parse_while(), - TokenKind::Break => self.parse_break(), - TokenKind::Continue => self.parse_continue(), - TokenKind::Return => self.parse_return(), - TokenKind::Throw => self.parse_throw(), - TokenKind::Import => self.parse_import(), - _ => self.parse_binary(0), +fn parse_return<'b>(i: &'b str) -> EResult { + /*let pos = self.expect_token(TokenKind::Return)?.position; + let expr = self.parse_expression()?; + Ok(expr!(ExprKind::Return(Some(expr)), pos))*/ + unimplemented!() +} + +fn parse_expression<'b>(i: &'b str) -> EResult { + /*match self.token.kind { + TokenKind::New => { + let pos = self.advance_token()?.position; + let calling = self.parse_expression()?; + Ok(expr!(ExprKind::New(calling), pos)) } - } + TokenKind::Fun => self.parse_function(), + TokenKind::Match => self.parse_match(), + TokenKind::Let | TokenKind::Var => self.parse_let(), + TokenKind::LBrace => self.parse_block(), + TokenKind::If => self.parse_if(), + TokenKind::While => self.parse_while(), + TokenKind::Break => self.parse_break(), + TokenKind::Continue => self.parse_continue(), + TokenKind::Return => self.parse_return(), + TokenKind::Throw => self.parse_throw(), + TokenKind::Import => self.parse_import(), + _ => self.parse_binary(0), + }*/ + unimplemented!() +} - fn parse_self(&mut self) -> EResult { - let pos = self.expect_token(TokenKind::This)?.position; - Ok(expr!(ExprKind::This, pos)) - } +fn parse_self<'b>(i: &'b str) -> EResult { + map(tag("self"), |s| exp!(ExprKind::This))(i) +} - fn parse_break(&mut self) -> EResult { - let pos = self.expect_token(TokenKind::Break)?.position; - Ok(expr!(ExprKind::Break, pos)) - } - fn parse_continue(&mut self) -> EResult { - let pos = self.expect_token(TokenKind::Continue)?.position; - Ok(expr!(ExprKind::Continue, pos)) - } +fn parse_break<'b>(i: &'b str) -> EResult { + map(tag("break"), |s| exp!(ExprKind::Break))(i) +} - fn parse_throw(&mut self) -> EResult { - let pos = self.token.position; - if let TokenKind::String(s) = self.token.clone().kind { - self.advance_token()?; - return Ok(expr!(ExprKind::Throw(s.clone()), pos)); - } else { - panic!("String expected at {}", pos) - } - } +fn parse_continue<'b>(i: &'b str) -> EResult { + map(tag("continue"), |s| exp!(ExprKind::Continue))(i) +} - fn parse_while(&mut self) -> EResult { - let pos = self.expect_token(TokenKind::While)?.position; - let cond = self.parse_expression()?; - let block = self.parse_block()?; - Ok(expr!(ExprKind::While(cond, block), pos)) - } +fn parse_throw<'b>(i: &'b str) -> EResult { + map(pair(tag("throw"), parse_expression), |(s, expr)| { + exp!(ExprKind::Throw(Box::new(expr))) + })(i) +} - fn parse_match(&mut self) -> EResult { - let pos = self.expect_token(TokenKind::Match)?.position; - let value = self.parse_expression()?; - self.expect_token(TokenKind::LBrace)?; - let mut data = vec![]; - let mut or = None; - while !self.token.is(TokenKind::RBrace) && !self.token.is_eof() { - if self.token.is(TokenKind::Underscore) { - self.expect_token(TokenKind::Underscore)?; - self.expect_token(TokenKind::Arrow)?; - let expr = self.parse_expression()?; - or = Some(expr); - continue; - } - let cond = self.parse_expression()?; +fn parse_while<'b>(i: &'b str) -> EResult { + /*let pos = self.expect_token(TokenKind::While)?.position; + let cond = self.parse_expression()?; + let block = self.parse_block()?; + Ok(expr!(ExprKind::While(cond, block), pos))*/ + unimplemented!() +} + +fn parse_match<'b>(i: &'b str) -> EResult { + /*let pos = self.expect_token(TokenKind::Match)?.position; + let value = self.parse_expression()?; + self.expect_token(TokenKind::LBrace)?; + let mut data = vec![]; + let mut or = None; + while !self.token.is(TokenKind::RBrace) && !self.token.is_eof() { + if self.token.is(TokenKind::Underscore) { + self.expect_token(TokenKind::Underscore)?; self.expect_token(TokenKind::Arrow)?; let expr = self.parse_expression()?; - data.push((cond, expr)); + or = Some(expr); + continue; } - - self.expect_token(TokenKind::RBrace)?; - - Ok(expr!(ExprKind::Match(value, data, or), pos)) + let cond = self.parse_expression()?; + self.expect_token(TokenKind::Arrow)?; + let expr = self.parse_expression()?; + data.push((cond, expr)); } - fn parse_if(&mut self) -> EResult { - let pos = self.expect_token(TokenKind::If)?.position; - let cond = self.parse_expression()?; - let then_block = self.parse_expression()?; - let else_block = if self.token.is(TokenKind::Else) { - self.advance_token()?; + self.expect_token(TokenKind::RBrace)?; - if self.token.is(TokenKind::If) { - let if_block = self.parse_if()?; - let block = expr!(ExprKind::Block(vec![if_block]), if_block.pos); + Ok(expr!(ExprKind::Match(value, data, or), pos))*/ + unimplemented!() +} - Some(block) - } else { - Some(self.parse_expression()?) - } - } else { - None - }; +fn parse_if<'b>(i: &'b str) -> EResult { + /*let pos = self.expect_token(TokenKind::If)?.position; + let cond = self.parse_expression()?; + let then_block = self.parse_expression()?; + let else_block = if self.token.is(TokenKind::Else) { + self.advance_token()?; - Ok(expr!(ExprKind::If(cond, then_block, else_block), pos)) - } + if self.token.is(TokenKind::If) { + let if_block = self.parse_if()?; + let block = expr!(ExprKind::Block(vec![if_block]), if_block.pos); - fn parse_block(&mut self) -> EResult { - let pos = self.expect_token(TokenKind::LBrace)?.position; - let mut exprs = vec![]; - while !self.token.is(TokenKind::RBrace) && !self.token.is_eof() { - let expr = self.parse_expression()?; - exprs.push(expr); + Some(block) + } else { + Some(self.parse_expression()?) } - self.expect_token(TokenKind::RBrace)?; - Ok(expr!(ExprKind::Block(exprs), pos)) - } + } else { + None + }; - fn create_binary(&mut self, tok: Token, left: Box, right: Box) -> Box { - let op = match tok.kind { - TokenKind::Eq => return expr!(ExprKind::Assign(left, right), tok.position), - TokenKind::Or => "||", - TokenKind::And => "&&", - TokenKind::BitOr => "|", - TokenKind::BitAnd => "&", - TokenKind::EqEq => "==", - TokenKind::Ne => "!=", - TokenKind::Lt => "<", - TokenKind::Gt => ">", - TokenKind::Le => "<=", - TokenKind::Ge => ">=", - TokenKind::Caret => "^", - TokenKind::Add => "+", - TokenKind::Sub => "-", - TokenKind::Mul => "*", - TokenKind::Div => "/", - TokenKind::LtLt => "<<", - TokenKind::GtGt => ">>", - TokenKind::Mod => "%", - _ => unimplemented!(), - }; + Ok(expr!(ExprKind::If(cond, then_block, else_block), pos))*/ + unimplemented!() +} - expr!(ExprKind::BinOp(left, op.to_owned(), right), tok.position) +fn parse_block<'b>(i: &'b str) -> EResult { + /*let pos = self.expect_token(TokenKind::LBrace)?.position; + let mut exprs = vec![]; + while !self.token.is(TokenKind::RBrace) && !self.token.is_eof() { + let expr = self.parse_expression()?; + exprs.push(expr); } + self.expect_token(TokenKind::RBrace)?; + Ok(expr!(ExprKind::Block(exprs), pos))*/ + unimplemented!() +} - fn parse_binary(&mut self, precedence: u32) -> EResult { - let mut left = self.parse_unary()?; - loop { - let right_precedence = match self.token.kind { - TokenKind::Or => 1, - TokenKind::And => 2, - TokenKind::Eq => 3, - TokenKind::EqEq - | TokenKind::Ne - | TokenKind::Lt - | TokenKind::Le - | TokenKind::Gt - | TokenKind::Ge => 4, - TokenKind::BitOr | TokenKind::BitAnd | TokenKind::Caret => 6, - TokenKind::LtLt | TokenKind::GtGt | TokenKind::Add | TokenKind::Sub => 8, - TokenKind::Mul | TokenKind::Div | TokenKind::Mod => 9, - _ => { - return Ok(left); - } - }; - if precedence >= right_precedence { - return Ok(left); - } +fn create_binary(tok: Token, left: Box, right: Box) -> Box { + /*let op = match tok.kind { + TokenKind::Eq => return expr!(ExprKind::Assign(left, right), tok.position), + TokenKind::Or => "||", + TokenKind::And => "&&", + TokenKind::BitOr => "|", + TokenKind::BitAnd => "&", + TokenKind::EqEq => "==", + TokenKind::Ne => "!=", + TokenKind::Lt => "<", + TokenKind::Gt => ">", + TokenKind::Le => "<=", + TokenKind::Ge => ">=", + TokenKind::Caret => "^", + TokenKind::Add => "+", + TokenKind::Sub => "-", + TokenKind::Mul => "*", + TokenKind::Div => "/", + TokenKind::LtLt => "<<", + TokenKind::GtGt => ">>", + TokenKind::Mod => "%", + _ => unimplemented!(), + }; - let tok = self.advance_token()?; - left = { - let right = self.parse_binary(right_precedence)?; - self.create_binary(tok, left, right) - }; - } - } + expr!(ExprKind::BinOp(left, op.to_owned(), right), tok.position)*/ + unimplemented!() +} - pub fn parse_unary(&mut self) -> EResult { - match self.token.kind { - TokenKind::Add | TokenKind::Sub | TokenKind::Not => { - let tok = self.advance_token()?; - let op = match tok.kind { - TokenKind::Add => String::from("+"), - TokenKind::Sub => String::from("-"), - TokenKind::Not => String::from("!"), - _ => unreachable!(), - }; - let expr = self.parse_primary()?; - Ok(expr!(ExprKind::Unop(op, expr), tok.position)) +fn parse_binary<'b>(i: &'b str, precedence: u32) -> EResult { + /*let mut left = self.parse_unary()?; + loop { + let right_precedence = match self.token.kind { + TokenKind::Or => 1, + TokenKind::And => 2, + TokenKind::Eq => 3, + TokenKind::EqEq + | TokenKind::Ne + | TokenKind::Lt + | TokenKind::Le + | TokenKind::Gt + | TokenKind::Ge => 4, + TokenKind::BitOr | TokenKind::BitAnd | TokenKind::Caret => 6, + TokenKind::LtLt | TokenKind::GtGt | TokenKind::Add | TokenKind::Sub => 8, + TokenKind::Mul | TokenKind::Div | TokenKind::Mod => 9, + _ => { + return Ok(left); } - _ => self.parse_primary(), + }; + if precedence >= right_precedence { + return Ok(left); } - } - /*pub fn parse_expression(&mut self) -> EResult { - self.parse_binary(0) + let tok = self.advance_token()?; + left = { + let right = self.parse_binary(right_precedence)?; + self.create_binary(tok, left, right) + }; }*/ + unimplemented!() +} - fn parse_call(&mut self) -> EResult { - let expr = self.parse_expression()?; - - self.expect_token(TokenKind::LParen)?; +pub fn parse_unary<'b>(i: &'b str) -> EResult<'b> { + map(pair(one_of("+-!"), parse_primary), |(op, expr)| { + exp!(ExprKind::Unop(op.to_string(), Box::new(expr))) + })(i) +} - let args = self.parse_comma_list(TokenKind::RParen, |p| p.parse_expression())?; +/*pub fn parse_expression(&mut self) -> EResult { + self.parse_binary(0) +}*/ - Ok(expr!(ExprKind::Call(expr, args), expr.pos)) - } +fn parse_call<'b>(i: &'b str) -> EResult { + /*let expr = self.parse_expression()?; - pub fn parse_primary(&mut self) -> EResult { - let mut left = self.parse_factor()?; - loop { - left = match self.token.kind { - TokenKind::Dot => { - let tok = self.advance_token()?; - let ident = self.expect_identifier()?; - expr!(ExprKind::Access(left, ident), tok.position) - } - TokenKind::LBracket => { - let tok = self.advance_token()?; - let index = self.parse_expression()?; - self.expect_token(TokenKind::RBracket)?; - expr!(ExprKind::ArrayIndex(left, index), tok.position) - } - _ => { - if self.token.is(TokenKind::LParen) { - let expr = left; + self.expect_token(TokenKind::LParen)?; - self.expect_token(TokenKind::LParen)?; + let args = self.parse_comma_list(TokenKind::RParen, |p| p.parse_expression())?; - let args = - self.parse_comma_list(TokenKind::RParen, |p| p.parse_expression())?; + Ok(expr!(ExprKind::Call(expr, args), expr.pos))*/ + unimplemented!() +} - expr!(ExprKind::Call(expr, args), expr.pos) - } else { - return Ok(left); - } - } +pub fn parse_primary<'b>(i: &'b str) -> EResult { + /*let mut left = self.parse_factor()?; + loop { + left = match self.token.kind { + TokenKind::Dot => { + let tok = self.advance_token()?; + let ident = self.expect_identifier()?; + expr!(ExprKind::Access(left, ident), tok.position) } - } - } + TokenKind::LBracket => { + let tok = self.advance_token()?; + let index = self.parse_expression()?; + self.expect_token(TokenKind::RBracket)?; + expr!(ExprKind::ArrayIndex(left, index), tok.position) + } + _ => { + if self.token.is(TokenKind::LParen) { + let expr = left; - fn expect_identifier(&mut self) -> Result { - let tok = self.advance_token()?; + self.expect_token(TokenKind::LParen)?; - if let TokenKind::Identifier(ref value) = tok.kind { - Ok(value.to_owned()) - } else { - Err(MsgWithPos::new( - tok.position, - Msg::ExpectedIdentifier(tok.name()), - )) - } - } + let args = + self.parse_comma_list(TokenKind::RParen, |p| p.parse_expression())?; - fn parse_comma_list( - &mut self, - stop: TokenKind, - mut parse: F, - ) -> Result, MsgWithPos> - where - F: FnMut(&mut Parser) -> Result, - { - let mut data = vec![]; - let mut comma = true; - - while !self.token.is(stop.clone()) && !self.token.is_eof() { - if !comma { - return Err(MsgWithPos::new( - self.token.position, - Msg::ExpectedToken(TokenKind::Comma.name().into(), self.token.name()), - )); + expr!(ExprKind::Call(expr, args), expr.pos) + } else { + return Ok(left); + } } + } + }*/ + unimplemented!() +} - let entry = parse(self)?; - data.push(entry); - - comma = self.token.is(TokenKind::Comma); - if comma { - self.advance_token()?; - } +fn parse_comma_list( + // &mut self, + stop: TokenKind, + mut parse: F, +) -> Result, MsgWithPos> +where + F: FnMut(&mut Parser) -> Result, +{ + /*let mut data = vec![]; + let mut comma = true; + + while !self.token.is(stop.clone()) && !self.token.is_eof() { + if !comma { + return Err(MsgWithPos::new( + self.token.position, + Msg::ExpectedToken(TokenKind::Comma.name().into(), self.token.name()), + )); } - self.expect_token(stop)?; + let entry = parse(self)?; + data.push(entry); - Ok(data) + comma = self.token.is(TokenKind::Comma); + if comma { + self.advance_token()?; + } } - fn advance_token(&mut self) -> Result { - let tok = self.lexer.read_token()?; + self.expect_token(stop)?; - Ok(mem::replace(&mut self.token, tok)) - } + Ok(data)*/ + unimplemented!() +} - fn parse_lambda(&mut self) -> EResult { - let tok = self.advance_token()?; - let params = if tok.kind == TokenKind::Or { - vec![] - } else { - self.parse_comma_list(TokenKind::BitOr, |f| f.parse_function_param())? - }; +fn advance_token(/*&mut self*/) -> Result { + /*let tok = self.lexer.read_token()?; - let block = self.parse_expression()?; - Ok(expr!(ExprKind::Lambda(params, block), tok.position)) - } - pub fn parse_factor(&mut self) -> EResult { - let expr = match self.token.kind { - TokenKind::Fun => self.parse_function(), - TokenKind::LParen => self.parse_parentheses(), - TokenKind::LitChar(_) => self.lit_char(), - TokenKind::LitInt(_, _, _) => self.lit_int(), - TokenKind::LitFloat(_) => self.lit_float(), - TokenKind::String(_) => self.lit_str(), - TokenKind::Identifier(_) => self.ident(), - TokenKind::This => self.parse_self(), - TokenKind::BitOr | TokenKind::Or => self.parse_lambda(), - TokenKind::True => self.parse_bool_literal(), - TokenKind::False => self.parse_bool_literal(), - TokenKind::Nil => self.parse_nil(), - _ => Err(MsgWithPos::new( - self.token.position, - Msg::ExpectedFactor(self.token.name().clone()), - )), - }; + Ok(mem::replace(&mut self.token, tok))*/ + unimplemented!() +} - expr - } +fn parse_lambda<'b>(i: &'b str) -> EResult { + /*let tok = self.advance_token()?; + let params = if tok.kind == TokenKind::Or { + vec![] + } else { + self.parse_comma_list(TokenKind::BitOr, |f| f.parse_function_param())? + }; - fn parse_parentheses(&mut self) -> EResult { - self.advance_token()?; - let expr = self.parse_expression(); - self.expect_token(TokenKind::RParen)?; - expr - } + let block = self.parse_expression()?; + Ok(expr!(ExprKind::Lambda(params, block), tok.position))*/ + unimplemented!() +} +pub fn parse_factor<'b>(i: &'b str) -> EResult { + alt(( + preceded(tag("function"), parse_function), + parse_parentheses, + lit_char, + lit_int, + lit_float, + lit_str, + parse_self, + ident, + parse_lambda, + parse_bool_literal, + parse_bool_literal, + parse_nil, + ))(i) +} - fn parse_nil(&mut self) -> EResult { - let tok = self.advance_token()?; - let pos = tok.position; - if let TokenKind::Nil = tok.kind { - Ok(expr!(ExprKind::Nil, pos)) - } else { - unreachable!() - } - } +fn parse_parentheses<'b>(i: &'b str) -> EResult { + preceded(tag("("), terminated(parse_expression, tag(")")))(i) +} - fn parse_bool_literal(&mut self) -> EResult { - let tok = self.advance_token()?; - let value = tok.is(TokenKind::True); - Ok(expr!(ExprKind::ConstBool(value), tok.position)) - } +fn ident<'b>(i: &'b str) -> EResult { + /*let pos = self.token.position; + let ident = self.expect_identifier()?; - fn lit_int(&mut self) -> EResult { - let tok = self.advance_token()?; - let pos = tok.position; - if let TokenKind::LitInt(i, _, _) = tok.kind { - Ok(expr!(ExprKind::ConstInt(i.parse().unwrap()), pos)) - } else { - unreachable!() - } - } + Ok(expr!(ExprKind::Ident(ident), pos))*/ + unimplemented!() +} - fn lit_char(&mut self) -> EResult { - let tok = self.advance_token()?; - let pos = tok.position; - if let TokenKind::LitChar(c) = tok.kind { - Ok(expr!(ExprKind::ConstChar(c), pos)) - } else { - unreachable!() - } - } - fn lit_float(&mut self) -> EResult { - let tok = self.advance_token()?; - let pos = tok.position; - if let TokenKind::LitFloat(c) = tok.kind { - Ok(expr!(ExprKind::ConstFloat(c.parse().unwrap()), pos)) - } else { - unreachable!() - } - } - fn lit_str(&mut self) -> EResult { - let tok = self.advance_token()?; - let pos = tok.position; - if let TokenKind::String(s) = tok.kind { - Ok(expr!(ExprKind::ConstStr(s), pos)) - } else { - unreachable!() +impl<'a> Parser<'a> { + pub fn new(reader: Reader, ast: &'a mut Vec>) -> Parser<'a> { + Self { + lexer: Lexer::new(reader), + token: Token::new(TokenKind::End, Position::new(1, 1)), + ast, } } - fn ident(&mut self) -> EResult { - let pos = self.token.position; - let ident = self.expect_identifier()?; + fn init(&mut self) -> Result<(), MsgWithPos> { + //self.advance_token()?; - Ok(expr!(ExprKind::Ident(ident), pos)) + Ok(()) } } diff --git a/src/token.rs b/src/token.rs index ac84ed4..d716141 100644 --- a/src/token.rs +++ b/src/token.rs @@ -1,7 +1,7 @@ use std::fmt; use std::result::Result; -#[derive(Clone, Debug, PartialEq, Eq, Copy)] +#[derive(Clone, Debug, PartialEq, Eq, Copy, Default)] pub struct Position { pub line: u32, pub column: u32, From ea217ad6745c4a5aaaddfd5bd59e74cfe1ad7afc Mon Sep 17 00:00:00 2001 From: namibj Date: Tue, 4 Feb 2020 21:40:30 +0000 Subject: [PATCH 02/26] Remove erraneous `#[derive(Default)]` on `enum ExprKind`. --- src/ast.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast.rs b/src/ast.rs index 3c3c38c..a266b4f 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -6,7 +6,7 @@ pub struct Expr { pub expr: ExprKind, } -#[derive(Clone, Debug, PartialEq, Default)] +#[derive(Clone, Debug, PartialEq)] pub enum ExprKind { Assign(Box, Box), BinOp(Box, String, Box), From 638b8a7b68acb80e29e789407f6b832304118124 Mon Sep 17 00:00:00 2001 From: namibj Date: Tue, 4 Feb 2020 21:49:37 +0000 Subject: [PATCH 03/26] Create rust.yml --- .github/workflows/rust.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..8ea7097 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,15 @@ +name: Rust + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Build + run: cargo build --frozen --verbose + - name: Run tests + run: cargo test --frozen --verbose From c2a063efb44ed7f1b95b0c52344bbf6e9c5700aa Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 5 Feb 2020 00:30:54 +0000 Subject: [PATCH 04/26] Implement parse_nil --- src/parser.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index ca03076..4a1b268 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3,6 +3,7 @@ use crate::lexer::*; use crate::msg::*; use crate::reader::Reader; use crate::token::*; +use nom::combinator::value; use nom::{ branch::alt, bytes::complete::{escaped, tag}, @@ -53,14 +54,7 @@ fn expect_identifier<'b>(i: &'b str) -> IResult<&'b str, &str> { } fn parse_nil<'b>(i: &'b str) -> EResult { - /*let tok = self.advance_token()?; - let pos = tok.position; - if let TokenKind::Nil = tok.kind { - Ok(expr!(ExprKind::Nil, pos)) - } else { - unreachable!() - }*/ - unimplemented!() + value(exp!(ExprKind::Nil), tag("nil"))(i) } fn parse_bool_literal<'b>(i: &'b str) -> EResult { From 5c4497741ebdcff3839366e2691ecc40c667e6e1 Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 5 Feb 2020 00:31:18 +0000 Subject: [PATCH 05/26] Implement parse_bool_literal --- src/parser.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 4a1b268..c04fec7 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -58,10 +58,10 @@ fn parse_nil<'b>(i: &'b str) -> EResult { } fn parse_bool_literal<'b>(i: &'b str) -> EResult { - /*let tok = self.advance_token()?; - let value = tok.is(TokenKind::True); - Ok(expr!(ExprKind::ConstBool(value), tok.position))*/ - unimplemented!() + let parse_true = value(exp!(ExprKind::ConstBool(true)), tag("true")); + let parse_false = value(exp!(ExprKind::ConstBool(false)), tag("false")); + + alt((parse_true, parse_false))(i) } fn lit_int<'b>(i: &'b str) -> EResult { From 230707c010455553e9e06bd5dfe24e6b9bb0b148 Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 5 Feb 2020 00:31:43 +0000 Subject: [PATCH 06/26] Remove parse_import --- src/parser.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index c04fec7..7b8db56 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -132,16 +132,6 @@ fn parse_function_param<'b>(i: &'b str) -> Result { unimplemented!() } -fn parse_import<'b>(i: &'b str) -> EResult { - /*let pos = self.expect_token(TokenKind::Import)?.position; - if let ExprKind::ConstStr(s) = self.lit_str()?.expr { - return Ok(expr!(ExprKind::Import(s.clone()), pos)); - } else { - unreachable!() - }*/ - unimplemented!() -} - fn parse_function<'b>(i: &'b str) -> EResult { let fn_arg_sep = tag(","); let fn_arg = expect_identifier; From 22b983a6ac3251ab9ef2e99f23584d7853ff1e8f Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 5 Feb 2020 00:33:07 +0000 Subject: [PATCH 07/26] Implement parse_let Borrowchecker avoided via copy & paste. Fix desired. --- src/parser.rs | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 7b8db56..4937191 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -149,19 +149,27 @@ fn parse_function<'b>(i: &'b str) -> EResult { } fn parse_let<'b>(i: &'b str) -> EResult { - /*let reassignable = self.token.is(TokenKind::Var); - - let pos = self.advance_token()?.position; - let ident = self.expect_identifier()?; - let expr = if self.token.is(TokenKind::Eq) { - self.expect_token(TokenKind::Eq)?; - let expr = self.parse_expression()?; - Some(expr) - } else { - None - }; - Ok(expr!(ExprKind::Var(reassignable, ident, expr), pos))*/ - unimplemented!() + //let reassignable = alt((value(true, tag("var")), value(false, tag("let")))); + + let initialization = map( + tuple(( + //reassignable, + alt((value(true, tag("var")), value(false, tag("let")))), + map_str(expect_identifier), + tag("="), + map(parse_expression, |expr| Some(Box::new(expr))), + )), + |(r, i, _, e)| exp!(ExprKind::Var(r, i, e)), + ); + let declaration = map( + tuple(( + //reassignable, + alt((value(true, tag("var")), value(false, tag("let")))), + map_str(expect_identifier), + )), + |(r, i)| exp!(ExprKind::Var(r, i, None)), + ); + alt((initialization, declaration))(i) } fn parse_return<'b>(i: &'b str) -> EResult { From 5ad2e44cc6a4d7d5752877a899c568ddaa97c9af Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 5 Feb 2020 00:33:28 +0000 Subject: [PATCH 08/26] Implement parse_return --- src/parser.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 4937191..05c8267 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -173,10 +173,12 @@ fn parse_let<'b>(i: &'b str) -> EResult { } fn parse_return<'b>(i: &'b str) -> EResult { - /*let pos = self.expect_token(TokenKind::Return)?.position; - let expr = self.parse_expression()?; - Ok(expr!(ExprKind::Return(Some(expr)), pos))*/ - unimplemented!() + map(pair(tag("return"), opt(parse_expression)), |(_, expr)| { + exp!(ExprKind::Return(match expr { + Some(expr) => Some(Box::new(expr)), + None => None, + })) + })(i) } fn parse_expression<'b>(i: &'b str) -> EResult { From ee538a6bf9e80f4f8a9d2565f14e344cbdbd2513 Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 5 Feb 2020 00:34:08 +0000 Subject: [PATCH 09/26] Fix unused variable warnings --- src/parser.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 05c8267..a059166 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -204,19 +204,19 @@ fn parse_expression<'b>(i: &'b str) -> EResult { } fn parse_self<'b>(i: &'b str) -> EResult { - map(tag("self"), |s| exp!(ExprKind::This))(i) + value(exp!(ExprKind::This), tag("self"))(i) } fn parse_break<'b>(i: &'b str) -> EResult { - map(tag("break"), |s| exp!(ExprKind::Break))(i) + value(exp!(ExprKind::Break), tag("break"))(i) } fn parse_continue<'b>(i: &'b str) -> EResult { - map(tag("continue"), |s| exp!(ExprKind::Continue))(i) + value(exp!(ExprKind::Continue), tag("continue"))(i) } fn parse_throw<'b>(i: &'b str) -> EResult { - map(pair(tag("throw"), parse_expression), |(s, expr)| { + map(pair(tag("throw"), parse_expression), |(_, expr)| { exp!(ExprKind::Throw(Box::new(expr))) })(i) } From 91f23207315090bd6ee53020ffc16e66f8f89d1a Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 5 Feb 2020 15:05:26 +0000 Subject: [PATCH 10/26] Implement parse_while --- src/parser.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index a059166..284e43b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -222,11 +222,9 @@ fn parse_throw<'b>(i: &'b str) -> EResult { } fn parse_while<'b>(i: &'b str) -> EResult { - /*let pos = self.expect_token(TokenKind::While)?.position; - let cond = self.parse_expression()?; - let block = self.parse_block()?; - Ok(expr!(ExprKind::While(cond, block), pos))*/ - unimplemented!() + map(tuple((tag("while"), parse_expression, parse_block)), |(_, expr, block)| { + exp!(ExprKind::While(Box::new(expr), Box::new(block))) + })(i) } fn parse_match<'b>(i: &'b str) -> EResult { From 79fb4d3169063e2fc0146b567b8ca8ffae2900ac Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 5 Feb 2020 15:05:50 +0000 Subject: [PATCH 11/26] Refactor parse_return --- src/parser.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 284e43b..028578e 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -174,10 +174,7 @@ fn parse_let<'b>(i: &'b str) -> EResult { fn parse_return<'b>(i: &'b str) -> EResult { map(pair(tag("return"), opt(parse_expression)), |(_, expr)| { - exp!(ExprKind::Return(match expr { - Some(expr) => Some(Box::new(expr)), - None => None, - })) + exp!(ExprKind::Return(expr.map(Box::new))) })(i) } From 6d221935a51a8acbc31ce77757244fea9a33042c Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 5 Feb 2020 15:28:59 +0000 Subject: [PATCH 12/26] Implement parse_expression --- src/parser.rs | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 028578e..0cf80ad 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -179,25 +179,21 @@ fn parse_return<'b>(i: &'b str) -> EResult { } fn parse_expression<'b>(i: &'b str) -> EResult { - /*match self.token.kind { - TokenKind::New => { - let pos = self.advance_token()?.position; - let calling = self.parse_expression()?; - Ok(expr!(ExprKind::New(calling), pos)) - } - TokenKind::Fun => self.parse_function(), - TokenKind::Match => self.parse_match(), - TokenKind::Let | TokenKind::Var => self.parse_let(), - TokenKind::LBrace => self.parse_block(), - TokenKind::If => self.parse_if(), - TokenKind::While => self.parse_while(), - TokenKind::Break => self.parse_break(), - TokenKind::Continue => self.parse_continue(), - TokenKind::Return => self.parse_return(), - TokenKind::Throw => self.parse_throw(), - _ => self.parse_binary(0), - }*/ - unimplemented!() + let parse_new = preceded(tag("new"), parse_expression); + alt(( + parse_new, + parse_function, + parse_match, + parse_let, + parse_block, + parse_if, + parse_while, + parse_break, + parse_continue, + parse_return, + parse_throw, + |i| parse_binary(i, 0), + ))(i) } fn parse_self<'b>(i: &'b str) -> EResult { From d5cbe4d88045435da2fa539b5e9ff6b4dd7f9e38 Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 5 Feb 2020 15:29:18 +0000 Subject: [PATCH 13/26] Formatting/cleanup (cosmetic) --- src/parser.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 0cf80ad..1441411 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -145,7 +145,6 @@ fn parse_function<'b>(i: &'b str) -> EResult { map(tup, |(name, _, params, _, block)| { exp!(ExprKind::Function(name, params, Box::new(block))) })(i) - //Ok(expr!(ExprKind::Function(name, params, block), pos)) } fn parse_let<'b>(i: &'b str) -> EResult { @@ -215,9 +214,10 @@ fn parse_throw<'b>(i: &'b str) -> EResult { } fn parse_while<'b>(i: &'b str) -> EResult { - map(tuple((tag("while"), parse_expression, parse_block)), |(_, expr, block)| { - exp!(ExprKind::While(Box::new(expr), Box::new(block))) - })(i) + map( + tuple((tag("while"), parse_expression, parse_block)), + |(_, expr, block)| exp!(ExprKind::While(Box::new(expr), Box::new(block))), + )(i) } fn parse_match<'b>(i: &'b str) -> EResult { From 88310f9b424c33ded3866ff3be90d941a5c05f6b Mon Sep 17 00:00:00 2001 From: namibj Date: Thu, 6 Feb 2020 17:54:49 +0000 Subject: [PATCH 14/26] Implement parse_lambda. Close #7 --- src/parser.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 1441411..81ceac8 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -439,17 +439,19 @@ fn advance_token(/*&mut self*/) -> Result { } fn parse_lambda<'b>(i: &'b str) -> EResult { - /*let tok = self.advance_token()?; - let params = if tok.kind == TokenKind::Or { - vec![] - } else { - self.parse_comma_list(TokenKind::BitOr, |f| f.parse_function_param())? - }; - - let block = self.parse_expression()?; - Ok(expr!(ExprKind::Lambda(params, block), tok.position))*/ - unimplemented!() + let fn_arg_sep = tag(","); + let fn_arg = expect_identifier; + let tup = tuple(( + tag("|"), + separated_list(fn_arg_sep, map_str(fn_arg)), + tag("|"), + parse_expression, + )); + map(tup, |(_, params, _, block)| { + exp!(ExprKind::Lambda(params, Box::new(block))) + })(i) } + pub fn parse_factor<'b>(i: &'b str) -> EResult { alt(( preceded(tag("function"), parse_function), From 3178be3d211abb220860d9dacf2547adc3f08359 Mon Sep 17 00:00:00 2001 From: namibj Date: Thu, 6 Feb 2020 18:09:35 +0000 Subject: [PATCH 15/26] Implement parse_if. Close #2 --- src/parser.rs | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 81ceac8..40fa031 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -247,26 +247,20 @@ fn parse_match<'b>(i: &'b str) -> EResult { } fn parse_if<'b>(i: &'b str) -> EResult { - /*let pos = self.expect_token(TokenKind::If)?.position; - let cond = self.parse_expression()?; - let then_block = self.parse_expression()?; - let else_block = if self.token.is(TokenKind::Else) { - self.advance_token()?; - - if self.token.is(TokenKind::If) { - let if_block = self.parse_if()?; - let block = expr!(ExprKind::Block(vec![if_block]), if_block.pos); - - Some(block) - } else { - Some(self.parse_expression()?) - } - } else { - None - }; - - Ok(expr!(ExprKind::If(cond, then_block, else_block), pos))*/ - unimplemented!() + map( + tuple(( + preceded(tag("if"), parse_expression), + preceded(tag("then"), parse_expression), + opt(preceded(tag("else"), parse_expression)), + )), + |(cond, then_block, else_block)| { + exp!(ExprKind::If( + Box::new(cond), + Box::new(then_block), + else_block.map(Box::new) + )) + }, + )(i) } fn parse_block<'b>(i: &'b str) -> EResult { From 82e29b7755b0cae8b12f460f6cfe355e105c5f1d Mon Sep 17 00:00:00 2001 From: namibj Date: Thu, 6 Feb 2020 18:26:52 +0000 Subject: [PATCH 16/26] Implement parse_block. Close #3 --- src/parser.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 40fa031..fe8e6d2 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -9,7 +9,7 @@ use nom::{ bytes::complete::{escaped, tag}, character::complete::{alpha1, anychar, none_of, one_of}, combinator::{map, opt}, - multi::separated_list, + multi::{many0, separated_list}, number::complete::double, sequence::{pair, preceded, terminated, tuple}, Err, IResult, @@ -264,15 +264,16 @@ fn parse_if<'b>(i: &'b str) -> EResult { } fn parse_block<'b>(i: &'b str) -> EResult { - /*let pos = self.expect_token(TokenKind::LBrace)?.position; - let mut exprs = vec![]; - while !self.token.is(TokenKind::RBrace) && !self.token.is_eof() { - let expr = self.parse_expression()?; - exprs.push(expr); - } - self.expect_token(TokenKind::RBrace)?; - Ok(expr!(ExprKind::Block(exprs), pos))*/ - unimplemented!() + map( + preceded( + tag("{"), + terminated( + many0(map(parse_expression, |expr| Box::new(expr))), + tag("}"), + ), + ), + |exprs| exp!(ExprKind::Block(exprs)), + )(i) } fn create_binary(tok: Token, left: Box, right: Box) -> Box { From b79c922bd44ecf2c2cca446ff964f481811e6e84 Mon Sep 17 00:00:00 2001 From: namibj Date: Thu, 6 Feb 2020 23:34:14 +0000 Subject: [PATCH 17/26] Implement parse_call. Close #5 --- src/parser.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index fe8e6d2..9474315 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -348,14 +348,17 @@ pub fn parse_unary<'b>(i: &'b str) -> EResult<'b> { }*/ fn parse_call<'b>(i: &'b str) -> EResult { - /*let expr = self.parse_expression()?; - - self.expect_token(TokenKind::LParen)?; - - let args = self.parse_comma_list(TokenKind::RParen, |p| p.parse_expression())?; - - Ok(expr!(ExprKind::Call(expr, args), expr.pos))*/ - unimplemented!() + let fn_arg_sep = tag(","); + let fn_arg = parse_expression; + let tup = tuple(( + parse_expression, + tag("("), + separated_list(fn_arg_sep, map(fn_arg, Box::new)), + tag(")"), + )); + map(tup, |(expr, _, params, _)| { + exp!(ExprKind::Call(Box::new(expr), params)) + })(i) } pub fn parse_primary<'b>(i: &'b str) -> EResult { From 97771e96a944abf1ee363542a82c5f0f609f8c99 Mon Sep 17 00:00:00 2001 From: namibj Date: Sat, 8 Feb 2020 01:13:26 +0000 Subject: [PATCH 18/26] Add creates: nom_locate, nom_recursive --- Cargo.lock | 39 +++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ 2 files changed, 41 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 527a77e..55a8c52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,6 +72,11 @@ name = "bitflags" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bytecount" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byteorder" version = "1.3.2" @@ -244,6 +249,8 @@ dependencies = [ "hmap 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "jlight-vm 0.1.0", "nom 5.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nom-recursive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nom_locate 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "regalloc 0.0.1", ] @@ -333,6 +340,34 @@ dependencies = [ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "nom-recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nom-recursive-macros 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nom_locate 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nom-recursive-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nom_locate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 5.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num_cpus" version = "1.12.0" @@ -551,6 +586,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum backtrace 0.3.43 (registry+https://github.com/rust-lang/crates.io-index)" = "7f80256bc78f67e7df7e36d77366f636ed976895d91fe2ab9efa3973e8fe8c4f" "checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum bytecount 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f861d9ce359f56dbcb6e0c2a1cb84e52ad732cadb57b806adeb3c7668caccbd8" "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" "checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" @@ -580,6 +616,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum mimalloc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "d1c5de6f9778f47c43e872159578a3d72779f55b40a1da4f57fe9220c14a39b1" "checksum nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" "checksum nom 5.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c433f4d505fe6ce7ff78523d2fa13a0b9f2690e181fc26168bcbe5ccc5d14e07" +"checksum nom-recursive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1fb9373dbaba1dc5ec078eaf12294a9a255207bce8341c5dfb6eb29f99d1e929" +"checksum nom-recursive-macros 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88458f0accd9f629bc0ba5bb8cf5a1aeeaf0d5a7237ed4524b62b4cb2bc7bb24" +"checksum nom_locate 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f932834fd8e391fc7710e2ba17e8f9f8645d846b55aa63207e17e110a1e1ce35" "checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" "checksum parking_lot 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" "checksum parking_lot_core 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" diff --git a/Cargo.toml b/Cargo.toml index 83afdd6..1851352 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,8 @@ hmap = "0.1" jlight-vm = {path = "jlight-vm"} regalloc = {path = "jlight-vm/regalloc.rs/lib"} hashlink = "0.4" +nom_locate = "1.0.0" +nom-recursive = "0.1.1" [dependencies.nom] version = "5.1.0" From 3646fb145d915e4f3cb3733d68e3cc9ac1ce9043 Mon Sep 17 00:00:00 2001 From: namibj Date: Sat, 8 Feb 2020 01:13:43 +0000 Subject: [PATCH 19/26] Change to using nom_locate Span instead of &str to prepare for nom_recursive. Enables #10 --- src/parser.rs | 86 +++++++++++++++++++++++++-------------------------- 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 9474315..23d9c27 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -14,6 +14,8 @@ use nom::{ sequence::{pair, preceded, terminated, tuple}, Err, IResult, }; +use nom_locate::LocatedSpanEx; +use nom_recursive::RecursiveInfo; use std::mem; //#[macro_use] extern crate nom; @@ -40,31 +42,27 @@ macro_rules! exp { }; } -type EResult<'a> = IResult<&'a str, Expr>; //Result, MsgWithPos>; +type Span<'a> = LocatedSpanEx<&'a str, RecursiveInfo>; -fn map_str<'b>( - i: impl (Fn(&'b str) -> IResult<&'b str, &'_ str>), -) -> impl Fn(&'b str) -> IResult<&'b str, String> { - map(i, |s| s.to_string()) -} +type EResult<'a> = IResult, Expr>; -fn expect_identifier<'b>(i: &'b str) -> IResult<&'b str, &str> { +fn expect_identifier<'b>(i: Span<'b>) -> IResult, String> { //re_match!(i, r"[a-zA-Z_][a-zA-Z0-9_]*") - alpha1(i) + map(alpha1, |s: Span| s.to_string())(i) } -fn parse_nil<'b>(i: &'b str) -> EResult { +fn parse_nil<'b>(i: Span) -> EResult { value(exp!(ExprKind::Nil), tag("nil"))(i) } -fn parse_bool_literal<'b>(i: &'b str) -> EResult { +fn parse_bool_literal<'b>(i: Span) -> EResult { let parse_true = value(exp!(ExprKind::ConstBool(true)), tag("true")); let parse_false = value(exp!(ExprKind::ConstBool(false)), tag("false")); alt((parse_true, parse_false))(i) } -fn lit_int<'b>(i: &'b str) -> EResult { +fn lit_int<'b>(i: Span) -> EResult { /*let tok = self.advance_token()?; let pos = tok.position; if let TokenKind::LitInt(i, _, _) = tok.kind { @@ -75,27 +73,27 @@ fn lit_int<'b>(i: &'b str) -> EResult { unimplemented!() } -fn lit_char<'b>(i: &'b str) -> EResult { +fn lit_char<'b>(i: Span) -> EResult { map(preceded(tag("'"), terminated(anychar, tag("'"))), |c| { exp!(ExprKind::ConstChar(c)) })(i) } -fn lit_float<'b>(i: &'b str) -> EResult { +fn lit_float<'b>(i: Span) -> EResult { map(double, |f| exp!(ExprKind::ConstFloat(f)))(i) } -fn lit_str<'b>(i: &'b str) -> EResult { +fn lit_str<'b>(i: Span) -> EResult { map( preceded( tag("\""), terminated(escaped(none_of("\\\""), '"', one_of("\\\"")), tag("\"")), ), - |s: &'b str| exp!(ExprKind::ConstStr(s.to_string())), + |s: Span| exp!(ExprKind::ConstStr(s.to_string())), )(i) } -pub fn parse<'b>(i: &'b str) -> Result<(), MsgWithPos> { +pub fn parse<'b>(i: Span) -> Result<(), MsgWithPos> { /* self.init()?; while !self.token.is_eof() { self.parse_top_level()?; @@ -104,7 +102,7 @@ pub fn parse<'b>(i: &'b str) -> Result<(), MsgWithPos> { unimplemented!() } -fn expect_token<'b>(i: &'b str, kind: TokenKind) -> Result { +fn expect_token<'b>(i: Span, kind: TokenKind) -> Result { /*if self.token.kind == kind { let token = self.advance_token()?; @@ -118,7 +116,7 @@ fn expect_token<'b>(i: &'b str, kind: TokenKind) -> Result { unimplemented!() } -fn parse_top_level<'b>(i: &'b str) -> Result<(), MsgWithPos> { +fn parse_top_level<'b>(i: Span) -> Result<(), MsgWithPos> { /*let expr = self.parse_expression()?; self.ast.push(expr); @@ -126,19 +124,19 @@ fn parse_top_level<'b>(i: &'b str) -> Result<(), MsgWithPos> { unimplemented!() } -fn parse_function_param<'b>(i: &'b str) -> Result { +fn parse_function_param<'b>(i: Span) -> Result { /*let name = self.expect_identifier()?; Ok(name)*/ unimplemented!() } -fn parse_function<'b>(i: &'b str) -> EResult { +fn parse_function<'b>(i: Span) -> EResult { let fn_arg_sep = tag(","); let fn_arg = expect_identifier; let tup = tuple(( - opt(map_str(expect_identifier)), + opt(expect_identifier), tag("("), - separated_list(fn_arg_sep, map_str(fn_arg)), + separated_list(fn_arg_sep, fn_arg), tag(")"), parse_block, )); @@ -147,14 +145,14 @@ fn parse_function<'b>(i: &'b str) -> EResult { })(i) } -fn parse_let<'b>(i: &'b str) -> EResult { +fn parse_let<'b>(i: Span) -> EResult { //let reassignable = alt((value(true, tag("var")), value(false, tag("let")))); let initialization = map( tuple(( //reassignable, alt((value(true, tag("var")), value(false, tag("let")))), - map_str(expect_identifier), + expect_identifier, tag("="), map(parse_expression, |expr| Some(Box::new(expr))), )), @@ -164,20 +162,20 @@ fn parse_let<'b>(i: &'b str) -> EResult { tuple(( //reassignable, alt((value(true, tag("var")), value(false, tag("let")))), - map_str(expect_identifier), + expect_identifier, )), |(r, i)| exp!(ExprKind::Var(r, i, None)), ); alt((initialization, declaration))(i) } -fn parse_return<'b>(i: &'b str) -> EResult { +fn parse_return<'b>(i: Span) -> EResult { map(pair(tag("return"), opt(parse_expression)), |(_, expr)| { exp!(ExprKind::Return(expr.map(Box::new))) })(i) } -fn parse_expression<'b>(i: &'b str) -> EResult { +fn parse_expression<'b>(i: Span) -> EResult { let parse_new = preceded(tag("new"), parse_expression); alt(( parse_new, @@ -195,32 +193,32 @@ fn parse_expression<'b>(i: &'b str) -> EResult { ))(i) } -fn parse_self<'b>(i: &'b str) -> EResult { +fn parse_self<'b>(i: Span) -> EResult { value(exp!(ExprKind::This), tag("self"))(i) } -fn parse_break<'b>(i: &'b str) -> EResult { +fn parse_break<'b>(i: Span) -> EResult { value(exp!(ExprKind::Break), tag("break"))(i) } -fn parse_continue<'b>(i: &'b str) -> EResult { +fn parse_continue<'b>(i: Span) -> EResult { value(exp!(ExprKind::Continue), tag("continue"))(i) } -fn parse_throw<'b>(i: &'b str) -> EResult { +fn parse_throw<'b>(i: Span) -> EResult { map(pair(tag("throw"), parse_expression), |(_, expr)| { exp!(ExprKind::Throw(Box::new(expr))) })(i) } -fn parse_while<'b>(i: &'b str) -> EResult { +fn parse_while<'b>(i: Span) -> EResult { map( tuple((tag("while"), parse_expression, parse_block)), |(_, expr, block)| exp!(ExprKind::While(Box::new(expr), Box::new(block))), )(i) } -fn parse_match<'b>(i: &'b str) -> EResult { +fn parse_match<'b>(i: Span) -> EResult { /*let pos = self.expect_token(TokenKind::Match)?.position; let value = self.parse_expression()?; self.expect_token(TokenKind::LBrace)?; @@ -246,7 +244,7 @@ fn parse_match<'b>(i: &'b str) -> EResult { unimplemented!() } -fn parse_if<'b>(i: &'b str) -> EResult { +fn parse_if<'b>(i: Span) -> EResult { map( tuple(( preceded(tag("if"), parse_expression), @@ -263,7 +261,7 @@ fn parse_if<'b>(i: &'b str) -> EResult { )(i) } -fn parse_block<'b>(i: &'b str) -> EResult { +fn parse_block<'b>(i: Span) -> EResult { map( preceded( tag("{"), @@ -304,7 +302,7 @@ fn create_binary(tok: Token, left: Box, right: Box) -> Box { unimplemented!() } -fn parse_binary<'b>(i: &'b str, precedence: u32) -> EResult { +fn parse_binary<'b>(i: Span, precedence: u32) -> EResult { /*let mut left = self.parse_unary()?; loop { let right_precedence = match self.token.kind { @@ -337,7 +335,7 @@ fn parse_binary<'b>(i: &'b str, precedence: u32) -> EResult { unimplemented!() } -pub fn parse_unary<'b>(i: &'b str) -> EResult<'b> { +pub fn parse_unary<'b>(i: Span<'b>) -> EResult<'b> { map(pair(one_of("+-!"), parse_primary), |(op, expr)| { exp!(ExprKind::Unop(op.to_string(), Box::new(expr))) })(i) @@ -347,7 +345,7 @@ pub fn parse_unary<'b>(i: &'b str) -> EResult<'b> { self.parse_binary(0) }*/ -fn parse_call<'b>(i: &'b str) -> EResult { +fn parse_call<'b>(i: Span) -> EResult { let fn_arg_sep = tag(","); let fn_arg = parse_expression; let tup = tuple(( @@ -361,7 +359,7 @@ fn parse_call<'b>(i: &'b str) -> EResult { })(i) } -pub fn parse_primary<'b>(i: &'b str) -> EResult { +pub fn parse_primary<'b>(i: Span) -> EResult { /*let mut left = self.parse_factor()?; loop { left = match self.token.kind { @@ -436,12 +434,12 @@ fn advance_token(/*&mut self*/) -> Result { unimplemented!() } -fn parse_lambda<'b>(i: &'b str) -> EResult { +fn parse_lambda<'b>(i: Span) -> EResult { let fn_arg_sep = tag(","); let fn_arg = expect_identifier; let tup = tuple(( tag("|"), - separated_list(fn_arg_sep, map_str(fn_arg)), + separated_list(fn_arg_sep, fn_arg), tag("|"), parse_expression, )); @@ -450,7 +448,7 @@ fn parse_lambda<'b>(i: &'b str) -> EResult { })(i) } -pub fn parse_factor<'b>(i: &'b str) -> EResult { +pub fn parse_factor<'b>(i: Span) -> EResult { alt(( preceded(tag("function"), parse_function), parse_parentheses, @@ -467,11 +465,11 @@ pub fn parse_factor<'b>(i: &'b str) -> EResult { ))(i) } -fn parse_parentheses<'b>(i: &'b str) -> EResult { +fn parse_parentheses<'b>(i: Span) -> EResult { preceded(tag("("), terminated(parse_expression, tag(")")))(i) } -fn ident<'b>(i: &'b str) -> EResult { +fn ident<'b>(i: Span) -> EResult { /*let pos = self.token.position; let ident = self.expect_identifier()?; From c40624dccdb1b45ea2d62bdc241ea1a6a2a0ceef Mon Sep 17 00:00:00 2001 From: namibj Date: Thu, 13 Feb 2020 23:33:12 +0000 Subject: [PATCH 20/26] Implement parse_binary. Close #4 --- Cargo.lock | 21 ++++++++++++ Cargo.toml | 1 + src/parser.rs | 95 +++++++++++++++++++++++++++++++++------------------ 3 files changed, 83 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 55a8c52..a174357 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -249,6 +249,7 @@ dependencies = [ "hmap 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "jlight-vm 0.1.0", "nom 5.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nom-packrat 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "nom-recursive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "nom_locate 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "regalloc 0.0.1", @@ -340,6 +341,24 @@ dependencies = [ "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "nom-packrat" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nom-packrat-macros 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nom_locate 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nom-packrat-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "nom-recursive" version = "0.1.1" @@ -616,6 +635,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum mimalloc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "d1c5de6f9778f47c43e872159578a3d72779f55b40a1da4f57fe9220c14a39b1" "checksum nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" "checksum nom 5.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c433f4d505fe6ce7ff78523d2fa13a0b9f2690e181fc26168bcbe5ccc5d14e07" +"checksum nom-packrat 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ebe0000c3976925b8061384e7c5d464415557a9df7eb650e672c08715015845" +"checksum nom-packrat-macros 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8f385978c3d5019786a9114bae4df8feea277da06ac4e4a4f924bd58ccc6c6c9" "checksum nom-recursive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1fb9373dbaba1dc5ec078eaf12294a9a255207bce8341c5dfb6eb29f99d1e929" "checksum nom-recursive-macros 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88458f0accd9f629bc0ba5bb8cf5a1aeeaf0d5a7237ed4524b62b4cb2bc7bb24" "checksum nom_locate 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f932834fd8e391fc7710e2ba17e8f9f8645d846b55aa63207e17e110a1e1ce35" diff --git a/Cargo.toml b/Cargo.toml index 1851352..c82f01f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ regalloc = {path = "jlight-vm/regalloc.rs/lib"} hashlink = "0.4" nom_locate = "1.0.0" nom-recursive = "0.1.1" +nom-packrat = "0.3.0" [dependencies.nom] version = "5.1.0" diff --git a/src/parser.rs b/src/parser.rs index 23d9c27..564ae57 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -15,7 +15,8 @@ use nom::{ Err, IResult, }; use nom_locate::LocatedSpanEx; -use nom_recursive::RecursiveInfo; +use nom_recursive::{recursive_parser, RecursiveInfo}; +use nom_packrat::{init, packrat_parser, storage, HasExtraState}; use std::mem; //#[macro_use] extern crate nom; @@ -42,7 +43,19 @@ macro_rules! exp { }; } -type Span<'a> = LocatedSpanEx<&'a str, RecursiveInfo>; +#[derive(Ord, PartialOrd, Eq, PartialEq, Clone)] +pub enum OperatorPrecedence { + None, + Or, + And, + Assign, + Cmp, + Bit, + AddShift, + Mul, +} + +type Span<'a> = LocatedSpanEx<&'a str, Option>; type EResult<'a> = IResult, Expr>; @@ -189,7 +202,7 @@ fn parse_expression<'b>(i: Span) -> EResult { parse_continue, parse_return, parse_throw, - |i| parse_binary(i, 0), + map(parse_binary, |(expr, _)| expr), ))(i) } @@ -302,37 +315,51 @@ fn create_binary(tok: Token, left: Box, right: Box) -> Box { unimplemented!() } -fn parse_binary<'b>(i: Span, precedence: u32) -> EResult { - /*let mut left = self.parse_unary()?; - loop { - let right_precedence = match self.token.kind { - TokenKind::Or => 1, - TokenKind::And => 2, - TokenKind::Eq => 3, - TokenKind::EqEq - | TokenKind::Ne - | TokenKind::Lt - | TokenKind::Le - | TokenKind::Gt - | TokenKind::Ge => 4, - TokenKind::BitOr | TokenKind::BitAnd | TokenKind::Caret => 6, - TokenKind::LtLt | TokenKind::GtGt | TokenKind::Add | TokenKind::Sub => 8, - TokenKind::Mul | TokenKind::Div | TokenKind::Mod => 9, - _ => { - return Ok(left); - } - }; - if precedence >= right_precedence { - return Ok(left); - } - - let tok = self.advance_token()?; - left = { - let right = self.parse_binary(right_precedence)?; - self.create_binary(tok, left, right) - }; - }*/ - unimplemented!() +//#[recursive_parser] +fn parse_binary<'b>(s: Span<'b>) -> IResult, (Expr, OperatorPrecedence)> { + let parse_operator = alt(( + map(tag("||"), |t| (t, OperatorPrecedence::Or)), + map(tag("&&"), |t| (t, OperatorPrecedence::And)), + map(tag("="), |t| (t, OperatorPrecedence::Assign)), + map( + alt(( + tag("=="), + tag("!="), + tag("<"), + tag("<="), + tag(">"), + tag(">="), + )), + |t| (t, OperatorPrecedence::Cmp), + ), + map(alt((tag("|"), tag("&"), tag("^"))), |t| { + (t, OperatorPrecedence::Bit) + }), + map(alt((tag("<<"), tag(">>"), tag("+"), tag("-"))), |t| { + (t, OperatorPrecedence::AddShift) + }), + map(alt((tag("*"), tag("/"), tag("%"))), |t| { + (t, OperatorPrecedence::Mul) + }), + )); + let subexpr = alt(( + map(parse_unary, |expr| (expr, OperatorPrecedence::None)), + parse_binary, + )); + let (s, (left, left_precedence)) = subexpr(s)?; + let (s, (op, precedence)) = parse_operator(s)?; + let (s, (right, right_precedence)) = subexpr(s)?; + Ok(( + s, + ( + exp!(ExprKind::BinOp( + Box::new(left), + op.fragment.to_string(), + Box::new(right) + )), + precedence, + ), + )) } pub fn parse_unary<'b>(i: Span<'b>) -> EResult<'b> { From 257a23eda648974420ccdf6abacef1b3775c0bbe Mon Sep 17 00:00:00 2001 From: namibj Date: Mon, 17 Feb 2020 01:00:42 +0000 Subject: [PATCH 21/26] Implemented parse_primary. Close #6 --- src/ast.rs | 28 +++++++++++++++++++++-- src/codegen.rs | 12 +++++----- src/parser.rs | 61 ++++++++++++++++++++------------------------------ 3 files changed, 56 insertions(+), 45 deletions(-) diff --git a/src/ast.rs b/src/ast.rs index a266b4f..203186e 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -1,4 +1,6 @@ use crate::token::Position; +use core::ops::Deref; +use std::borrow::Borrow; #[derive(Clone, PartialEq, Default)] pub struct Expr { @@ -6,13 +8,35 @@ pub struct Expr { pub expr: ExprKind, } +#[derive(Clone, Debug, PartialEq)] +pub struct Ident(String); + +impl From for String { + fn from(ident: Ident) -> Self { + ident.0 + } +} + +impl Deref for Ident { + type Target = String; + fn deref(&self) -> &String { + &self.0 + } +} + +impl Borrow for Ident { + fn borrow(&self) -> &String { + &self.0 + } +} + #[derive(Clone, Debug, PartialEq)] pub enum ExprKind { Assign(Box, Box), BinOp(Box, String, Box), Unop(String, Box), - Access(Box, String), - Ident(String), + Access(Box, Ident), + Ident(Ident), Function(Option, Vec, Box), Class(String, Box, Option>), Lambda(Vec, Box), diff --git a/src/codegen.rs b/src/codegen.rs index 475d079..af358c0 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -236,23 +236,23 @@ impl Context { pub fn compile_access(&mut self, e: &ExprKind) -> Access { match e { ExprKind::Ident(name) => { - let l = self.locals.get(name); + let l = self.locals.get(name as &str); let s: &str = name; if l.is_some() { let l = *l.unwrap(); - return Access::Stack(name.to_owned(), l); + return Access::Stack(name.to_owned().to_string(), l); } else if self.env.contains_key(s) { let l = self.env.get(s); self.used_upvars.insert(s.to_owned(), *l.unwrap()); self.nenv += 1; return Access::Env(*l.unwrap()); } else { - let (g, n) = self.global(&Global::Var(name.to_owned())); - return Access::Global(g, n, name.to_owned()); + let (g, n) = self.global(&Global::Var(name.to_owned().to_string())); + return Access::Global(g, n, name.to_owned().to_string()); } } ExprKind::Access(e, f) => { - return Access::Field(e.clone(), f.to_owned()); + return Access::Field(e.clone(), String::from(f.to_owned())); } ExprKind::This => Access::This, ExprKind::ArrayIndex(ea, ei) => { @@ -583,7 +583,7 @@ impl Context { ExprKind::Access(object, fields) => { let this = self.compile(object, tail); let field = self.new_reg(); - let (s, _) = self.global(&Global::Str(fields.to_owned())); + let (s, _) = self.global(&Global::Str(fields.to_owned().to_string())); let sr = self.new_reg(); self.write(Instruction::LoadGlobal(sr, s as _)); self.write(Instruction::Load(field, this, sr)); diff --git a/src/parser.rs b/src/parser.rs index 564ae57..cecc77e 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,3 +1,4 @@ +use crate::ast::Ident; use crate::ast::*; use crate::lexer::*; use crate::msg::*; @@ -372,52 +373,38 @@ pub fn parse_unary<'b>(i: Span<'b>) -> EResult<'b> { self.parse_binary(0) }*/ -fn parse_call<'b>(i: Span) -> EResult { +fn parse_call<'b>( + expr_parser: impl Fn(Span<'b>) -> EResult<'b>, +) -> impl Fn(Span<'b>) -> EResult<'b> { let fn_arg_sep = tag(","); let fn_arg = parse_expression; let tup = tuple(( - parse_expression, + expr_parser, tag("("), separated_list(fn_arg_sep, map(fn_arg, Box::new)), tag(")"), )); map(tup, |(expr, _, params, _)| { exp!(ExprKind::Call(Box::new(expr), params)) - })(i) + }) } -pub fn parse_primary<'b>(i: Span) -> EResult { - /*let mut left = self.parse_factor()?; - loop { - left = match self.token.kind { - TokenKind::Dot => { - let tok = self.advance_token()?; - let ident = self.expect_identifier()?; - expr!(ExprKind::Access(left, ident), tok.position) - } - TokenKind::LBracket => { - let tok = self.advance_token()?; - let index = self.parse_expression()?; - self.expect_token(TokenKind::RBracket)?; - expr!(ExprKind::ArrayIndex(left, index), tok.position) - } - _ => { - if self.token.is(TokenKind::LParen) { - let expr = left; - - self.expect_token(TokenKind::LParen)?; - - let args = - self.parse_comma_list(TokenKind::RParen, |p| p.parse_expression())?; - - expr!(ExprKind::Call(expr, args), expr.pos) - } else { - return Ok(left); - } - } - } - }*/ - unimplemented!() +fn parse_primary<'b>(s: Span<'b>) -> EResult<'b> { + alt(( + map( + tuple((parse_factor, preceded(tag("."), ident))), + |(left, ident)| exp!(ExprKind::Access(Box::new(left), ident)), + ), + map( + tuple(( + parse_factor, + preceded(tag("["), terminated(parse_expression, tag("]"))), + )), + |(left, index)| exp!(ExprKind::ArrayIndex(Box::new(left), Box::new(index))), + ), + parse_call(parse_factor), + parse_factor, + ))(s) } fn parse_comma_list( @@ -484,7 +471,7 @@ pub fn parse_factor<'b>(i: Span) -> EResult { lit_float, lit_str, parse_self, - ident, + map(ident, |ident| exp!(ExprKind::Ident(ident))), parse_lambda, parse_bool_literal, parse_bool_literal, @@ -496,7 +483,7 @@ fn parse_parentheses<'b>(i: Span) -> EResult { preceded(tag("("), terminated(parse_expression, tag(")")))(i) } -fn ident<'b>(i: Span) -> EResult { +fn ident<'b>(i: Span) -> IResult, Ident> { /*let pos = self.token.position; let ident = self.expect_identifier()?; From f7ad397ec7cce42be0e24783a8408cd794d54430 Mon Sep 17 00:00:00 2001 From: namibj Date: Tue, 18 Feb 2020 22:18:32 +0000 Subject: [PATCH 22/26] Added Checkout Submodules step to fix CI. --- .github/workflows/rust.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8ea7097..cb418dc 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -9,6 +9,14 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Checkout submodules + shell: bash + run: | + # If your submodules are configured to use SSH instead of HTTPS please uncomment the following line + # git config --global url."https://github.com/".insteadOf "git@github.com:" + auth_header="$(git config --local --get http.https://github.com/.extraheader)" + git submodule sync --recursive + git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Build run: cargo build --frozen --verbose - name: Run tests From 77ca7d1886e34764ee7d1f2ea02c1a43ba5656be Mon Sep 17 00:00:00 2001 From: namibj Date: Tue, 18 Feb 2020 22:20:48 +0000 Subject: [PATCH 23/26] Switched build from --frozen to --locked to fix CI --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index cb418dc..2c821bc 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -18,6 +18,6 @@ jobs: git submodule sync --recursive git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 - name: Build - run: cargo build --frozen --verbose + run: cargo build --locked --verbose - name: Run tests - run: cargo test --frozen --verbose + run: cargo test --locked --verbose From 6567ce6b1fe67dff8813648e639c1b4c63603c6a Mon Sep 17 00:00:00 2001 From: namibj Date: Wed, 11 Mar 2020 23:38:03 +0000 Subject: [PATCH 24/26] Fix the syntax accepted by expect_identifier to prepare for #8 --- src/parser.rs | 71 ++++++++++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index cecc77e..6dd8b90 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -4,11 +4,18 @@ use crate::lexer::*; use crate::msg::*; use crate::reader::Reader; use crate::token::*; +use nom::combinator::all_consuming; +use nom::combinator::map_opt; +use nom::combinator::map_parser; +use nom::combinator::not; +use nom::combinator::recognize; use nom::combinator::value; +use nom::AsChar; +use nom::InputTakeAtPosition; use nom::{ branch::alt, bytes::complete::{escaped, tag}, - character::complete::{alpha1, anychar, none_of, one_of}, + character::complete::{anychar, none_of, one_of}, combinator::{map, opt}, multi::{many0, separated_list}, number::complete::double, @@ -19,7 +26,6 @@ use nom_locate::LocatedSpanEx; use nom_recursive::{recursive_parser, RecursiveInfo}; use nom_packrat::{init, packrat_parser, storage, HasExtraState}; use std::mem; -//#[macro_use] extern crate nom; pub struct Parser<'a> { lexer: Lexer, @@ -60,9 +66,33 @@ type Span<'a> = LocatedSpanEx<&'a str, Option>; type EResult<'a> = IResult, Expr>; +fn alphanumeric_underscore0<'b>(input: Span<'b>) -> IResult, Span<'b>> +where + as InputTakeAtPosition>::Item: AsChar, +{ + input.split_at_position_complete(|item| !(item.is_alphanum() || item == '_')) +} + fn expect_identifier<'b>(i: Span<'b>) -> IResult, String> { - //re_match!(i, r"[a-zA-Z_][a-zA-Z0-9_]*") - map(alpha1, |s: Span| s.to_string())(i) + map( + map_parser( + recognize(tuple(( + map_opt(anychar, |c| { + if c.is_alpha() || c == '_' { + Some(()) + } else { + None + } + }), + alphanumeric_underscore0, + ))), + recognize( + // Filter out lone underscores, because those aren't identifiers. + not(all_consuming(tag("_"))), + ), + ), + |s: Span| s.to_string(), + )(i) } fn parse_nil<'b>(i: Span) -> EResult { @@ -407,39 +437,6 @@ fn parse_primary<'b>(s: Span<'b>) -> EResult<'b> { ))(s) } -fn parse_comma_list( - // &mut self, - stop: TokenKind, - mut parse: F, -) -> Result, MsgWithPos> -where - F: FnMut(&mut Parser) -> Result, -{ - /*let mut data = vec![]; - let mut comma = true; - - while !self.token.is(stop.clone()) && !self.token.is_eof() { - if !comma { - return Err(MsgWithPos::new( - self.token.position, - Msg::ExpectedToken(TokenKind::Comma.name().into(), self.token.name()), - )); - } - - let entry = parse(self)?; - data.push(entry); - - comma = self.token.is(TokenKind::Comma); - if comma { - self.advance_token()?; - } - } - - self.expect_token(stop)?; - - Ok(data)*/ - unimplemented!() -} fn advance_token(/*&mut self*/) -> Result { /*let tok = self.lexer.read_token()?; From 470485928c4f5f32a7112f33344579733390d528 Mon Sep 17 00:00:00 2001 From: namibj Date: Thu, 12 Mar 2020 00:06:54 +0000 Subject: [PATCH 25/26] Change ExprKind::Function and ExprKind::Lambda to Ident. Part of #8 --- src/ast.rs | 6 +++--- src/codegen.rs | 18 ++++++++++-------- src/parser.rs | 6 +++--- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/ast.rs b/src/ast.rs index 203186e..4a27901 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -9,7 +9,7 @@ pub struct Expr { } #[derive(Clone, Debug, PartialEq)] -pub struct Ident(String); +pub struct Ident(pub String); impl From for String { fn from(ident: Ident) -> Self { @@ -37,9 +37,9 @@ pub enum ExprKind { Unop(String, Box), Access(Box, Ident), Ident(Ident), - Function(Option, Vec, Box), + Function(Option, Vec, Box), Class(String, Box, Option>), - Lambda(Vec, Box), + Lambda(Vec, Box), Match(Box, Vec<(Box, Box)>, Option>), If(Box, Box, Option>), ConstInt(i64), diff --git a/src/codegen.rs b/src/codegen.rs index af358c0..e11f623 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -654,9 +654,9 @@ impl Context { pub fn compile_function( &mut self, - params: &[String], + params: &[Ident], e: &Box, - vname: Option, + vname: Option, ) -> u32 { let mut ctx = Context { g: self.g.clone(), @@ -681,15 +681,15 @@ impl Context { ctx.stack += 1; let r = ctx.new_reg(); ctx.write(Instruction::Pop(r)); - ctx.locals.insert(p.to_owned(), r as _); + ctx.locals.insert(String::from(p.to_owned()), r as _); } let gid = ctx.g.borrow().table.len(); if vname.is_some() { - ctx.g - .borrow_mut() - .globals - .insert(Global::Var(vname.as_ref().unwrap().to_owned()), gid as i32); + ctx.g.borrow_mut().globals.insert( + Global::Var(String::from(vname.as_ref().unwrap().to_owned())), + gid as i32, + ); } ctx.g.borrow_mut().table.push(Global::Func(gid as i32, -1)); let r = ctx.compile(e, true); @@ -704,7 +704,9 @@ impl Context { ctx.pos.clone(), gid as i32, params.len() as i32, - vname.unwrap_or(String::from("")), + vname + .map(String::from) + .unwrap_or(String::from("")), )); if ctx.nenv > 0 { diff --git a/src/parser.rs b/src/parser.rs index 6dd8b90..dcecd80 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -176,9 +176,9 @@ fn parse_function_param<'b>(i: Span) -> Result { fn parse_function<'b>(i: Span) -> EResult { let fn_arg_sep = tag(","); - let fn_arg = expect_identifier; + let fn_arg = map(expect_identifier, Ident); let tup = tuple(( - opt(expect_identifier), + opt(map(expect_identifier, Ident)), tag("("), separated_list(fn_arg_sep, fn_arg), tag(")"), @@ -447,7 +447,7 @@ fn advance_token(/*&mut self*/) -> Result { fn parse_lambda<'b>(i: Span) -> EResult { let fn_arg_sep = tag(","); - let fn_arg = expect_identifier; + let fn_arg = map(expect_identifier, Ident); let tup = tuple(( tag("|"), separated_list(fn_arg_sep, fn_arg), From 49a91635a35c9983a99e8cdd9e97742cca5fb76d Mon Sep 17 00:00:00 2001 From: namibj Date: Thu, 12 Mar 2020 00:17:44 +0000 Subject: [PATCH 26/26] Change ExprKind::Var to Ident. Part of #8 --- src/ast.rs | 2 +- src/codegen.rs | 2 +- src/parser.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ast.rs b/src/ast.rs index 4a27901..9b0eee8 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -48,7 +48,7 @@ pub enum ExprKind { New(Box), ConstFloat(f64), Object(Vec<(Box, Box)>), - Var(bool, String, Option>), + Var(bool, Ident, Option>), While(Box, Box), Block(Vec>), Return(Option>), diff --git a/src/codegen.rs b/src/codegen.rs index e11f623..7d58a3c 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -430,7 +430,7 @@ impl Context { r } }; - self.locals.insert(name.to_owned(), r as _); + self.locals.insert(String::from(name.to_owned()), r as _); r } diff --git a/src/parser.rs b/src/parser.rs index dcecd80..3a98a8a 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -196,7 +196,7 @@ fn parse_let<'b>(i: Span) -> EResult { tuple(( //reassignable, alt((value(true, tag("var")), value(false, tag("let")))), - expect_identifier, + map(expect_identifier, Ident), tag("="), map(parse_expression, |expr| Some(Box::new(expr))), )), @@ -206,7 +206,7 @@ fn parse_let<'b>(i: Span) -> EResult { tuple(( //reassignable, alt((value(true, tag("var")), value(false, tag("let")))), - expect_identifier, + map(expect_identifier, Ident), )), |(r, i)| exp!(ExprKind::Var(r, i, None)), );