forked from Armavica/rustlex
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathcomplex.rs
80 lines (73 loc) · 2.72 KB
/
complex.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
#![feature(rustc_private,plugin)]
#![plugin(rustlex)]
#[allow(plugin_as_library)]
extern crate rustlex;
#[macro_use] extern crate log;
use std::io::BufReader;
// The Token type is returned by the lexer function on
// each call and must be declared in the same module
// as where the rustlex! macro is invoked
use self::Token::{TokInt, TokFloat, TokId, TokString};
#[derive(PartialEq,Debug)]
pub enum Token {
TokInt(u32),
TokFloat(f32),
TokId(String),
TokString(String)
}
rustlex! ComplexLexer {
// define some regular expressions that matches
// float and int constants allowed in C
// definitions are of the form
// let name = regex;
// a complete description of the regex syntax
// is available in the manual
let INT = ['0'-'9']+["uUlL"]?;
let HEX = '0'["xX"]['a'-'f''A'-'F''0'-'9']+["uUlL"]?;
let FLOAT = (['0'-'9']+'.'|['0'-'9']*'.'['0'-'9']+)(["eE"]["+-"]?['0'-'9']+)?["fFlL"]?;
let DEC_FLOAT = ['0'-'9']+(["eE"]["+-"]?['0'-'9']+)["fFlL"]?;
let HEX_FLOAT = '0'["xX"]['a'-'f''A'-'F''0'-'9']*'.'['a'-'f''A'-'F''0'-'9']*(["pP"]['0'-'9']+)?["fFlL"]?;
let INTCONST = (INT|HEX);
let FLTCONST = (FLOAT|HEX_FLOAT|DEC_FLOAT);
let STR = '"' ([^'\\''"']|'\\'.)* '"' ;
let ID = ['a'-'z''A'-'Z''_']['a'-'z''A'-'Z''_''0'-'9']*;
// define rules that associate a pattern
// to a Rust snippet to be executed when a
// token matching the pattern is encountered
// each rule is of the form
// regex => action
// action can be a block or a single statement
. => |_:&mut ComplexLexer<R>| { None }
INT => |lexer:&mut ComplexLexer<R>| Some(TokInt(lexer.yystr()[..].parse().unwrap()))
HEX => |lexer:&mut ComplexLexer<R>| {
let s = lexer.yystr();
let i:u32 = u32::from_str_radix(&s[2 .. s.len()-1], 16).unwrap();
Some(TokInt(i))
}
FLTCONST => |lexer:&mut ComplexLexer<R>| Some(TokFloat(lexer.yystr()[..].parse().unwrap()))
ID => |lexer:&mut ComplexLexer<R>| Some(TokId(lexer.yystr()))
STR => |lexer:&mut ComplexLexer<R>| Some(TokString(lexer.yystr()))
}
#[test]
fn test_complex() {
let expected = vec!(
TokId("foo".to_string()),
TokId("bar".to_string()),
TokId("baz".to_string()),
TokFloat(0.1),
TokInt(212u32),
TokString("\"a\"".to_string()),
TokInt(0x121u32),
TokId("baz".to_string()),
TokInt(123),
TokId("foo".to_string()));
let str = "foo bar baz 0.10 212 \"a\" 0x121u baz 123foo ";
let inp = BufReader::new(str.as_bytes());
let lexer = ComplexLexer::new(inp);
let mut iter = expected.iter();
for tok in lexer {
let expect = iter.next().unwrap();
assert!(expect == &tok);
}
assert!(iter.next() == None);
}