Skip to content

Commit 97713ee

Browse files
StunxFSStunxFS
committed
fx panic
1 parent bc7add4 commit 97713ee

File tree

2 files changed

+68
-60
lines changed

2 files changed

+68
-60
lines changed

‎lib/rivet/src/tokenizer/mod.ri

Lines changed: 34 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,40 @@ public struct Tokenizer {
224224
self.pos += len;
225225
}
226226

227+
// NOTE: this function doesn't do any decoding... it just replaces '\xc0' with
228+
// the byte 0xc0
229+
func decode_h_escape_single(self, str: string, idx: uint) -> (uint, string) {
230+
end_idx := idx + 4; // "\xXX".len == 4
231+
if idx + 2 > str.len or end_idx > str.len {
232+
self.error("unfinished single hex escape started at");
233+
return (0, "");
234+
}
235+
return (
236+
end_idx,
237+
@as(uint8, conv.parse_uint(str[idx + 2..end_idx], 16, 8) catch 0).to_string()
238+
);
239+
}
240+
241+
// only handle single-byte inline escapes like '\xc0'
242+
func decode_h_escapes(self, s: string, start: uint, escapes_pos: []uint) -> string {
243+
if escapes_pos.is_empty() {
244+
return s;
245+
}
246+
mut ss := @vec(string, escapes_pos.len * 2 + 1);
247+
ss.push(s[..escapes_pos[escapes_pos.len - 1] - start]);
248+
for i, pos in escapes_pos {
249+
idx := pos - start;
250+
(end_idx, segment) := self.decode_h_escape_single(s, idx);
251+
ss.push(segment);
252+
ss.push(if i + 1 < escapes_pos.len {
253+
s[end_idx..escapes_pos[i + 1] - start]
254+
} else {
255+
s[end_idx..]
256+
});
257+
}
258+
return utils.join(ss, "");
259+
}
260+
227261
#[inline]
228262
func error(self, msg: string, pos: token.Pos := self.current_pos()) {
229263
report.error(msg, pos);
@@ -245,36 +279,6 @@ public struct Tokenizer {
245279
}
246280
}
247281

248-
// NOTE: this function doesn't do any decoding... it just replaces '\xc0' with
249-
// the byte 0xc0
250-
func decode_h_escape_single(str: string, idx: uint) -> (uint, string) {
251-
end_idx := idx + 4; // "\xXX".len == 4
252-
return (
253-
end_idx,
254-
@as(uint8, conv.parse_uint(str[idx + 2..end_idx], 16, 8) catch 0).to_string()
255-
);
256-
}
257-
258-
// only handle single-byte inline escapes like '\xc0'
259-
func decode_h_escapes(s: string, start: uint, escapes_pos: []uint) -> string {
260-
if escapes_pos.is_empty() {
261-
return s;
262-
}
263-
mut ss := @vec(string, escapes_pos.len * 2 + 1);
264-
ss.push(s[..escapes_pos[escapes_pos.len - 1] - start]);
265-
for i, pos in escapes_pos {
266-
idx := pos - start;
267-
(end_idx, segment) := decode_h_escape_single(s, idx);
268-
ss.push(segment);
269-
ss.push(if i + 1 < escapes_pos.len {
270-
s[end_idx..escapes_pos[i + 1] - start]
271-
} else {
272-
s[end_idx..]
273-
});
274-
}
275-
return utils.join(ss, "");
276-
}
277-
278282
// handle single-byte inline octal escapes like '\###'
279283
// NOTE: this function doesn't do any decoding... it just replaces '\141' with
280284
// the byte 0o141

‎lib/rivet/src/tokenizer/next.ri

Lines changed: 34 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -605,6 +605,9 @@ extend Tokenizer {
605605
len -= 1;
606606

607607
mut ch := self.text[start + 1..self.pos].clone();
608+
if self.prefs.is_fmt {
609+
return ch;
610+
}
608611
if len != 1 {
609612
// the string inside the backticks is longer than one character
610613
// but we might only have one rune... attempt to decode escapes
@@ -625,7 +628,7 @@ extend Tokenizer {
625628
}
626629
}
627630
ch = if escaped_hex {
628-
decode_h_escapes(ch, 0, escapes_pos)
631+
self.decode_h_escapes(ch, 0, escapes_pos)
629632
} else {
630633
decode_o_escapes(ch, 0, escapes_pos)
631634
};
@@ -712,10 +715,9 @@ extend Tokenizer {
712715
self.error(r"`\u` incomplete unicode character value", pos);
713716
}
714717
u_escapes_pos.push(self.pos - 1);
715-
}
716-
// unknown escape sequence
717-
if c !in [b'x', b'u', b'e', b'n', b'r', b't', b'v', b'a', b'f', b'b',
718-
b'\\', b'@', b'?', b'{', b'}', b'\'', b'"'] and !c.is_digit() {
718+
} else if c !in [b'x', b'u', b'e', b'n', b'r', b't', b'v', b'a', b'f', b'b',
719+
b'\\', b'@', b'?', b'{', b'}', b'\'', b'"', b'$'] and !c.is_digit() {
720+
// unknown escape sequence
719721
self.pos -= 1;
720722
self.error(
721723
"unknown escape sequence `{}`".fmt(c.to_ascii()),
@@ -732,35 +734,37 @@ extend Tokenizer {
732734
mut lit := "";
733735
if start <= self.pos {
734736
lit = self.text[start + 1..self.pos].clone();
735-
mut segment_idx: uint := 0;
736-
mut str_segments := @vec(string);
737-
if u_escapes_pos.len + h_escapes_pos.len > 0 {
738-
mut all_pos := @vec(uint, u_escapes_pos.len + h_escapes_pos.len);
739-
for pos1 in u_escapes_pos {
740-
all_pos.push(pos1);
741-
}
742-
for pos1 in h_escapes_pos {
743-
all_pos.push(pos1);
744-
}
745-
for pos1 in all_pos {
746-
str_segments.push(lit[segment_idx..(pos1 - start)]);
747-
segment_idx = pos1 - start;
748-
if pos1 in u_escapes_pos {
749-
(end_idx, segment) := decode_u_escape_single(lit, segment_idx);
750-
str_segments.push(segment);
751-
segment_idx = end_idx;
737+
if !self.prefs.is_fmt {
738+
mut segment_idx: uint := 0;
739+
mut str_segments := @vec(string);
740+
if u_escapes_pos.len + h_escapes_pos.len > 0 {
741+
mut all_pos := @vec(uint, u_escapes_pos.len + h_escapes_pos.len);
742+
for pos1 in u_escapes_pos {
743+
all_pos.push(pos1);
752744
}
753-
if pos1 in h_escapes_pos {
754-
(end_idx, segment) := decode_h_escape_single(lit, segment_idx);
755-
str_segments.push(segment);
756-
segment_idx = end_idx;
745+
for pos1 in h_escapes_pos {
746+
all_pos.push(pos1);
747+
}
748+
for pos1 in all_pos {
749+
str_segments.push(lit[segment_idx..(pos1 - start)]);
750+
segment_idx = pos1 - start;
751+
if pos1 in u_escapes_pos {
752+
(end_idx, segment) := decode_u_escape_single(lit, segment_idx);
753+
str_segments.push(segment);
754+
segment_idx = end_idx;
755+
}
756+
if pos1 in h_escapes_pos {
757+
(end_idx, segment) := self.decode_h_escape_single(lit, segment_idx);
758+
str_segments.push(segment);
759+
segment_idx = end_idx;
760+
}
757761
}
758762
}
763+
if segment_idx < lit.len {
764+
str_segments.push(lit[segment_idx..]);
765+
}
766+
lit = utils.join(str_segments, "");
759767
}
760-
if segment_idx < lit.len {
761-
str_segments.push(lit[segment_idx..]);
762-
}
763-
lit = utils.join(str_segments, "");
764768
if n_cr_chars > 0 {
765769
lit = lit.replace("\r", "");
766770
}

0 commit comments

Comments
 (0)