From 27e8d65819d44388de3fe7f2a2d63823198af6ad Mon Sep 17 00:00:00 2001 From: JanHoefelmeyer Date: Tue, 8 Aug 2023 11:08:45 +0200 Subject: [PATCH 1/9] Basic Structure --- jsonpath.go | 28 ++++++++++++++++++---------- parse.go | 10 +++++----- placeholder.go | 4 ++-- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/jsonpath.go b/jsonpath.go index 8571e33..33272ed 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -44,22 +44,30 @@ func Get(path string, value interface{}) (interface{}, error) { return eval(context.Background(), value) } -var lang = gval.NewLanguage( - gval.Base(), - gval.PrefixExtension('$', parseRootPath), - gval.PrefixExtension('@', parseCurrentPath), -) +var lang = func() gval.Language { + l := gval.NewLanguage( + gval.Base(), + gval.PrefixExtension('$', parseRootPath), + gval.PrefixExtension('@', parseCurrentPath), + ) + l.CreateScanner(createScanner) + return l +}() // Language is the JSONPath Language func Language() gval.Language { return lang } -var placeholderExtension = gval.NewLanguage( - lang, - gval.PrefixExtension('{', parseJSONObject), - gval.PrefixExtension('#', parsePlaceholder), -) +var placeholderExtension = func() gval.Language { + l := gval.NewLanguage( + lang, + gval.PrefixExtension('{', parseJSONObject), + gval.PrefixExtension('#', parsePlaceholder), + ) + l.CreateScanner(createScanner) + return l +}() // PlaceholderExtension is the JSONPath Language with placeholder func PlaceholderExtension() gval.Language { diff --git a/parse.go b/parse.go index 977d5b4..c2db144 100644 --- a/parse.go +++ b/parse.go @@ -4,7 +4,7 @@ import ( "context" "fmt" "math" - "text/scanner" + goscanner "text/scanner" "github.com/PaesslerAG/gval" ) @@ -81,7 +81,7 @@ func (p *parser) parsePath(c context.Context) error { func (p *parser) parseSelect(c context.Context) error { scan := p.Scan() switch scan { - case scanner.Ident: + case goscanner.Ident: p.appendPlainSelector(directSelector(p.Const(p.TokenText()))) return p.parsePath(c) case '.': @@ -91,7 +91,7 @@ func (p *parser) parseSelect(c context.Context) error { p.appendAmbiguousSelector(starSelector()) return p.parsePath(c) default: - return p.Expected("JSON select", scanner.Ident, '.', '*') + return p.Expected("JSON select", goscanner.Ident, '.', '*') } } @@ -154,7 +154,7 @@ func (p *parser) parseBracket(c context.Context) (keys []gval.Evaluable, seperat func (p *parser) parseMapper(c context.Context) error { scan := p.Scan() switch scan { - case scanner.Ident: + case goscanner.Ident: p.appendPlainSelector(directSelector(p.Const(p.TokenText()))) case '[': keys, seperator, err := p.parseBracket(c) @@ -178,7 +178,7 @@ func (p *parser) parseMapper(c context.Context) error { case '(': return p.parseScript(c) default: - return p.Expected("JSON mapper", '[', scanner.Ident, '*') + return p.Expected("JSON mapper", '[', goscanner.Ident, '*') } return p.parsePath(c) } diff --git a/placeholder.go b/placeholder.go index d1cd063..be796b3 100644 --- a/placeholder.go +++ b/placeholder.go @@ -5,7 +5,7 @@ import ( "context" "fmt" "strconv" - "text/scanner" + goscanner "text/scanner" "github.com/PaesslerAG/gval" ) @@ -134,7 +134,7 @@ func parsePlaceholder(c context.Context, p *gval.Parser) (gval.Evaluable, error) } *(hasWildcard.(*bool)) = true switch p.Scan() { - case scanner.Int: + case goscanner.Int: id, err := strconv.Atoi(p.TokenText()) if err != nil { return nil, err From a69f89f58c62cadae751e4dd28260c1df8180939 Mon Sep 17 00:00:00 2001 From: JanHoefelmeyer Date: Tue, 8 Aug 2023 11:20:24 +0200 Subject: [PATCH 2/9] Add scanner.go --- scanner.go | 773 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 773 insertions(+) create mode 100644 scanner.go diff --git a/scanner.go b/scanner.go new file mode 100644 index 0000000..e266c72 --- /dev/null +++ b/scanner.go @@ -0,0 +1,773 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package scanner provides a scanner and tokenizer for UTF-8-encoded text. +// It takes an io.Reader providing the source, which then can be tokenized +// through repeated calls to the Scan function. For compatibility with +// existing tools, the NUL character is not allowed. If the first character +// in the source is a UTF-8 encoded byte order mark (BOM), it is discarded. +// +// By default, a Scanner skips white space and Go comments and recognizes all +// literals as defined by the Go language specification. It may be +// customized to recognize only a subset of those literals and to recognize +// different identifier and white space characters. +package jsonpath + +import ( + "bytes" + "fmt" + "io" + "os" + "strconv" + goscanner "text/scanner" + "unicode" + "unicode/utf8" + + "github.com/PaesslerAG/gval" +) + +// Predefined mode bits to control recognition of tokens. For instance, +// to configure a Scanner such that it only recognizes (Go) identifiers, +// integers, and skips comments, set the Scanner's Mode field to: +// +// ScanIdents | ScanInts | SkipComments +// +// With the exceptions of comments, which are skipped if SkipComments is +// set, unrecognized tokens are not ignored. Instead, the scanner simply +// returns the respective individual characters (or possibly sub-tokens). +// For instance, if the mode is ScanIdents (not ScanStrings), the string +// "foo" is scanned as the token sequence '"' Ident '"'. +// +// Use GoTokens to configure the Scanner such that it accepts all Go +// literal tokens including Go identifiers. Comments will be skipped. + +// The result of Scan is one of these tokens or a Unicode character. +const ( + quotedString = goscanner.Comment + 2 + scanQuotedString = 1 << -quotedString +) + +const jsonpathTokens = goscanner.ScanIdents | goscanner.ScanFloats | goscanner.ScanChars | + goscanner.ScanStrings | goscanner.ScanComments | goscanner.SkipComments + +const bufLen = 1024 // at least utf8.UTFMax + +// A scanner implements reading of Unicode characters and tokens from an io.Reader. +type scanner struct { + // Input + src io.Reader + + // Source buffer + srcBuf [bufLen + 1]byte // +1 for sentinel for common case of s.next() + srcPos int // reading position (srcBuf index) + srcEnd int // source end (srcBuf index) + + // Source position + srcBufOffset int // byte offset of srcBuf[0] in source + line int // line count + column int // character count + lastLineLen int // length of last line in characters (for correct column reporting) + lastCharLen int // length of last character in bytes + + // Token text buffer + // Typically, token text is stored completely in srcBuf, but in general + // the token text's head may be buffered in tokBuf while the token text's + // tail is stored in srcBuf. + tokBuf bytes.Buffer // token text head that is not in srcBuf anymore + tokPos int // token text tail position (srcBuf index); valid if >= 0 + tokEnd int // token text tail end (srcBuf index) + + // One character look-ahead + ch rune // character before current srcPos + + // Error is called for each error encountered. If no Error + // function is set, the error is reported to os.Stderr. + Error func(s gval.Scanner, msg string) + + // ErrorCount is incremented by one for each error encountered. + ErrorCount int + + // The Mode field controls which tokens are recognized. For instance, + // to recognize Ints, set the ScanInts bit in Mode. The field may be + // changed at any time. + Mode uint + + // The Whitespace field controls which characters are recognized + // as white space. To recognize a character ch <= ' ' as white space, + // set the ch'th bit in Whitespace (the Scanner's behavior is undefined + // for values ch > ' '). The field may be changed at any time. + Whitespace uint64 + + // IsIdentRune is a predicate controlling the characters accepted + // as the ith rune in an identifier. The set of valid characters + // must not intersect with the set of white space characters. + // If no IsIdentRune function is set, regular Go identifiers are + // accepted instead. The field may be changed at any time. + IsIdentRune func(ch rune, i int) bool + + // Start position of most recently scanned token; set by Scan. + // Calling Init or Next invalidates the position (Line == 0). + // The Filename field is always left untouched by the Scanner. + // If an error is reported (via Error) and Position is invalid, + // the scanner is not inside a token. Call Pos to obtain an error + // position in that case, or to obtain the position immediately + // after the most recently scanned token. + goscanner.Position +} + +func createScanner() gval.Scanner { + return &scanner{} +} + +// Init initializes a Scanner with a new source and returns s. +// Error is set to nil, ErrorCount is set to 0, Mode is set to GoTokens, +// and Whitespace is set to GoWhitespace. +func (s *scanner) Init(src io.Reader) { + s.InitMode(src, jsonpathTokens) +} + +func (s *scanner) SetError(fn func(s gval.Scanner, msg string)) { + s.Error = fn +} + +func (s *scanner) SetFilename(filename string) { + s.Filename = filename +} + +func (s *scanner) SetWhitespace(ws uint64) { + s.Whitespace = ws +} + +func (s *scanner) GetWhitespace() uint64 { + return s.Whitespace +} + +func (s *scanner) SetMode(m uint) { + s.Mode = m +} + +func (s *scanner) GetMode() uint { + return s.Mode +} + +func (s *scanner) SetIsIdentRune(fn func(ch rune, i int) bool) { + s.IsIdentRune = fn +} + +func (s *scanner) GetIsIdentRune() func(ch rune, i int) bool { + return s.IsIdentRune +} + +func (s *scanner) GetPosition() goscanner.Position { + return s.Position +} + +func (s *scanner) Unquote(in string) (string, error) { + // Hack: Replace single quotes with double quotes. + if n := len(in); n > 1 && in[0] == '\'' && in[n-1] == '\'' { + in = `"` + in[1:n-1] + `"` + } + return strconv.Unquote(in) +} + +// InitMode initializes a Scanner with a new source and returns s. +// Error is set to nil, ErrorCount is set to 0, Mode is set to mode, +// and Whitespace is set to GoWhitespace. +func (s *scanner) InitMode(src io.Reader, mode uint) { + s.src = src + + // initialize source buffer + // (the first call to next() will fill it by calling src.Read) + s.srcBuf[0] = utf8.RuneSelf // sentinel + s.srcPos = 0 + s.srcEnd = 0 + + // initialize source position + s.srcBufOffset = 0 + s.line = 1 + s.column = 0 + s.lastLineLen = 0 + s.lastCharLen = 0 + + // initialize token text buffer + // (required for first call to next()). + s.tokPos = -1 + + // initialize one character look-ahead + s.ch = -2 // no char read yet, not EOF + + // initialize public fields + s.Error = nil + s.ErrorCount = 0 + s.Mode = mode + s.Whitespace = goscanner.GoWhitespace + s.Line = 0 // invalidate token position +} + +// next reads and returns the next Unicode character. It is designed such +// that only a minimal amount of work needs to be done in the common ASCII +// case (one test to check for both ASCII and end-of-buffer, and one test +// to check for newlines). +func (s *scanner) next() rune { + ch, width := rune(s.srcBuf[s.srcPos]), 1 + + if ch >= utf8.RuneSelf { + // uncommon case: not ASCII or not enough bytes + for s.srcPos+utf8.UTFMax > s.srcEnd && !utf8.FullRune(s.srcBuf[s.srcPos:s.srcEnd]) { + // not enough bytes: read some more, but first + // save away token text if any + if s.tokPos >= 0 { + s.tokBuf.Write(s.srcBuf[s.tokPos:s.srcPos]) + s.tokPos = 0 + // s.tokEnd is set by Scan() + } + // move unread bytes to beginning of buffer + copy(s.srcBuf[0:], s.srcBuf[s.srcPos:s.srcEnd]) + s.srcBufOffset += s.srcPos + // read more bytes + // (an io.Reader must return io.EOF when it reaches + // the end of what it is reading - simply returning + // n == 0 will make this loop retry forever; but the + // error is in the reader implementation in that case) + i := s.srcEnd - s.srcPos + n, err := s.src.Read(s.srcBuf[i:bufLen]) + s.srcPos = 0 + s.srcEnd = i + n + s.srcBuf[s.srcEnd] = utf8.RuneSelf // sentinel + if err != nil { + if err != io.EOF { + s.error(err.Error()) + } + if s.srcEnd == 0 { + if s.lastCharLen > 0 { + // previous character was not EOF + s.column++ + } + s.lastCharLen = 0 + return goscanner.EOF + } + // If err == EOF, we won't be getting more + // bytes; break to avoid infinite loop. If + // err is something else, we don't know if + // we can get more bytes; thus also break. + break + } + } + // at least one byte + ch = rune(s.srcBuf[s.srcPos]) + if ch >= utf8.RuneSelf { + // uncommon case: not ASCII + ch, width = utf8.DecodeRune(s.srcBuf[s.srcPos:s.srcEnd]) + if ch == utf8.RuneError && width == 1 { + // advance for correct error position + s.srcPos += width + s.lastCharLen = width + s.column++ + s.error("invalid UTF-8 encoding") + return ch + } + } + } + + // advance + s.srcPos += width + s.lastCharLen = width + s.column++ + + // special situations + switch ch { + case 0: + // for compatibility with other tools + s.error("invalid character NUL") + case '\n': + s.line++ + s.lastLineLen = s.column + s.column = 0 + } + + return ch +} + +// Next reads and returns the next Unicode character. +// It returns EOF at the end of the source. It reports +// a read error by calling s.Error, if not nil; otherwise +// it prints an error message to os.Stderr. Next does not +// update the Scanner's Position field; use Pos() to +// get the current position. +func (s *scanner) Next() rune { + s.tokPos = -1 // don't collect token text + s.Line = 0 // invalidate token position + ch := s.Peek() + if ch != goscanner.EOF { + s.ch = s.next() + } + return ch +} + +// Peek returns the next Unicode character in the source without advancing +// the scanner. It returns EOF if the scanner's position is at the last +// character of the source. +func (s *scanner) Peek() rune { + if s.ch == -2 { + // this code is only run for the very first character + s.ch = s.next() + if s.ch == '\uFEFF' { + s.ch = s.next() // ignore BOM + } + } + return s.ch +} + +func (s *scanner) error(msg string) { + s.tokEnd = s.srcPos - s.lastCharLen // make sure token text is terminated + s.ErrorCount++ + if s.Error != nil { + s.Error(s, msg) + return + } + pos := s.Position + if !pos.IsValid() { + pos = s.Pos() + } + fmt.Fprintf(os.Stderr, "%s: %s\n", pos, msg) +} + +func (s *scanner) errorf(format string, args ...interface{}) { + s.error(fmt.Sprintf(format, args...)) +} + +func (s *scanner) isIdentRune(ch rune, i int) bool { + if s.IsIdentRune != nil { + return ch != goscanner.EOF && s.IsIdentRune(ch, i) + } + return ch == '_' || unicode.IsLetter(ch) || unicode.IsDigit(ch) && i > 0 +} + +func (s *scanner) scanIdentifier() rune { + // we know the zero'th rune is OK; start scanning at the next one + ch := s.next() + for i := 1; s.isIdentRune(ch, i); i++ { + ch = s.next() + } + return ch +} + +func lower(ch rune) rune { return ('a' - 'A') | ch } // returns lower-case ch iff ch is ASCII letter +func isDecimal(ch rune) bool { return '0' <= ch && ch <= '9' } +func isHex(ch rune) bool { return '0' <= ch && ch <= '9' || 'a' <= lower(ch) && lower(ch) <= 'f' } + +// digits accepts the sequence { digit | '_' } starting with ch0. +// If base <= 10, digits accepts any decimal digit but records +// the first invalid digit >= base in *invalid if *invalid == 0. +// digits returns the first rune that is not part of the sequence +// anymore, and a bitset describing whether the sequence contained +// digits (bit 0 is set), or separators '_' (bit 1 is set). +func (s *scanner) digits(ch0 rune, base int, invalid *rune) (ch rune, digsep int) { + ch = ch0 + if base <= 10 { + max := rune('0' + base) + for isDecimal(ch) || ch == '_' { + ds := 1 + if ch == '_' { + ds = 2 + } else if ch >= max && *invalid == 0 { + *invalid = ch + } + digsep |= ds + ch = s.next() + } + } else { + for isHex(ch) || ch == '_' { + ds := 1 + if ch == '_' { + ds = 2 + } + digsep |= ds + ch = s.next() + } + } + return +} + +func (s *scanner) scanNumber(ch rune, seenDot bool) (rune, rune) { + base := 10 // number base + prefix := rune(0) // one of 0 (decimal), '0' (0-octal), 'x', 'o', or 'b' + digsep := 0 // bit 0: digit present, bit 1: '_' present + invalid := rune(0) // invalid digit in literal, or 0 + + // integer part + var tok rune + var ds int + if !seenDot { + tok = goscanner.Int + if ch == '0' { + ch = s.next() + switch lower(ch) { + case 'x': + ch = s.next() + base, prefix = 16, 'x' + case 'o': + ch = s.next() + base, prefix = 8, 'o' + case 'b': + ch = s.next() + base, prefix = 2, 'b' + default: + base, prefix = 8, '0' + digsep = 1 // leading 0 + } + } + ch, ds = s.digits(ch, base, &invalid) + digsep |= ds + if ch == '.' && s.Mode&goscanner.ScanFloats != 0 { + ch = s.next() + seenDot = true + } + } + + // fractional part + if seenDot { + tok = goscanner.Float + if prefix == 'o' || prefix == 'b' { + s.error("invalid radix point in " + litname(prefix)) + } + ch, ds = s.digits(ch, base, &invalid) + digsep |= ds + } + + if digsep&1 == 0 { + s.error(litname(prefix) + " has no digits") + } + + // exponent + if e := lower(ch); (e == 'e' || e == 'p') && s.Mode&goscanner.ScanFloats != 0 { + switch { + case e == 'e' && prefix != 0 && prefix != '0': + s.errorf("%q exponent requires decimal mantissa", ch) + case e == 'p' && prefix != 'x': + s.errorf("%q exponent requires hexadecimal mantissa", ch) + } + ch = s.next() + tok = goscanner.Float + if ch == '+' || ch == '-' { + ch = s.next() + } + ch, ds = s.digits(ch, 10, nil) + digsep |= ds + if ds&1 == 0 { + s.error("exponent has no digits") + } + } else if prefix == 'x' && tok == goscanner.Float { + s.error("hexadecimal mantissa requires a 'p' exponent") + } + + if tok == goscanner.Int && invalid != 0 { + s.errorf("invalid digit %q in %s", invalid, litname(prefix)) + } + + if digsep&2 != 0 { + s.tokEnd = s.srcPos - s.lastCharLen // make sure token text is terminated + if i := invalidSep(s.TokenText()); i >= 0 { + s.error("'_' must separate successive digits") + } + } + + return tok, ch +} + +func litname(prefix rune) string { + switch prefix { + default: + return "decimal literal" + case 'x': + return "hexadecimal literal" + case 'o', '0': + return "octal literal" + case 'b': + return "binary literal" + } +} + +// invalidSep returns the index of the first invalid separator in x, or -1. +func invalidSep(x string) int { + x1 := ' ' // prefix char, we only care if it's 'x' + d := '.' // digit, one of '_', '0' (a digit), or '.' (anything else) + i := 0 + + // a prefix counts as a digit + if len(x) >= 2 && x[0] == '0' { + x1 = lower(rune(x[1])) + if x1 == 'x' || x1 == 'o' || x1 == 'b' { + d = '0' + i = 2 + } + } + + // mantissa and exponent + for ; i < len(x); i++ { + p := d // previous digit + d = rune(x[i]) + switch { + case d == '_': + if p != '0' { + return i + } + case isDecimal(d) || x1 == 'x' && isHex(d): + d = '0' + default: + if p == '_' { + return i - 1 + } + d = '.' + } + } + if d == '_' { + return len(x) - 1 + } + + return -1 +} + +func digitVal(ch rune) int { + switch { + case '0' <= ch && ch <= '9': + return int(ch - '0') + case 'a' <= lower(ch) && lower(ch) <= 'f': + return int(lower(ch) - 'a' + 10) + } + return 16 // larger than any legal digit val +} + +func (s *scanner) scanDigits(ch rune, base, n int) rune { + for n > 0 && digitVal(ch) < base { + ch = s.next() + n-- + } + if n > 0 { + s.error("invalid char escape") + } + return ch +} + +func (s *scanner) scanEscape(quote rune) rune { + ch := s.next() // read character after '/' + switch ch { + case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote: + // nothing to do + ch = s.next() + case '0', '1', '2', '3', '4', '5', '6', '7': + ch = s.scanDigits(ch, 8, 3) + case 'x': + ch = s.scanDigits(s.next(), 16, 2) + case 'u': + ch = s.scanDigits(s.next(), 16, 4) + case 'U': + ch = s.scanDigits(s.next(), 16, 8) + default: + s.error("invalid char escape") + } + return ch +} + +func (s *scanner) scanString(quote rune) (n int) { + ch := s.next() // read character after quote + for ch != quote { + if ch == '\n' || ch < 0 { + s.error("literal not terminated") + return + } + if ch == '\\' { + ch = s.scanEscape(quote) + } else { + ch = s.next() + } + n++ + } + return +} + +func (s *scanner) scanChar() rune { + if s.Mode&scanQuotedString != 0 { + s.scanString('\'') + return goscanner.String + } + if s.scanString('\'') != 1 { + s.error("invalid char literal") + } + return goscanner.Char +} + +func (s *scanner) scanComment(ch rune) rune { + // ch == '/' || ch == '*' + if ch == '/' { + // line comment + ch = s.next() // read character after "//" + for ch != '\n' && ch >= 0 { + ch = s.next() + } + return ch + } + + // general comment + ch = s.next() // read character after "/*" + for { + if ch < 0 { + s.error("comment not terminated") + break + } + ch0 := ch + ch = s.next() + if ch0 == '*' && ch == '/' { + ch = s.next() + break + } + } + return ch +} + +// Scan reads the next token or Unicode character from source and returns it. +// It only recognizes tokens t for which the respective Mode bit (1<<-t) is set. +// It returns EOF at the end of the source. It reports scanner errors (read and +// token errors) by calling s.Error, if not nil; otherwise it prints an error +// message to os.Stderr. +func (s *scanner) Scan() rune { + ch := s.Peek() + + // reset token text position + s.tokPos = -1 + s.Line = 0 + +redo: + // skip white space + for s.Whitespace&(1< 0 { + // common case: last character was not a '\n' + s.Line = s.line + s.Column = s.column + } else { + // last character was a '\n' + // (we cannot be at the beginning of the source + // since we have called next() at least once) + s.Line = s.line - 1 + s.Column = s.lastLineLen + } + + // determine token value + tok := ch + switch { + case s.isIdentRune(ch, 0): + if s.Mode&goscanner.ScanIdents != 0 { + tok = goscanner.Ident + ch = s.scanIdentifier() + } else { + ch = s.next() + } + case isDecimal(ch): + if s.Mode&(goscanner.ScanInts|goscanner.ScanFloats) != 0 { + tok, ch = s.scanNumber(ch, false) + } else { + ch = s.next() + } + default: + switch ch { + case goscanner.EOF: + break + case '"': + if s.Mode&goscanner.ScanStrings != 0 { + s.scanString('"') + tok = goscanner.String + } + ch = s.next() + case '\'': + if s.Mode&goscanner.ScanChars != 0 { + tok = s.scanChar() + } + ch = s.next() + case '.': + ch = s.next() + if isDecimal(ch) && s.Mode&goscanner.ScanFloats != 0 { + tok, ch = s.scanNumber(ch, true) + } + case '/': + ch = s.next() + if (ch == '/' || ch == '*') && s.Mode&goscanner.ScanComments != 0 { + if s.Mode&goscanner.SkipComments != 0 { + s.tokPos = -1 // don't collect token text + ch = s.scanComment(ch) + goto redo + } + ch = s.scanComment(ch) + tok = goscanner.Comment + } + default: + ch = s.next() + } + } + + // end of token text + s.tokEnd = s.srcPos - s.lastCharLen + + s.ch = ch + return tok +} + +// Pos returns the position of the character immediately after +// the character or token returned by the last call to Next or Scan. +// Use the Scanner's Position field for the start position of the most +// recently scanned token. +func (s *scanner) Pos() (pos goscanner.Position) { + pos.Filename = s.Filename + pos.Offset = s.srcBufOffset + s.srcPos - s.lastCharLen + switch { + case s.column > 0: + // common case: last character was not a '\n' + pos.Line = s.line + pos.Column = s.column + case s.lastLineLen > 0: + // last character was a '\n' + pos.Line = s.line - 1 + pos.Column = s.lastLineLen + default: + // at the beginning of the source + pos.Line = 1 + pos.Column = 1 + } + return +} + +// TokenText returns the string corresponding to the most recently scanned token. +// Valid after calling Scan and in calls of Scanner.Error. +func (s *scanner) TokenText() string { + if s.tokPos < 0 { + // no token text + return "" + } + + if s.tokEnd < s.tokPos { + // if EOF was reached, s.tokEnd is set to -1 (s.srcPos == 0) + s.tokEnd = s.tokPos + } + // s.tokEnd >= s.tokPos + + if s.tokBuf.Len() == 0 { + // common case: the entire token text is still in srcBuf + return string(s.srcBuf[s.tokPos:s.tokEnd]) + } + + // part of the token text was saved in tokBuf: save the rest in + // tokBuf as well and return its content + s.tokBuf.Write(s.srcBuf[s.tokPos:s.tokEnd]) + s.tokPos = s.tokEnd // ensure idempotency of TokenText() call + return s.tokBuf.String() +} From 1cf68e39db7c68aafa33155e0987404fb6a4ba45 Mon Sep 17 00:00:00 2001 From: JanHoefelmeyer Date: Tue, 8 Aug 2023 13:14:16 +0200 Subject: [PATCH 3/9] Remove fixed single quoted exceptions from test-suite_test.go --- test-suite_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test-suite_test.go b/test-suite_test.go index 59134d0..114ddb4 100644 --- a/test-suite_test.go +++ b/test-suite_test.go @@ -28,9 +28,6 @@ var knownParsingErrors = map[string]string{ `dot_notation_with_number`: `parsing error: $.2 :1:2 - 1:4 unexpected Float while scanning operator`, `dot_notation_with_number_-1`: `parsing error: $.-1 :1:3 - 1:4 unexpected "-" while scanning JSON select expected Ident, "." or "*"`, `dot_notation_with_number_on_object`: `parsing error: $.2 :1:2 - 1:4 unexpected Float while scanning operator`, - `dot_notation_with_single_quotes`: `parsing error: $.'key' :1:3 - 1:8 unexpected Char while scanning JSON select expected Ident, "." or "*"`, - `dot_notation_with_single_quotes_after_recursive_descent`: `parsing error: $..'key' :1:4 - 1:9 unexpected Char while scanning JSON mapper expected "[", Ident or "*"`, - `dot_notation_with_single_quotes_and_dot`: `parsing error: $.'some.key' :1:3 - 1:13 unexpected Char while scanning JSON select expected Ident, "." or "*"`, `dot_notation_without_root`: `parsing error: .key :1:1 - 1:2 unexpected "." while scanning extensions`, `empty`: `parsing error: - 1:1 unexpected EOF while scanning extensions`, `filter_expression_with_boolean_and_operator`: `parsing error: $[?(@.key>42 && @.key<44)] - 1:16 unknown operator &&`, @@ -50,7 +47,6 @@ var knownParsingErrors = map[string]string{ `filter_expression_with_equals_array_for_array_slice_with_range_1`: `parsing error: $[?(@[0:1]==[1])] :1:13 - 1:14 unexpected "[" while scanning extensions`, `filter_expression_with_equals_array_for_dot_notation_with_star`: `parsing error: $[?(@.*==[1,2])] :1:10 - 1:11 unexpected "[" while scanning extensions`, `filter_expression_with_equals_array_or_equals_true`: `parsing error: $[?(@.d==["v1","v2"] || (@.d == true))] :1:10 - 1:11 unexpected "[" while scanning extensions`, - `filter_expression_with_equals_array_with_single_quotes`: `parsing error: $[?(@.d==['v1','v2'])] :1:10 - 1:11 unexpected "[" while scanning extensions`, `filter_expression_with_equals_object`: `parsing error: $[?(@.d=={"k":"v"})] :1:10 - 1:11 unexpected "{" while scanning extensions`, `filter_expression_with_equals_string_with_single_quotes`: `parsing error: $[?(@.key=='value')] :1:12 - 1:19 could not parse string: invalid syntax`, `filter_expression_with_in_array_of_values`: `parsing error: $[?(@.d in [2, 3])] :1:9 - 1:11 unexpected Ident while scanning parentheses expected ")"`, From cdf9cf8aac51534a40b3269e72d40b59b990fcc9 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 8 Aug 2023 16:47:30 +0200 Subject: [PATCH 4/9] Add example for single quoted expression. --- example_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/example_test.go b/example_test.go index 6715599..c709e40 100644 --- a/example_test.go +++ b/example_test.go @@ -79,6 +79,29 @@ func ExampleGet_filter() { // II } +func ExampleGet_filterQuoted() { + v := interface{}(nil) + + json.Unmarshal([]byte(`[ + {"key":"alpha","value" : "I"}, + {"key":"beta","value" : "II"}, + {"key":"gamma","value" : "III"} + ]`), &v) + + values, err := jsonpath.Get(`$[? @.key=='beta'].value`, v) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + for _, value := range values.([]interface{}) { + fmt.Println(value) + } + + // Output: + // II +} + func Example_gval() { builder := gval.Full(jsonpath.PlaceholderExtension()) From fda50c5dc27f2831d4d0a44d7a41738805902f7a Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 8 Aug 2023 16:58:16 +0200 Subject: [PATCH 5/9] Remove char scanning. --- scanner.go | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/scanner.go b/scanner.go index e266c72..4591b01 100644 --- a/scanner.go +++ b/scanner.go @@ -43,12 +43,8 @@ import ( // literal tokens including Go identifiers. Comments will be skipped. // The result of Scan is one of these tokens or a Unicode character. -const ( - quotedString = goscanner.Comment + 2 - scanQuotedString = 1 << -quotedString -) -const jsonpathTokens = goscanner.ScanIdents | goscanner.ScanFloats | goscanner.ScanChars | +const jsonpathTokens = goscanner.ScanIdents | goscanner.ScanFloats | goscanner.ScanStrings | goscanner.ScanComments | goscanner.SkipComments const bufLen = 1024 // at least utf8.UTFMax @@ -587,17 +583,6 @@ func (s *scanner) scanString(quote rune) (n int) { return } -func (s *scanner) scanChar() rune { - if s.Mode&scanQuotedString != 0 { - s.scanString('\'') - return goscanner.String - } - if s.scanString('\'') != 1 { - s.error("invalid char literal") - } - return goscanner.Char -} - func (s *scanner) scanComment(ch rune) rune { // ch == '/' || ch == '*' if ch == '/' { @@ -690,8 +675,9 @@ redo: } ch = s.next() case '\'': - if s.Mode&goscanner.ScanChars != 0 { - tok = s.scanChar() + if s.Mode&goscanner.ScanStrings != 0 { + s.scanString('\'') + tok = goscanner.String } ch = s.next() case '.': From a707f965bcadef02da32cd537559743c24072234 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Tue, 8 Aug 2023 22:17:54 +0200 Subject: [PATCH 6/9] Make CreateScanner public --- jsonpath.go | 4 ++-- scanner.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jsonpath.go b/jsonpath.go index 33272ed..4a97f8b 100644 --- a/jsonpath.go +++ b/jsonpath.go @@ -50,7 +50,7 @@ var lang = func() gval.Language { gval.PrefixExtension('$', parseRootPath), gval.PrefixExtension('@', parseCurrentPath), ) - l.CreateScanner(createScanner) + l.CreateScanner(CreateScanner) return l }() @@ -65,7 +65,7 @@ var placeholderExtension = func() gval.Language { gval.PrefixExtension('{', parseJSONObject), gval.PrefixExtension('#', parsePlaceholder), ) - l.CreateScanner(createScanner) + l.CreateScanner(CreateScanner) return l }() diff --git a/scanner.go b/scanner.go index 4591b01..bcaf232 100644 --- a/scanner.go +++ b/scanner.go @@ -112,7 +112,7 @@ type scanner struct { goscanner.Position } -func createScanner() gval.Scanner { +func CreateScanner() gval.Scanner { return &scanner{} } From 8c8ddd1eb3f96e9f004551f3c5e6a1891231c84a Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Wed, 9 Aug 2023 12:29:00 +0200 Subject: [PATCH 7/9] Use custom unquote in scanner. Adjust tests. --- quote.go | 109 ++++++++++++++++++++++++++++++++++++++++ scanner.go | 7 +-- test-suite_test.go | 122 +++++++++++++++++++++++---------------------- 3 files changed, 172 insertions(+), 66 deletions(-) create mode 100644 quote.go diff --git a/quote.go b/quote.go new file mode 100644 index 0000000..14d120d --- /dev/null +++ b/quote.go @@ -0,0 +1,109 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package jsonpath + +import ( + "strconv" + "strings" + "unicode/utf8" +) + +// contains reports whether the string contains the byte c. +func contains(s string, c byte) bool { + return strings.IndexByte(s, c) != -1 +} + +// unquote interprets s as a single-quoted, double-quoted, +// or backquoted Go string literal, returning the string value +// that s quotes. (If s is single-quoted, it would be a Go +// character literal; Unquote returns the corresponding +// one-character string.) +func unquote(s string) (string, error) { + out, rem, err := unquoteInternal(s) + if len(rem) > 0 { + return "", strconv.ErrSyntax + } + return out, err +} + +// unquote parses a quoted string at the start of the input, +// returning the parsed prefix, the remaining suffix, and any parse errors. +// If unescape is true, the parsed prefix is unescaped, +// otherwise the input prefix is provided verbatim. +func unquoteInternal(in string) (out, rem string, err error) { + // In our use case it's a constant. + const unescape = true + // Determine the quote form and optimistically find the terminating quote. + if len(in) < 2 { + return "", in, strconv.ErrSyntax + } + quote := in[0] + end := strings.IndexByte(in[1:], quote) + if end < 0 { + return "", in, strconv.ErrSyntax + } + end += 2 // position after terminating quote; may be wrong if escape sequences are present + + switch quote { + case '"', '\'': + // Handle quoted strings without any escape sequences. + if !contains(in[:end], '\\') && !contains(in[:end], '\n') { + var ofs int + if quote == '\'' { + ofs = len(`"`) + } else { + ofs = len(`'`) + } + valid := utf8.ValidString(in[ofs : end-ofs]) + if valid { + out = in[:end] + if unescape { + out = out[1 : end-1] // exclude quotes + } + return out, in[end:], nil + } + } + + // Handle quoted strings with escape sequences. + var buf []byte + in0 := in + in = in[1:] // skip starting quote + if unescape { + buf = make([]byte, 0, 3*end/2) // try to avoid more allocations + } + for len(in) > 0 && in[0] != quote { + // Process the next character, + // rejecting any unescaped newline characters which are invalid. + r, multibyte, rem, err := strconv.UnquoteChar(in, quote) + if in[0] == '\n' || err != nil { + return "", in0, strconv.ErrSyntax + } + in = rem + + // Append the character if unescaping the input. + if unescape { + if r < utf8.RuneSelf || !multibyte { + buf = append(buf, byte(r)) + } else { + var arr [utf8.UTFMax]byte + n := utf8.EncodeRune(arr[:], r) + buf = append(buf, arr[:n]...) + } + } + } + + // Verify that the string ends with a terminating quote. + if !(len(in) > 0 && in[0] == quote) { + return "", in0, strconv.ErrSyntax + } + in = in[1:] // skip terminating quote + + if unescape { + return string(buf), in, nil + } + return in0[:len(in0)-len(in)], in, nil + default: + return "", in, strconv.ErrSyntax + } +} diff --git a/scanner.go b/scanner.go index bcaf232..0424cf2 100644 --- a/scanner.go +++ b/scanner.go @@ -19,7 +19,6 @@ import ( "fmt" "io" "os" - "strconv" goscanner "text/scanner" "unicode" "unicode/utf8" @@ -160,11 +159,7 @@ func (s *scanner) GetPosition() goscanner.Position { } func (s *scanner) Unquote(in string) (string, error) { - // Hack: Replace single quotes with double quotes. - if n := len(in); n > 1 && in[0] == '\'' && in[n-1] == '\'' { - in = `"` + in[1:n-1] + `"` - } - return strconv.Unquote(in) + return unquote(in) } // InitMode initializes a Scanner with a new source and returns s. diff --git a/test-suite_test.go b/test-suite_test.go index 114ddb4..1f65f84 100644 --- a/test-suite_test.go +++ b/test-suite_test.go @@ -1,4 +1,4 @@ -package jsonpath_test +package jsonpath import ( "context" @@ -6,72 +6,73 @@ import ( "testing" "github.com/PaesslerAG/gval" - "github.com/PaesslerAG/jsonpath" + //"github.com/PaesslerAG/jsonpath" "github.com/google/go-cmp/cmp" "gopkg.in/yaml.v3" ) var knownParsingErrors = map[string]string{ - `bracket_notation`: `parsing error: $['key'] :1:3 - 1:8 could not parse string: invalid syntax`, - `bracket_notation_on_object_without_key`: `parsing error: $['missing'] :1:3 - 1:12 could not parse string: invalid syntax`, - `bracket_notation_with_dot`: `parsing error: $['two.some'] :1:3 - 1:13 could not parse string: invalid syntax`, - `bracket_notation_with_quoted_dot_wildcard`: `parsing error: $['.*'] :1:3 - 1:7 could not parse string: invalid syntax`, - `bracket_notation_with_quoted_special_characters_combined`: `parsing error: $[':@."$,*\'\\'] :1:3 - 1:16 could not parse string: invalid syntax`, - `bracket_notation_with_string_including_dot_wildcard`: `parsing error: $['ni.*'] :1:3 - 1:9 could not parse string: invalid syntax`, - `dot_bracket_notation`: `parsing error: $.['key'] :1:3 - 1:4 unexpected "[" while scanning JSON select expected Ident, "." or "*"`, - `dot_bracket_notation_with_double_quotes`: `parsing error: $.["key"] :1:3 - 1:4 unexpected "[" while scanning JSON select expected Ident, "." or "*"`, - `dot_notation_after_recursive_descent_with_extra_dot`: `parsing error: $...key :1:4 - 1:5 unexpected "." while scanning JSON mapper expected "[", Ident or "*"`, - `dot_notation_after_union_with_keys`: `parsing error: $['one','three'].key :1:3 - 1:8 could not parse string: invalid syntax`, - `dot_notation_with_double_quotes`: `parsing error: $."key" :1:3 - 1:8 unexpected String while scanning JSON select expected Ident, "." or "*"`, - `dot_notation_with_double_quotes_after_recursive_descent`: `parsing error: $.."key" :1:4 - 1:9 unexpected String while scanning JSON mapper expected "[", Ident or "*"`, - `dot_notation_with_key_root_literal`: `parsing error: $.$ :1:3 - 1:4 unexpected "$" while scanning JSON select expected Ident, "." or "*"`, - `dot_notation_with_number`: `parsing error: $.2 :1:2 - 1:4 unexpected Float while scanning operator`, - `dot_notation_with_number_-1`: `parsing error: $.-1 :1:3 - 1:4 unexpected "-" while scanning JSON select expected Ident, "." or "*"`, - `dot_notation_with_number_on_object`: `parsing error: $.2 :1:2 - 1:4 unexpected Float while scanning operator`, - `dot_notation_without_root`: `parsing error: .key :1:1 - 1:2 unexpected "." while scanning extensions`, - `empty`: `parsing error: - 1:1 unexpected EOF while scanning extensions`, - `filter_expression_with_boolean_and_operator`: `parsing error: $[?(@.key>42 && @.key<44)] - 1:16 unknown operator &&`, - `filter_expression_with_boolean_and_operator_and_value_false`: `parsing error: $[?(@.key>0 && false)] - 1:15 unknown operator &&`, - `filter_expression_with_boolean_and_operator_and_value_true`: `parsing error: $[?(@.key>0 && true)] - 1:15 unknown operator &&`, - `filter_expression_with_boolean_or_operator`: `parsing error: $[?(@.key>43 || @.key<43)] - 1:16 unknown operator ||`, - `filter_expression_with_boolean_or_operator_and_value_false`: `parsing error: $[?(@.key>0 || false)] - 1:15 unknown operator ||`, - `filter_expression_with_boolean_or_operator_and_value_true`: `parsing error: $[?(@.key>0 || true)] - 1:15 unknown operator ||`, - `filter_expression_with_bracket_notation`: `parsing error: $[?(@['key']==42)] :1:7 - 1:12 could not parse string: invalid syntax`, - `filter_expression_with_bracket_notation_and_current_object_literal`: `parsing error: $[?(@['@key']==42)] :1:7 - 1:13 could not parse string: invalid syntax`, - `filter_expression_with_different_grouped_operators`: `parsing error: $[?(@.a && (@.b || @.c))] - 1:11 unknown operator &&`, - `filter_expression_with_different_ungrouped_operators`: `parsing error: $[?(@.a && @.b || @.c)] - 1:11 unknown operator &&`, - `filter_expression_with_dot_notation_with_dash`: `parsing error: $[?(@.key-dash == 'value')] :1:19 - 1:26 could not parse string: invalid syntax`, - `filter_expression_with_dot_notation_with_number`: `parsing error: $[?(@.2 == 'second')] :1:6 - 1:8 unexpected Float while scanning parentheses expected ")"`, - `filter_expression_with_dot_notation_with_number_on_array`: `parsing error: $[?(@.2 == 'third')] :1:6 - 1:8 unexpected Float while scanning parentheses expected ")"`, - `filter_expression_with_equals_array`: `parsing error: $[?(@.d==["v1","v2"])] :1:10 - 1:11 unexpected "[" while scanning extensions`, - `filter_expression_with_equals_array_for_array_slice_with_range_1`: `parsing error: $[?(@[0:1]==[1])] :1:13 - 1:14 unexpected "[" while scanning extensions`, - `filter_expression_with_equals_array_for_dot_notation_with_star`: `parsing error: $[?(@.*==[1,2])] :1:10 - 1:11 unexpected "[" while scanning extensions`, - `filter_expression_with_equals_array_or_equals_true`: `parsing error: $[?(@.d==["v1","v2"] || (@.d == true))] :1:10 - 1:11 unexpected "[" while scanning extensions`, - `filter_expression_with_equals_object`: `parsing error: $[?(@.d=={"k":"v"})] :1:10 - 1:11 unexpected "{" while scanning extensions`, - `filter_expression_with_equals_string_with_single_quotes`: `parsing error: $[?(@.key=='value')] :1:12 - 1:19 could not parse string: invalid syntax`, - `filter_expression_with_in_array_of_values`: `parsing error: $[?(@.d in [2, 3])] :1:9 - 1:11 unexpected Ident while scanning parentheses expected ")"`, - `filter_expression_with_in_current_object`: `parsing error: $[?(2 in @.d)] :1:7 - 1:9 unexpected Ident while scanning parentheses expected ")"`, - `filter_expression_with_length_function`: `parsing error: $[?(@.length() == 4)] :1:14 - 1:15 unexpected ")" while scanning extensions`, - `filter_expression_with_negation_and_equals`: `parsing error: $[?(!(@.key==42))] :1:5 - 1:6 unexpected "!" while scanning extensions`, - `filter_expression_with_negation_and_equals_array_or_equals_true`: `parsing error: $[?(!(@.d==["v1","v2"]) || (@.d == true))] :1:5 - 1:6 unexpected "!" while scanning extensions`, - `filter_expression_with_negation_and_less_than`: `parsing error: $[?(!(@.key<42))] :1:5 - 1:6 unexpected "!" while scanning extensions`, - `filter_expression_with_negation_and_without_value`: `parsing error: $[?(!@.key)] :1:5 - 1:6 unexpected "!" while scanning extensions`, - `filter_expression_with_not_equals_array_or_equals_true`: `parsing error: $[?((@.d!=["v1","v2"]) || (@.d == true))] :1:11 - 1:12 unexpected "[" while scanning extensions`, - `filter_expression_with_regular_expression`: `parsing error: $[?(@.name=~/hello.*/)] - 1:13 unknown operator =~`, - `filter_expression_with_regular_expression_from_member`: `parsing error: $[?(@.name=~/@.pattern/)] - 1:13 unknown operator =~`, - `filter_expression_with_subpaths`: `parsing error: $[?(@.address.city=='Berlin')] :1:21 - 1:29 could not parse string: invalid syntax`, - `filter_expression_with_triple_equal`: `parsing error: $[?(@.key===42)] :1:12 - 1:13 unexpected "=" while scanning extensions`, - `function_sum`: `parsing error: $.data.sum() :1:12 - 1:13 unexpected ")" while scanning extensions`, - `recursive_descent`: `parsing error: $.. :1:4 - 1:4 unexpected EOF while scanning JSON mapper expected "[", Ident or "*"`, - `recursive_descent_after_dot_notation`: `parsing error: $.key.. :1:8 - 1:8 unexpected EOF while scanning JSON mapper expected "[", Ident or "*"`, - `union_with_filter`: `parsing error: $[?(@.key<3),?(@.key>6)] :1:13 - 1:14 mixed 63 and 44 in JSON bracket`, - `union_with_keys`: `parsing error: $['key','another'] :1:3 - 1:8 could not parse string: invalid syntax`, - `union_with_keys_on_object_without_key`: `parsing error: $['missing','key'] :1:3 - 1:12 could not parse string: invalid syntax`, - `union_with_repeated_matches_after_dot_notation_with_wildcard`: `parsing error: $.*[0,:5] :1:7 - 1:8 mixed 44 and 58 in JSON bracket`, - `union_with_slice_and_number`: `parsing error: $[1:3,4] :1:6 - 1:7 mixed 58 and 44 in JSON bracket`, + //`bracket_notation`: `parsing error: $['key'] :1:3 - 1:8 could not parse string: invalid syntax`, + //`bracket_notation_on_object_without_key`: `parsing error: $['missing'] :1:3 - 1:12 could not parse string: invalid syntax`, + //`bracket_notation_with_dot`: `parsing error: $['two.some'] :1:3 - 1:13 could not parse string: invalid syntax`, + //`bracket_notation_with_quoted_dot_wildcard`: `parsing error: $['.*'] :1:3 - 1:7 could not parse string: invalid syntax`, + //`bracket_notation_with_quoted_special_characters_combined`: `parsing error: $[':@."$,*\'\\'] :1:3 - 1:16 could not parse string: invalid syntax`, + //`bracket_notation_with_string_including_dot_wildcard`: `parsing error: $['ni.*'] :1:3 - 1:9 could not parse string: invalid syntax`, + //`dot_bracket_notation`: `parsing error: $.['key'] :1:3 - 1:4 unexpected "[" while scanning JSON select expected Ident, "." or "*"`, + //`dot_bracket_notation_with_double_quotes`: `parsing error: $.["key"] :1:3 - 1:4 unexpected "[" while scanning JSON select expected Ident, "." or "*"`, + //`dot_notation_after_recursive_descent_with_extra_dot`: `parsing error: $...key :1:4 - 1:5 unexpected "." while scanning JSON mapper expected "[", Ident or "*"`, + //`dot_notation_after_union_with_keys`: `parsing error: $['one','three'].key :1:3 - 1:8 could not parse string: invalid syntax`, + //`dot_notation_with_double_quotes`: `parsing error: $."key" :1:3 - 1:8 unexpected String while scanning JSON select expected Ident, "." or "*"`, + //`dot_notation_with_double_quotes_after_recursive_descent`: `parsing error: $.."key" :1:4 - 1:9 unexpected String while scanning JSON mapper expected "[", Ident or "*"`, + //`dot_notation_with_key_root_literal`: `parsing error: $.$ :1:3 - 1:4 unexpected "$" while scanning JSON select expected Ident, "." or "*"`, + //`dot_notation_with_number`: `parsing error: $.2 :1:2 - 1:4 unexpected Float while scanning operator`, + `dot_notation_with_number_-1`: `parsing error: $.-1 :1:3 - 1:4 unexpected "-" while scanning JSON select expected Ident, "." or "*"`, + `dot_notation_with_number_on_object`: `parsing error: $.2 :1:2 - 1:4 unexpected Float while scanning operator`, + //`dot_notation_without_root`: `parsing error: .key :1:1 - 1:2 unexpected "." while scanning extensions`, + //`empty`: `parsing error: - 1:1 unexpected EOF while scanning extensions`, + `filter_expression_with_boolean_and_operator`: `parsing error: $[?(@.key>42 && @.key<44)] - 1:16 unknown operator &&`, + //`filter_expression_with_boolean_and_operator_and_value_false`: `parsing error: $[?(@.key>0 && false)] - 1:15 unknown operator &&`, + //`filter_expression_with_boolean_and_operator_and_value_true`: `parsing error: $[?(@.key>0 && true)] - 1:15 unknown operator &&`, + `filter_expression_with_boolean_or_operator`: `parsing error: $[?(@.key>43 || @.key<43)] - 1:16 unknown operator ||`, + //`filter_expression_with_boolean_or_operator_and_value_false`: `parsing error: $[?(@.key>0 || false)] - 1:15 unknown operator ||`, + //`filter_expression_with_boolean_or_operator_and_value_true`: `parsing error: $[?(@.key>0 || true)] - 1:15 unknown operator ||`, + //`filter_expression_with_bracket_notation`: `parsing error: $[?(@['key']==42)] :1:7 - 1:12 could not parse string: invalid syntax`, + //`filter_expression_with_bracket_notation_and_current_object_literal`: `parsing error: $[?(@['@key']==42)] :1:7 - 1:13 could not parse string: invalid syntax`, + //`filter_expression_with_different_grouped_operators`: `parsing error: $[?(@.a && (@.b || @.c))] - 1:11 unknown operator &&`, + //`filter_expression_with_different_ungrouped_operators`: `parsing error: $[?(@.a && @.b || @.c)] - 1:11 unknown operator &&`, + //`filter_expression_with_dot_notation_with_dash`: `parsing error: $[?(@.key-dash == 'value')] :1:19 - 1:26 could not parse string: invalid syntax`, + //`filter_expression_with_dot_notation_with_number`: `parsing error: $[?(@.2 == 'second')] :1:6 - 1:8 unexpected Float while scanning parentheses expected ")"`, + //`filter_expression_with_dot_notation_with_number_on_array`: `parsing error: $[?(@.2 == 'third')] :1:6 - 1:8 unexpected Float while scanning parentheses expected ")"`, + //`filter_expression_with_equals_array`: `parsing error: $[?(@.d==["v1","v2"])] :1:10 - 1:11 unexpected "[" while scanning extensions`, + //`filter_expression_with_equals_array_for_array_slice_with_range_1`: `parsing error: $[?(@[0:1]==[1])] :1:13 - 1:14 unexpected "[" while scanning extensions`, + //`filter_expression_with_equals_array_for_dot_notation_with_star`: `parsing error: $[?(@.*==[1,2])] :1:10 - 1:11 unexpected "[" while scanning extensions`, + //`filter_expression_with_equals_array_or_equals_true`: `parsing error: $[?(@.d==["v1","v2"] || (@.d == true))] :1:10 - 1:11 unexpected "[" while scanning extensions`, + //`filter_expression_with_equals_object`: `parsing error: $[?(@.d=={"k":"v"})] :1:10 - 1:11 unexpected "{" while scanning extensions`, + //`filter_expression_with_equals_string_with_single_quotes`: `parsing error: $[?(@.key=='value')] :1:12 - 1:19 could not parse string: invalid syntax`, + //`filter_expression_with_in_array_of_values`: `parsing error: $[?(@.d in [2, 3])] :1:9 - 1:11 unexpected Ident while scanning parentheses expected ")"`, + //`filter_expression_with_in_current_object`: `parsing error: $[?(2 in @.d)] :1:7 - 1:9 unexpected Ident while scanning parentheses expected ")"`, + //`filter_expression_with_length_function`: `parsing error: $[?(@.length() == 4)] :1:14 - 1:15 unexpected ")" while scanning extensions`, + //`filter_expression_with_negation_and_equals`: `parsing error: $[?(!(@.key==42))] :1:5 - 1:6 unexpected "!" while scanning extensions`, + //`filter_expression_with_negation_and_equals_array_or_equals_true`: `parsing error: $[?(!(@.d==["v1","v2"]) || (@.d == true))] :1:5 - 1:6 unexpected "!" while scanning extensions`, + //`filter_expression_with_negation_and_less_than`: `parsing error: $[?(!(@.key<42))] :1:5 - 1:6 unexpected "!" while scanning extensions`, + //`filter_expression_with_negation_and_without_value`: `parsing error: $[?(!@.key)] :1:5 - 1:6 unexpected "!" while scanning extensions`, + //`filter_expression_with_not_equals_array_or_equals_true`: `parsing error: $[?((@.d!=["v1","v2"]) || (@.d == true))] :1:11 - 1:12 unexpected "[" while scanning extensions`, + //`filter_expression_with_regular_expression`: `parsing error: $[?(@.name=~/hello.*/)] - 1:13 unknown operator =~`, + //`filter_expression_with_regular_expression_from_member`: `parsing error: $[?(@.name=~/@.pattern/)] - 1:13 unknown operator =~`, + //`filter_expression_with_subpaths`: `parsing error: $[?(@.address.city=='Berlin')] :1:21 - 1:29 could not parse string: invalid syntax`, + //`filter_expression_with_triple_equal`: `parsing error: $[?(@.key===42)] :1:12 - 1:13 unexpected "=" while scanning extensions`, + //`function_sum`: `parsing error: $.data.sum() :1:12 - 1:13 unexpected ")" while scanning extensions`, + //`recursive_descent`: `parsing error: $.. :1:4 - 1:4 unexpected EOF while scanning JSON mapper expected "[", Ident or "*"`, + //`recursive_descent_after_dot_notation`: `parsing error: $.key.. :1:8 - 1:8 unexpected EOF while scanning JSON mapper expected "[", Ident or "*"`, + //`union_with_filter`: `parsing error: $[?(@.key<3),?(@.key>6)] :1:13 - 1:14 mixed 63 and 44 in JSON bracket`, + //`union_with_keys`: `parsing error: $['key','another'] :1:3 - 1:8 could not parse string: invalid syntax`, + //`union_with_keys_on_object_without_key`: `parsing error: $['missing','key'] :1:3 - 1:12 could not parse string: invalid syntax`, + //`union_with_repeated_matches_after_dot_notation_with_wildcard`: `parsing error: $.*[0,:5] :1:7 - 1:8 mixed 44 and 58 in JSON bracket`, + //`union_with_slice_and_number`: `parsing error: $[1:3,4] :1:6 - 1:7 mixed 58 and 44 in JSON bracket`, } var knownEvaluationErrors = map[string]string{ + `bracket_notation_on_object_without_key`: `failed to evaluate selector: $['missing'] -> unknown key missing`, `bracket_notation_with_NFC_path_on_NFD_key`: `unknown key ΓΌ`, `bracket_notation_with_number_on_string`: `unsupported value type string for select, expected map[string]interface{}, []interface{} or Array`, `bracket_notation_with_quoted_wildcard_literal_on_object_without_key`: `unknown key *`, @@ -134,9 +135,10 @@ func TestRegressionDocument(t *testing.T) { } // gval language language := gval.NewLanguage( - jsonpath.Language(), + Language(), gval.Arithmetic(), ) + // focused tests focused := map[string]struct{}{ //"array_slice_with_positive_start_and_negative_end_and_range_of_1": {}, From 59fce14b21966fdde3f785d86182e5b536909c47 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Wed, 9 Aug 2023 14:49:48 +0200 Subject: [PATCH 8/9] Adjusted doc comments. --- quote.go | 10 +++++++++- scanner.go | 37 ++++++++----------------------------- 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/quote.go b/quote.go index 14d120d..a0cf99f 100644 --- a/quote.go +++ b/quote.go @@ -1,7 +1,15 @@ +package jsonpath + // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package jsonpath +// This is mainly taken from +// +// https://cs.opensource.google/go/go/+/refs/tags/go1.21.0:src/strconv/quote.go +// +// and adjusted to meet the needs of unquoting JSONPath strings. +// Mainly handling single quoted strings right and removed support for +// Go raw strings. import ( "strconv" diff --git a/scanner.go b/scanner.go index 0424cf2..b34bec6 100644 --- a/scanner.go +++ b/scanner.go @@ -1,18 +1,13 @@ +package jsonpath + // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. - -// Package scanner provides a scanner and tokenizer for UTF-8-encoded text. -// It takes an io.Reader providing the source, which then can be tokenized -// through repeated calls to the Scan function. For compatibility with -// existing tools, the NUL character is not allowed. If the first character -// in the source is a UTF-8 encoded byte order mark (BOM), it is discarded. +// This is mainly taken from // -// By default, a Scanner skips white space and Go comments and recognizes all -// literals as defined by the Go language specification. It may be -// customized to recognize only a subset of those literals and to recognize -// different identifier and white space characters. -package jsonpath +// https://cs.opensource.google/go/go/+/refs/tags/go1.21.0:src/text/scanner/scanner.go +// +// and adjusted to meet the needs of parsing JSONPath expressions. import ( "bytes" @@ -26,29 +21,12 @@ import ( "github.com/PaesslerAG/gval" ) -// Predefined mode bits to control recognition of tokens. For instance, -// to configure a Scanner such that it only recognizes (Go) identifiers, -// integers, and skips comments, set the Scanner's Mode field to: -// -// ScanIdents | ScanInts | SkipComments -// -// With the exceptions of comments, which are skipped if SkipComments is -// set, unrecognized tokens are not ignored. Instead, the scanner simply -// returns the respective individual characters (or possibly sub-tokens). -// For instance, if the mode is ScanIdents (not ScanStrings), the string -// "foo" is scanned as the token sequence '"' Ident '"'. -// -// Use GoTokens to configure the Scanner such that it accepts all Go -// literal tokens including Go identifiers. Comments will be skipped. - -// The result of Scan is one of these tokens or a Unicode character. - const jsonpathTokens = goscanner.ScanIdents | goscanner.ScanFloats | goscanner.ScanStrings | goscanner.ScanComments | goscanner.SkipComments const bufLen = 1024 // at least utf8.UTFMax -// A scanner implements reading of Unicode characters and tokens from an io.Reader. +// A scanner implements a [github.com/PaesslerAG/gval/Scanner]. type scanner struct { // Input src io.Reader @@ -111,6 +89,7 @@ type scanner struct { goscanner.Position } +// CreateScanner returns new Scanner suitable to scan JSONPath expressions. func CreateScanner() gval.Scanner { return &scanner{} } From 5cc41ae2a9a6c3ed5422c2e3621c71b6478db315 Mon Sep 17 00:00:00 2001 From: "Sascha L. Teichmann" Date: Wed, 9 Aug 2023 15:10:08 +0200 Subject: [PATCH 9/9] Remove out-commented expected parsing errors in test suite. --- test-suite_test.go | 59 +++------------------------------------------- 1 file changed, 3 insertions(+), 56 deletions(-) diff --git a/test-suite_test.go b/test-suite_test.go index 1f65f84..7feab14 100644 --- a/test-suite_test.go +++ b/test-suite_test.go @@ -12,63 +12,10 @@ import ( ) var knownParsingErrors = map[string]string{ - //`bracket_notation`: `parsing error: $['key'] :1:3 - 1:8 could not parse string: invalid syntax`, - //`bracket_notation_on_object_without_key`: `parsing error: $['missing'] :1:3 - 1:12 could not parse string: invalid syntax`, - //`bracket_notation_with_dot`: `parsing error: $['two.some'] :1:3 - 1:13 could not parse string: invalid syntax`, - //`bracket_notation_with_quoted_dot_wildcard`: `parsing error: $['.*'] :1:3 - 1:7 could not parse string: invalid syntax`, - //`bracket_notation_with_quoted_special_characters_combined`: `parsing error: $[':@."$,*\'\\'] :1:3 - 1:16 could not parse string: invalid syntax`, - //`bracket_notation_with_string_including_dot_wildcard`: `parsing error: $['ni.*'] :1:3 - 1:9 could not parse string: invalid syntax`, - //`dot_bracket_notation`: `parsing error: $.['key'] :1:3 - 1:4 unexpected "[" while scanning JSON select expected Ident, "." or "*"`, - //`dot_bracket_notation_with_double_quotes`: `parsing error: $.["key"] :1:3 - 1:4 unexpected "[" while scanning JSON select expected Ident, "." or "*"`, - //`dot_notation_after_recursive_descent_with_extra_dot`: `parsing error: $...key :1:4 - 1:5 unexpected "." while scanning JSON mapper expected "[", Ident or "*"`, - //`dot_notation_after_union_with_keys`: `parsing error: $['one','three'].key :1:3 - 1:8 could not parse string: invalid syntax`, - //`dot_notation_with_double_quotes`: `parsing error: $."key" :1:3 - 1:8 unexpected String while scanning JSON select expected Ident, "." or "*"`, - //`dot_notation_with_double_quotes_after_recursive_descent`: `parsing error: $.."key" :1:4 - 1:9 unexpected String while scanning JSON mapper expected "[", Ident or "*"`, - //`dot_notation_with_key_root_literal`: `parsing error: $.$ :1:3 - 1:4 unexpected "$" while scanning JSON select expected Ident, "." or "*"`, - //`dot_notation_with_number`: `parsing error: $.2 :1:2 - 1:4 unexpected Float while scanning operator`, - `dot_notation_with_number_-1`: `parsing error: $.-1 :1:3 - 1:4 unexpected "-" while scanning JSON select expected Ident, "." or "*"`, - `dot_notation_with_number_on_object`: `parsing error: $.2 :1:2 - 1:4 unexpected Float while scanning operator`, - //`dot_notation_without_root`: `parsing error: .key :1:1 - 1:2 unexpected "." while scanning extensions`, - //`empty`: `parsing error: - 1:1 unexpected EOF while scanning extensions`, + `dot_notation_with_number_-1`: `parsing error: $.-1 :1:3 - 1:4 unexpected "-" while scanning JSON select expected Ident, "." or "*"`, + `dot_notation_with_number_on_object`: `parsing error: $.2 :1:2 - 1:4 unexpected Float while scanning operator`, `filter_expression_with_boolean_and_operator`: `parsing error: $[?(@.key>42 && @.key<44)] - 1:16 unknown operator &&`, - //`filter_expression_with_boolean_and_operator_and_value_false`: `parsing error: $[?(@.key>0 && false)] - 1:15 unknown operator &&`, - //`filter_expression_with_boolean_and_operator_and_value_true`: `parsing error: $[?(@.key>0 && true)] - 1:15 unknown operator &&`, - `filter_expression_with_boolean_or_operator`: `parsing error: $[?(@.key>43 || @.key<43)] - 1:16 unknown operator ||`, - //`filter_expression_with_boolean_or_operator_and_value_false`: `parsing error: $[?(@.key>0 || false)] - 1:15 unknown operator ||`, - //`filter_expression_with_boolean_or_operator_and_value_true`: `parsing error: $[?(@.key>0 || true)] - 1:15 unknown operator ||`, - //`filter_expression_with_bracket_notation`: `parsing error: $[?(@['key']==42)] :1:7 - 1:12 could not parse string: invalid syntax`, - //`filter_expression_with_bracket_notation_and_current_object_literal`: `parsing error: $[?(@['@key']==42)] :1:7 - 1:13 could not parse string: invalid syntax`, - //`filter_expression_with_different_grouped_operators`: `parsing error: $[?(@.a && (@.b || @.c))] - 1:11 unknown operator &&`, - //`filter_expression_with_different_ungrouped_operators`: `parsing error: $[?(@.a && @.b || @.c)] - 1:11 unknown operator &&`, - //`filter_expression_with_dot_notation_with_dash`: `parsing error: $[?(@.key-dash == 'value')] :1:19 - 1:26 could not parse string: invalid syntax`, - //`filter_expression_with_dot_notation_with_number`: `parsing error: $[?(@.2 == 'second')] :1:6 - 1:8 unexpected Float while scanning parentheses expected ")"`, - //`filter_expression_with_dot_notation_with_number_on_array`: `parsing error: $[?(@.2 == 'third')] :1:6 - 1:8 unexpected Float while scanning parentheses expected ")"`, - //`filter_expression_with_equals_array`: `parsing error: $[?(@.d==["v1","v2"])] :1:10 - 1:11 unexpected "[" while scanning extensions`, - //`filter_expression_with_equals_array_for_array_slice_with_range_1`: `parsing error: $[?(@[0:1]==[1])] :1:13 - 1:14 unexpected "[" while scanning extensions`, - //`filter_expression_with_equals_array_for_dot_notation_with_star`: `parsing error: $[?(@.*==[1,2])] :1:10 - 1:11 unexpected "[" while scanning extensions`, - //`filter_expression_with_equals_array_or_equals_true`: `parsing error: $[?(@.d==["v1","v2"] || (@.d == true))] :1:10 - 1:11 unexpected "[" while scanning extensions`, - //`filter_expression_with_equals_object`: `parsing error: $[?(@.d=={"k":"v"})] :1:10 - 1:11 unexpected "{" while scanning extensions`, - //`filter_expression_with_equals_string_with_single_quotes`: `parsing error: $[?(@.key=='value')] :1:12 - 1:19 could not parse string: invalid syntax`, - //`filter_expression_with_in_array_of_values`: `parsing error: $[?(@.d in [2, 3])] :1:9 - 1:11 unexpected Ident while scanning parentheses expected ")"`, - //`filter_expression_with_in_current_object`: `parsing error: $[?(2 in @.d)] :1:7 - 1:9 unexpected Ident while scanning parentheses expected ")"`, - //`filter_expression_with_length_function`: `parsing error: $[?(@.length() == 4)] :1:14 - 1:15 unexpected ")" while scanning extensions`, - //`filter_expression_with_negation_and_equals`: `parsing error: $[?(!(@.key==42))] :1:5 - 1:6 unexpected "!" while scanning extensions`, - //`filter_expression_with_negation_and_equals_array_or_equals_true`: `parsing error: $[?(!(@.d==["v1","v2"]) || (@.d == true))] :1:5 - 1:6 unexpected "!" while scanning extensions`, - //`filter_expression_with_negation_and_less_than`: `parsing error: $[?(!(@.key<42))] :1:5 - 1:6 unexpected "!" while scanning extensions`, - //`filter_expression_with_negation_and_without_value`: `parsing error: $[?(!@.key)] :1:5 - 1:6 unexpected "!" while scanning extensions`, - //`filter_expression_with_not_equals_array_or_equals_true`: `parsing error: $[?((@.d!=["v1","v2"]) || (@.d == true))] :1:11 - 1:12 unexpected "[" while scanning extensions`, - //`filter_expression_with_regular_expression`: `parsing error: $[?(@.name=~/hello.*/)] - 1:13 unknown operator =~`, - //`filter_expression_with_regular_expression_from_member`: `parsing error: $[?(@.name=~/@.pattern/)] - 1:13 unknown operator =~`, - //`filter_expression_with_subpaths`: `parsing error: $[?(@.address.city=='Berlin')] :1:21 - 1:29 could not parse string: invalid syntax`, - //`filter_expression_with_triple_equal`: `parsing error: $[?(@.key===42)] :1:12 - 1:13 unexpected "=" while scanning extensions`, - //`function_sum`: `parsing error: $.data.sum() :1:12 - 1:13 unexpected ")" while scanning extensions`, - //`recursive_descent`: `parsing error: $.. :1:4 - 1:4 unexpected EOF while scanning JSON mapper expected "[", Ident or "*"`, - //`recursive_descent_after_dot_notation`: `parsing error: $.key.. :1:8 - 1:8 unexpected EOF while scanning JSON mapper expected "[", Ident or "*"`, - //`union_with_filter`: `parsing error: $[?(@.key<3),?(@.key>6)] :1:13 - 1:14 mixed 63 and 44 in JSON bracket`, - //`union_with_keys`: `parsing error: $['key','another'] :1:3 - 1:8 could not parse string: invalid syntax`, - //`union_with_keys_on_object_without_key`: `parsing error: $['missing','key'] :1:3 - 1:12 could not parse string: invalid syntax`, - //`union_with_repeated_matches_after_dot_notation_with_wildcard`: `parsing error: $.*[0,:5] :1:7 - 1:8 mixed 44 and 58 in JSON bracket`, - //`union_with_slice_and_number`: `parsing error: $[1:3,4] :1:6 - 1:7 mixed 58 and 44 in JSON bracket`, + `filter_expression_with_boolean_or_operator`: `parsing error: $[?(@.key>43 || @.key<43)] - 1:16 unknown operator ||`, } var knownEvaluationErrors = map[string]string{