From 573107bd9e4c397810d84ac1b6a1d0feb4bbd419 Mon Sep 17 00:00:00 2001 From: Niall Newman Date: Thu, 8 Dec 2022 15:54:08 +0000 Subject: [PATCH 01/16] Copy byte array when unmarshalling RawMessage --- raw.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/raw.go b/raw.go index 81bd002e..ee7367ae 100644 --- a/raw.go +++ b/raw.go @@ -25,7 +25,8 @@ func (v *RawMessage) UnmarshalEasyJSON(l *jlexer.Lexer) { // UnmarshalJSON implements encoding/json.Unmarshaler interface. func (v *RawMessage) UnmarshalJSON(data []byte) error { - *v = data + *v = make([]byte, len(data)) + copy(*v, data) return nil } From 37cdbe9d67307cc60e0425c92c5ee9bf782527ca Mon Sep 17 00:00:00 2001 From: guoguangwu Date: Thu, 29 Jun 2023 15:18:30 +0800 Subject: [PATCH 02/16] chore: use ret.String() instead of string(ret.Bytes()) --- gen/generator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gen/generator.go b/gen/generator.go index 79f4d6f7..8598fe0b 100644 --- a/gen/generator.go +++ b/gen/generator.go @@ -517,7 +517,7 @@ func camelToSnake(name string) string { if lastUpper != 0 { ret.WriteRune(unicode.ToLower(lastUpper)) } - return string(ret.Bytes()) + return ret.String() } func (SnakeCaseFieldNamer) GetJSONFieldName(t reflect.Type, f reflect.StructField) string { From 3bd36b7ac9e3174500af6886480f45543b3a30eb Mon Sep 17 00:00:00 2001 From: Alexander Tumin Date: Sat, 6 Jan 2024 18:16:38 +0300 Subject: [PATCH 03/16] Make current token kind public and accessible via Lexer.CurrentToken Updated implementation of #308 --- jlexer/lexer.go | 113 ++++++++++++++++++++++++------------------- jlexer/lexer_test.go | 15 ++++++ 2 files changed, 78 insertions(+), 50 deletions(-) diff --git a/jlexer/lexer.go b/jlexer/lexer.go index b5f5e261..a27705b1 100644 --- a/jlexer/lexer.go +++ b/jlexer/lexer.go @@ -19,21 +19,21 @@ import ( "github.com/josharian/intern" ) -// tokenKind determines type of a token. -type tokenKind byte +// TokenKind determines type of a token. +type TokenKind byte const ( - tokenUndef tokenKind = iota // No token. - tokenDelim // Delimiter: one of '{', '}', '[' or ']'. - tokenString // A string literal, e.g. "abc\u1234" - tokenNumber // Number literal, e.g. 1.5e5 - tokenBool // Boolean literal: true or false. - tokenNull // null keyword. + TokenUndef TokenKind = iota // No token. + TokenDelim // Delimiter: one of '{', '}', '[' or ']'. + TokenString // A string literal, e.g. "abc\u1234" + TokenNumber // Number literal, e.g. 1.5e5 + TokenBool // Boolean literal: true or false. + TokenNull // null keyword. ) // token describes a single token: type, position in the input and value. type token struct { - kind tokenKind // Type of a token. + kind TokenKind // Type of a token. boolValue bool // Value if a boolean literal token. byteValueCloned bool // true if byteValue was allocated and does not refer to original json body @@ -47,7 +47,7 @@ type Lexer struct { start int // Start of the current token. pos int // Current unscanned position in the input stream. - token token // Last scanned token, if token.kind != tokenUndef. + token token // Last scanned token, if token.kind != TokenUndef. firstElement bool // Whether current element is the first in array or an object. wantSep byte // A comma or a colon character, which need to occur before a token. @@ -59,7 +59,7 @@ type Lexer struct { // FetchToken scans the input for the next token. func (r *Lexer) FetchToken() { - r.token.kind = tokenUndef + r.token.kind = TokenUndef r.start = r.pos // Check if r.Data has r.pos element @@ -90,7 +90,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } - r.token.kind = tokenString + r.token.kind = TokenString r.fetchString() return @@ -99,7 +99,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } r.firstElement = true - r.token.kind = tokenDelim + r.token.kind = TokenDelim r.token.delimValue = r.Data[r.pos] r.pos++ return @@ -109,7 +109,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } r.wantSep = 0 - r.token.kind = tokenDelim + r.token.kind = TokenDelim r.token.delimValue = r.Data[r.pos] r.pos++ return @@ -118,7 +118,7 @@ func (r *Lexer) FetchToken() { if r.wantSep != 0 { r.errSyntax() } - r.token.kind = tokenNumber + r.token.kind = TokenNumber r.fetchNumber() return @@ -127,7 +127,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } - r.token.kind = tokenNull + r.token.kind = TokenNull r.fetchNull() return @@ -136,7 +136,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } - r.token.kind = tokenBool + r.token.kind = TokenBool r.token.boolValue = true r.fetchTrue() return @@ -146,7 +146,7 @@ func (r *Lexer) FetchToken() { r.errSyntax() } - r.token.kind = tokenBool + r.token.kind = TokenBool r.token.boolValue = false r.fetchFalse() return @@ -391,7 +391,7 @@ func (r *Lexer) fetchString() { // scanToken scans the next token if no token is currently available in the lexer. func (r *Lexer) scanToken() { - if r.token.kind != tokenUndef || r.fatalError != nil { + if r.token.kind != TokenUndef || r.fatalError != nil { return } @@ -400,7 +400,7 @@ func (r *Lexer) scanToken() { // consume resets the current token to allow scanning the next one. func (r *Lexer) consume() { - r.token.kind = tokenUndef + r.token.kind = TokenUndef r.token.byteValueCloned = false r.token.delimValue = 0 } @@ -443,10 +443,10 @@ func (r *Lexer) errInvalidToken(expected string) { switch expected { case "[": r.token.delimValue = ']' - r.token.kind = tokenDelim + r.token.kind = TokenDelim case "{": r.token.delimValue = '}' - r.token.kind = tokenDelim + r.token.kind = TokenDelim } r.addNonfatalError(&LexerError{ Reason: fmt.Sprintf("expected %s", expected), @@ -475,7 +475,7 @@ func (r *Lexer) GetPos() int { // Delim consumes a token and verifies that it is the given delimiter. func (r *Lexer) Delim(c byte) { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } @@ -489,7 +489,7 @@ func (r *Lexer) Delim(c byte) { // IsDelim returns true if there was no scanning error and next token is the given delimiter. func (r *Lexer) IsDelim(c byte) bool { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } return !r.Ok() || r.token.delimValue == c @@ -497,10 +497,10 @@ func (r *Lexer) IsDelim(c byte) bool { // Null verifies that the next token is null and consumes it. func (r *Lexer) Null() { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenNull { + if !r.Ok() || r.token.kind != TokenNull { r.errInvalidToken("null") } r.consume() @@ -508,15 +508,15 @@ func (r *Lexer) Null() { // IsNull returns true if the next token is a null keyword. func (r *Lexer) IsNull() bool { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - return r.Ok() && r.token.kind == tokenNull + return r.Ok() && r.token.kind == TokenNull } // Skip skips a single token. func (r *Lexer) Skip() { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } r.consume() @@ -621,10 +621,10 @@ func (r *Lexer) Consumed() { } func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenString { + if !r.Ok() || r.token.kind != TokenString { r.errInvalidToken("string") return "", nil } @@ -664,10 +664,10 @@ func (r *Lexer) UnsafeFieldName(skipUnescape bool) string { // String reads a string literal. func (r *Lexer) String() string { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenString { + if !r.Ok() || r.token.kind != TokenString { r.errInvalidToken("string") return "" } @@ -687,10 +687,10 @@ func (r *Lexer) String() string { // StringIntern reads a string literal, and performs string interning on it. func (r *Lexer) StringIntern() string { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenString { + if !r.Ok() || r.token.kind != TokenString { r.errInvalidToken("string") return "" } @@ -705,10 +705,10 @@ func (r *Lexer) StringIntern() string { // Bytes reads a string literal and base64 decodes it into a byte slice. func (r *Lexer) Bytes() []byte { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenString { + if !r.Ok() || r.token.kind != TokenString { r.errInvalidToken("string") return nil } @@ -731,10 +731,10 @@ func (r *Lexer) Bytes() []byte { // Bool reads a true or false boolean keyword. func (r *Lexer) Bool() bool { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenBool { + if !r.Ok() || r.token.kind != TokenBool { r.errInvalidToken("bool") return false } @@ -744,10 +744,10 @@ func (r *Lexer) Bool() bool { } func (r *Lexer) number() string { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } - if !r.Ok() || r.token.kind != tokenNumber { + if !r.Ok() || r.token.kind != TokenNumber { r.errInvalidToken("number") return "" } @@ -1151,7 +1151,7 @@ func (r *Lexer) GetNonFatalErrors() []*LexerError { // JsonNumber fetches and json.Number from 'encoding/json' package. // Both int, float or string, contains them are valid values func (r *Lexer) JsonNumber() json.Number { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } if !r.Ok() { @@ -1160,11 +1160,11 @@ func (r *Lexer) JsonNumber() json.Number { } switch r.token.kind { - case tokenString: + case TokenString: return json.Number(r.String()) - case tokenNumber: + case TokenNumber: return json.Number(r.Raw()) - case tokenNull: + case TokenNull: r.Null() return json.Number("") default: @@ -1175,7 +1175,7 @@ func (r *Lexer) JsonNumber() json.Number { // Interface fetches an interface{} analogous to the 'encoding/json' package. func (r *Lexer) Interface() interface{} { - if r.token.kind == tokenUndef && r.Ok() { + if r.token.kind == TokenUndef && r.Ok() { r.FetchToken() } @@ -1183,13 +1183,13 @@ func (r *Lexer) Interface() interface{} { return nil } switch r.token.kind { - case tokenString: + case TokenString: return r.String() - case tokenNumber: + case TokenNumber: return r.Float64() - case tokenBool: + case TokenBool: return r.Bool() - case tokenNull: + case TokenNull: r.Null() return nil } @@ -1242,3 +1242,16 @@ func (r *Lexer) WantColon() { r.wantSep = ':' r.firstElement = false } + +// CurrentToken returns current token kind if there were no errors and TokenUndef otherwise +func (r *Lexer) CurrentToken() TokenKind { + if r.token.kind == TokenUndef && r.Ok() { + r.FetchToken() + } + + if !r.Ok() { + return TokenUndef + } + + return r.token.kind +} diff --git a/jlexer/lexer_test.go b/jlexer/lexer_test.go index 0cee611d..b2bbad76 100644 --- a/jlexer/lexer_test.go +++ b/jlexer/lexer_test.go @@ -373,3 +373,18 @@ func TestFetchStringUnterminatedString(t *testing.T) { } } } + +func TestCurrentToken(t *testing.T) { + data := []byte(`{"foo"`) + tokens := []TokenKind{TokenDelim, TokenString, TokenUndef} + l := Lexer{Data: data} + + for _, want := range tokens { + got := l.CurrentToken() + if got != want { + t.Errorf("CurrentToken() = %v; want %v (err %s)", got, want, l.Error()) + } + + l.Skip() + } +} From 34d2f3ad1a4848da7c1422bce917f14038fcee8b Mon Sep 17 00:00:00 2001 From: Niall Newman <21335031+niallnsec@users.noreply.github.com> Date: Sun, 28 Jan 2024 19:13:32 +0000 Subject: [PATCH 04/16] Only add tags to run command if set --- bootstrap/bootstrap.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 5755beea..aff4fb49 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -187,7 +187,10 @@ func (g *Generator) Run() error { buildFlags := buildFlagsRegexp.FindAllString(g.GenBuildFlags, -1) execArgs = append(execArgs, buildFlags...) } - execArgs = append(execArgs, "-tags", g.BuildTags, filepath.Base(path)) + if len(g.BuildTags) > 0 { + execArgs = append(execArgs, "-tags", g.BuildTags) + } + execArgs = append(execArgs, filepath.Base(path)) cmd := exec.Command("go", execArgs...) cmd.Stdout = f From 32296272946138958f1507179d341a69a878e902 Mon Sep 17 00:00:00 2001 From: Artem Utkin Date: Fri, 19 Apr 2024 11:01:15 +0300 Subject: [PATCH 05/16] Fix null key in map --- gen/encoder.go | 2 +- jwriter/writer.go | 12 ++++++++++++ tests/data.go | 9 ++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/gen/encoder.go b/gen/encoder.go index ed6d6ad5..22db5e98 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -242,7 +242,7 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT // NOTE: extra check for TextMarshaler. It overrides default methods. if reflect.PtrTo(key).Implements(reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()) { - fmt.Fprintln(g.out, ws+" "+fmt.Sprintf("out.RawText(("+tmpVar+"Name).MarshalText()"+")")) + fmt.Fprintln(g.out, ws+" "+fmt.Sprintf("out.RawBytesString(("+tmpVar+"Name).MarshalText()"+")")) } else if keyEnc != "" { fmt.Fprintln(g.out, ws+" "+fmt.Sprintf(keyEnc, tmpVar+"Name")) } else { diff --git a/jwriter/writer.go b/jwriter/writer.go index 2c5b2010..34b0ade4 100644 --- a/jwriter/writer.go +++ b/jwriter/writer.go @@ -67,6 +67,18 @@ func (w *Writer) RawString(s string) { w.Buffer.AppendString(s) } +// RawBytesString appends string from bytes to the buffer. +func (w *Writer) RawBytesString(data []byte, err error) { + switch { + case w.Error != nil: + return + case err != nil: + w.Error = err + default: + w.String(string(data)) + } +} + // Raw appends raw binary data to the buffer or sets the error if it is given. Useful for // calling with results of MarshalJSON-like functions. func (w *Writer) Raw(data []byte, err error) { diff --git a/tests/data.go b/tests/data.go index f64a48e2..2018c882 100644 --- a/tests/data.go +++ b/tests/data.go @@ -545,21 +545,24 @@ type Maps struct { InterfaceMap map[string]interface{} NilMap map[string]string - CustomMap map[Str]Str + CustomMap map[Str]Str + CustomMapWithEmptyKey map[Str]Str } var mapsValue = Maps{ Map: map[string]string{"A": "b"}, // only one item since map iteration is randomized InterfaceMap: map[string]interface{}{"G": float64(1)}, - CustomMap: map[Str]Str{"c": "d"}, + CustomMap: map[Str]Str{"c": "d"}, + CustomMapWithEmptyKey: map[Str]Str{"": "d"}, } var mapsString = `{` + `"Map":{"A":"b"},` + `"InterfaceMap":{"G":1},` + `"NilMap":null,` + - `"CustomMap":{"c":"d"}` + + `"CustomMap":{"c":"d"},` + + `"CustomMapWithEmptyKey":{"":"d"}` + `}` type NamedSlice []Str From 46715aab064260bc137b2ee412802f7dee5a6be0 Mon Sep 17 00:00:00 2001 From: IakovLeven <88191828+IakovLeven@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:22:44 +0300 Subject: [PATCH 06/16] Fix Unmarshaler interface description --- helpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers.go b/helpers.go index 78dacb1b..efe34bf2 100644 --- a/helpers.go +++ b/helpers.go @@ -17,7 +17,7 @@ type Marshaler interface { MarshalEasyJSON(w *jwriter.Writer) } -// Marshaler is an easyjson-compatible unmarshaler interface. +// Unmarshaler is an easyjson-compatible unmarshaler interface. type Unmarshaler interface { UnmarshalEasyJSON(w *jlexer.Lexer) } From 8ef38d7618660af569e7e673c9c052d31faa94bc Mon Sep 17 00:00:00 2001 From: Vasily Romanov Date: Sat, 14 Dec 2024 21:16:31 +0300 Subject: [PATCH 07/16] upd test version --- .github/workflows/easyjson.yml | 8 ++++---- go.mod | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/easyjson.yml b/.github/workflows/easyjson.yml index 5f0b6e10..14a193ea 100644 --- a/.github/workflows/easyjson.yml +++ b/.github/workflows/easyjson.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - go: [ 1.17, 1.16, 1.15 ] + go: [ 1.23 ] steps: - uses: actions/checkout@v2 @@ -33,14 +33,14 @@ jobs: - name: Build and Run tests run: make - test-non-amd64: + test-arm64: runs-on: ubuntu-latest name: Test on ${{ matrix.distro }} ${{ matrix.arch }} strategy: matrix: include: - - arch: ppc64le - distro: ubuntu20.04 + - arch: arm64 + distro: ubuntu24.04 steps: - uses: actions/checkout@v2 - uses: uraimo/run-on-arch-action@master diff --git a/go.mod b/go.mod index f4945d34..1eac8d04 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/mailru/easyjson -go 1.12 +go 1.23 require github.com/josharian/intern v1.0.0 From 0e683d55cb7f288d05824e1f01c7a54041db29d1 Mon Sep 17 00:00:00 2001 From: Vasily Romanov Date: Sat, 14 Dec 2024 21:18:23 +0300 Subject: [PATCH 08/16] only default tests --- .github/workflows/easyjson.yml | 56 +++++++++++++++++----------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/easyjson.yml b/.github/workflows/easyjson.yml index 14a193ea..9d9bc792 100644 --- a/.github/workflows/easyjson.yml +++ b/.github/workflows/easyjson.yml @@ -33,31 +33,31 @@ jobs: - name: Build and Run tests run: make - test-arm64: - runs-on: ubuntu-latest - name: Test on ${{ matrix.distro }} ${{ matrix.arch }} - strategy: - matrix: - include: - - arch: arm64 - distro: ubuntu24.04 - steps: - - uses: actions/checkout@v2 - - uses: uraimo/run-on-arch-action@master - with: - arch: ${{ matrix.arch }} - distro: ${{ matrix.distro }} - install: | - apt-get update - apt install -y curl wget make gcc - latestGo=$(curl "https://golang.org/VERSION?m=text") - wget --quiet "https://dl.google.com/go/${latestGo}.linux-${{ matrix.arch }}.tar.gz" - rm -f $(which go) - rm -rf /usr/local/go - tar -C /usr/local -xzf "${latestGo}.linux-${{ matrix.arch }}.tar.gz" - run: | - export PATH=/usr/local/go/bin:$PATH - export PATH=~/go/bin:$PATH - printf "Go Version: $(go version)\n" - go install golang.org/x/lint/golint@latest - make \ No newline at end of file + # test-arm64: + # runs-on: ubuntu-latest + # name: Test on ${{ matrix.distro }} ${{ matrix.arch }} + # strategy: + # matrix: + # include: + # - arch: arm64 + # distro: ubuntu24.04 + # steps: + # - uses: actions/checkout@v2 + # - uses: uraimo/run-on-arch-action@master + # with: + # arch: ${{ matrix.arch }} + # distro: ${{ matrix.distro }} + # install: | + # apt-get update + # apt install -y curl wget make gcc + # latestGo=$(curl "https://golang.org/VERSION?m=text") + # wget --quiet "https://dl.google.com/go/${latestGo}.linux-${{ matrix.arch }}.tar.gz" + # rm -f $(which go) + # rm -rf /usr/local/go + # tar -C /usr/local -xzf "${latestGo}.linux-${{ matrix.arch }}.tar.gz" + # run: | + # export PATH=/usr/local/go/bin:$PATH + # export PATH=~/go/bin:$PATH + # printf "Go Version: $(go version)\n" + # go install golang.org/x/lint/golint@latest + # make \ No newline at end of file From 907f46a3eb29e4817ffd234da48438fb0d3cb5b6 Mon Sep 17 00:00:00 2001 From: Vasily Romanov Date: Sat, 14 Dec 2024 21:21:20 +0300 Subject: [PATCH 09/16] up go version to 1.20 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1eac8d04..5384ce61 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/mailru/easyjson -go 1.23 +go 1.20 require github.com/josharian/intern v1.0.0 From 529b1f6f7ec18fbae380d7ce72d38061e5201498 Mon Sep 17 00:00:00 2001 From: Neal Patel Date: Wed, 18 Dec 2024 21:43:31 -0800 Subject: [PATCH 10/16] Fix unmarshal null to existing value --- gen/decoder.go | 6 ------ tests/basic_test.go | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index 0a0faa26..df1a4bd3 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -320,7 +320,6 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field return fmt.Errorf("don't know how to decode %v", t) } return nil - } func (g *Generator) interfaceIsEasyjsonUnmarshaller(t reflect.Type) bool { @@ -514,11 +513,6 @@ func (g *Generator) genStructDecoder(t reflect.Type) error { fmt.Fprintln(g.out, " for !in.IsDelim('}') {") fmt.Fprintf(g.out, " key := in.UnsafeFieldName(%v)\n", g.skipMemberNameUnescaping) fmt.Fprintln(g.out, " in.WantColon()") - fmt.Fprintln(g.out, " if in.IsNull() {") - fmt.Fprintln(g.out, " in.Skip()") - fmt.Fprintln(g.out, " in.WantComma()") - fmt.Fprintln(g.out, " continue") - fmt.Fprintln(g.out, " }") fmt.Fprintln(g.out, " switch key {") for _, f := range fs { diff --git a/tests/basic_test.go b/tests/basic_test.go index e18e515c..190f6d01 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -205,7 +205,6 @@ func TestEncodingFlags(t *testing.T) { t.Errorf("[%v] easyjson.Marshal(%+v) = %v; want %v", i, test.In, v, test.Want) } } - } func TestNestedEasyJsonMarshal(t *testing.T) { @@ -329,3 +328,17 @@ func TestNil(t *testing.T) { t.Errorf("Wanted null, got %q", s) } } + +func TestUnmarshalNull(t *testing.T) { + p := primitiveTypesValue + + data := `{"Ptr":null}` + + if err := easyjson.Unmarshal([]byte(data), &p); err != nil { + t.Errorf("easyjson.Unmarshal() error: %v", err) + } + + if p.Ptr != nil { + t.Errorf("Wanted nil, got %q", *p.Ptr) + } +} From baefa5cf6e059ea01dc5e9924035bffa6cebaa64 Mon Sep 17 00:00:00 2001 From: Neal Patel Date: Thu, 9 Jan 2025 11:27:28 -0800 Subject: [PATCH 11/16] Fix decoding null values on non-pointer fields --- gen/decoder.go | 40 ++++++++++++++++++++++++++++++++-------- tests/basic_test.go | 11 +++++++++-- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/gen/decoder.go b/gen/decoder.go index df1a4bd3..2af6aaf3 100644 --- a/gen/decoder.go +++ b/gen/decoder.go @@ -63,22 +63,34 @@ func (g *Generator) genTypeDecoder(t reflect.Type, out string, tags fieldTags, i unmarshalerIface := reflect.TypeOf((*easyjson.Unmarshaler)(nil)).Elem() if reflect.PtrTo(t).Implements(unmarshalerIface) { - fmt.Fprintln(g.out, ws+"("+out+").UnmarshalEasyJSON(in)") + fmt.Fprintln(g.out, ws+"if in.IsNull() {") + fmt.Fprintln(g.out, ws+" in.Skip()") + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" ("+out+").UnmarshalEasyJSON(in)") + fmt.Fprintln(g.out, ws+"}") return nil } unmarshalerIface = reflect.TypeOf((*json.Unmarshaler)(nil)).Elem() if reflect.PtrTo(t).Implements(unmarshalerIface) { - fmt.Fprintln(g.out, ws+"if data := in.Raw(); in.Ok() {") - fmt.Fprintln(g.out, ws+" in.AddError( ("+out+").UnmarshalJSON(data) )") + fmt.Fprintln(g.out, ws+"if in.IsNull() {") + fmt.Fprintln(g.out, ws+" in.Skip()") + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" if data := in.Raw(); in.Ok() {") + fmt.Fprintln(g.out, ws+" in.AddError( ("+out+").UnmarshalJSON(data) )") + fmt.Fprintln(g.out, ws+" }") fmt.Fprintln(g.out, ws+"}") return nil } unmarshalerIface = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() if reflect.PtrTo(t).Implements(unmarshalerIface) { - fmt.Fprintln(g.out, ws+"if data := in.UnsafeBytes(); in.Ok() {") - fmt.Fprintln(g.out, ws+" in.AddError( ("+out+").UnmarshalText(data) )") + fmt.Fprintln(g.out, ws+"if in.IsNull() {") + fmt.Fprintln(g.out, ws+" in.Skip()") + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" if data := in.UnsafeBytes(); in.Ok() {") + fmt.Fprintln(g.out, ws+" in.AddError( ("+out+").UnmarshalText(data) )") + fmt.Fprintln(g.out, ws+" }") fmt.Fprintln(g.out, ws+"}") return nil } @@ -110,13 +122,21 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field ws := strings.Repeat(" ", indent) // Check whether type is primitive, needs to be done after interface check. if dec := customDecoders[t.String()]; dec != "" { - fmt.Fprintln(g.out, ws+out+" = "+dec) + fmt.Fprintln(g.out, ws+"if in.IsNull() {") + fmt.Fprintln(g.out, ws+" in.Skip()") + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" "+out+" = "+dec) + fmt.Fprintln(g.out, ws+"}") return nil } else if dec := primitiveStringDecoders[t.Kind()]; dec != "" && tags.asString { if tags.intern && t.Kind() == reflect.String { dec = "in.StringIntern()" } - fmt.Fprintln(g.out, ws+out+" = "+g.getType(t)+"("+dec+")") + fmt.Fprintln(g.out, ws+"if in.IsNull() {") + fmt.Fprintln(g.out, ws+" in.Skip()") + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" "+out+" = "+g.getType(t)+"("+dec+")") + fmt.Fprintln(g.out, ws+"}") return nil } else if dec := primitiveDecoders[t.Kind()]; dec != "" { if tags.intern && t.Kind() == reflect.String { @@ -125,7 +145,11 @@ func (g *Generator) genTypeDecoderNoCheck(t reflect.Type, out string, tags field if tags.noCopy && t.Kind() == reflect.String { dec = "in.UnsafeString()" } - fmt.Fprintln(g.out, ws+out+" = "+g.getType(t)+"("+dec+")") + fmt.Fprintln(g.out, ws+"if in.IsNull() {") + fmt.Fprintln(g.out, ws+" in.Skip()") + fmt.Fprintln(g.out, ws+"} else {") + fmt.Fprintln(g.out, ws+" "+out+" = "+g.getType(t)+"("+dec+")") + fmt.Fprintln(g.out, ws+"}") return nil } diff --git a/tests/basic_test.go b/tests/basic_test.go index 190f6d01..5bf93f1a 100644 --- a/tests/basic_test.go +++ b/tests/basic_test.go @@ -330,14 +330,21 @@ func TestNil(t *testing.T) { } func TestUnmarshalNull(t *testing.T) { - p := primitiveTypesValue + p := PrimitiveTypes{ + String: str, + Ptr: &str, + } - data := `{"Ptr":null}` + data := `{"String":null,"Ptr":null}` if err := easyjson.Unmarshal([]byte(data), &p); err != nil { t.Errorf("easyjson.Unmarshal() error: %v", err) } + if p.String != str { + t.Errorf("Wanted %q, got %q", str, p.String) + } + if p.Ptr != nil { t.Errorf("Wanted nil, got %q", *p.Ptr) } From 31b2360af5d02800c548b502403408794cd38b9a Mon Sep 17 00:00:00 2001 From: dmitrybarsukov Date: Sun, 25 May 2025 22:54:08 +0200 Subject: [PATCH 12/16] Return error if trying to marshal +Inf, -Inf or NaN --- jwriter/writer.go | 27 +++++++++++++++++++++++++++ tests/errors.go | 6 ++++++ tests/errors_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/jwriter/writer.go b/jwriter/writer.go index 34b0ade4..6f1403a6 100644 --- a/jwriter/writer.go +++ b/jwriter/writer.go @@ -2,7 +2,9 @@ package jwriter import ( + "fmt" "io" + "math" "strconv" "unicode/utf8" @@ -248,11 +250,19 @@ func (w *Writer) Int64Str(n int64) { } func (w *Writer) Float32(n float32) { + if w.checkIsUnsupportedFloat(float64(n)) { + return + } + w.Buffer.EnsureSpace(20) w.Buffer.Buf = strconv.AppendFloat(w.Buffer.Buf, float64(n), 'g', -1, 32) } func (w *Writer) Float32Str(n float32) { + if w.checkIsUnsupportedFloat(float64(n)) { + return + } + w.Buffer.EnsureSpace(20) w.Buffer.Buf = append(w.Buffer.Buf, '"') w.Buffer.Buf = strconv.AppendFloat(w.Buffer.Buf, float64(n), 'g', -1, 32) @@ -260,11 +270,19 @@ func (w *Writer) Float32Str(n float32) { } func (w *Writer) Float64(n float64) { + if w.checkIsUnsupportedFloat(n) { + return + } + w.Buffer.EnsureSpace(20) w.Buffer.Buf = strconv.AppendFloat(w.Buffer.Buf, n, 'g', -1, 64) } func (w *Writer) Float64Str(n float64) { + if w.checkIsUnsupportedFloat(n) { + return + } + w.Buffer.EnsureSpace(20) w.Buffer.Buf = append(w.Buffer.Buf, '"') w.Buffer.Buf = strconv.AppendFloat(w.Buffer.Buf, float64(n), 'g', -1, 64) @@ -415,3 +433,12 @@ func (w *Writer) base64(in []byte) { w.Buffer.Buf = append(w.Buffer.Buf, byte(padChar), byte(padChar)) } } + +func (w *Writer) checkIsUnsupportedFloat(val float64) bool { + isUnsupported := math.IsNaN(val) || math.IsInf(val, 0) + if isUnsupported && w.Error == nil { + w.Error = fmt.Errorf("json: unsupported value: %v", val) + } + + return isUnsupported +} diff --git a/tests/errors.go b/tests/errors.go index 14360fcc..2a545d1a 100644 --- a/tests/errors.go +++ b/tests/errors.go @@ -24,3 +24,9 @@ type ErrorNestedStruct struct { //easyjson:json type ErrorIntMap map[uint32]string + +//easyjson:json +type ErrorFloatTypes struct { + Float64 float64 `json:"float64"` + Float32 float32 `json:"float32"` +} diff --git a/tests/errors_test.go b/tests/errors_test.go index 40fa3354..756ef111 100644 --- a/tests/errors_test.go +++ b/tests/errors_test.go @@ -1,6 +1,7 @@ package tests import ( + "math" "testing" "github.com/mailru/easyjson/jlexer" @@ -283,3 +284,40 @@ func TestMultipleErrorsIntMap(t *testing.T) { } } } + +func TestUnsupportedFloatValues(t *testing.T) { + for i, test := range []struct { + Value ErrorFloatTypes + ExpectedErr string + }{ + { + Value: ErrorFloatTypes{Float64: math.NaN()}, + ExpectedErr: "json: unsupported value: NaN", + }, + { + Value: ErrorFloatTypes{Float64: math.Inf(1)}, + ExpectedErr: "json: unsupported value: +Inf", + }, + { + Value: ErrorFloatTypes{Float64: math.Inf(-1)}, + ExpectedErr: "json: unsupported value: -Inf", + }, + { + Value: ErrorFloatTypes{Float32: float32(math.NaN())}, + ExpectedErr: "json: unsupported value: NaN", + }, + { + Value: ErrorFloatTypes{Float32: float32(math.Inf(1))}, + ExpectedErr: "json: unsupported value: +Inf", + }, + { + Value: ErrorFloatTypes{Float32: float32(math.Inf(-1))}, + ExpectedErr: "json: unsupported value: -Inf", + }, + } { + _, err := test.Value.MarshalJSON() + if err == nil || err.Error() != test.ExpectedErr { + t.Errorf("[%d] TestUnsupportedFloatValues(): error: want %s, got %v", i, test.ExpectedErr, err) + } + } +} From 9b7ae67e7e2001606b2ca31eaa5e8e265a219766 Mon Sep 17 00:00:00 2001 From: Andrey Berezin Date: Fri, 6 Jun 2025 10:19:01 +0300 Subject: [PATCH 13/16] fix null after MarshalText work --- Makefile | 3 ++- gen/encoder.go | 2 +- jwriter/writer.go | 16 +-------------- tests/text_marshaler.go | 15 ++++++++++++++ tests/text_marshaler_test.go | 40 ++++++++++++++++++++++++++++++++++++ 5 files changed, 59 insertions(+), 17 deletions(-) create mode 100644 tests/text_marshaler.go create mode 100644 tests/text_marshaler_test.go diff --git a/Makefile b/Makefile index cc5ebbad..043bae88 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,8 @@ generate: build ./tests/intern.go \ ./tests/nocopy.go \ ./tests/escaping.go \ - ./tests/nested_marshaler.go + ./tests/nested_marshaler.go \ + ./tests/text_marshaler.go bin/easyjson -snake_case ./tests/snake.go bin/easyjson -omit_empty ./tests/omitempty.go bin/easyjson -build_tags=use_easyjson -disable_members_unescape ./benchmark/data.go diff --git a/gen/encoder.go b/gen/encoder.go index 22db5e98..ed6d6ad5 100644 --- a/gen/encoder.go +++ b/gen/encoder.go @@ -242,7 +242,7 @@ func (g *Generator) genTypeEncoderNoCheck(t reflect.Type, in string, tags fieldT // NOTE: extra check for TextMarshaler. It overrides default methods. if reflect.PtrTo(key).Implements(reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()) { - fmt.Fprintln(g.out, ws+" "+fmt.Sprintf("out.RawBytesString(("+tmpVar+"Name).MarshalText()"+")")) + fmt.Fprintln(g.out, ws+" "+fmt.Sprintf("out.RawText(("+tmpVar+"Name).MarshalText()"+")")) } else if keyEnc != "" { fmt.Fprintln(g.out, ws+" "+fmt.Sprintf(keyEnc, tmpVar+"Name")) } else { diff --git a/jwriter/writer.go b/jwriter/writer.go index 34b0ade4..894c1981 100644 --- a/jwriter/writer.go +++ b/jwriter/writer.go @@ -67,18 +67,6 @@ func (w *Writer) RawString(s string) { w.Buffer.AppendString(s) } -// RawBytesString appends string from bytes to the buffer. -func (w *Writer) RawBytesString(data []byte, err error) { - switch { - case w.Error != nil: - return - case err != nil: - w.Error = err - default: - w.String(string(data)) - } -} - // Raw appends raw binary data to the buffer or sets the error if it is given. Useful for // calling with results of MarshalJSON-like functions. func (w *Writer) Raw(data []byte, err error) { @@ -102,10 +90,8 @@ func (w *Writer) RawText(data []byte, err error) { return case err != nil: w.Error = err - case len(data) > 0: - w.String(string(data)) default: - w.RawString("null") + w.String(string(data)) } } diff --git a/tests/text_marshaler.go b/tests/text_marshaler.go new file mode 100644 index 00000000..8d3b51e8 --- /dev/null +++ b/tests/text_marshaler.go @@ -0,0 +1,15 @@ +package tests + +import ( + "strings" +) + +//easyjson:json +type StructWrappedTextMarshaler struct { + Value namedWithTextMarshaler +} +type namedWithTextMarshaler string + +func (n namedWithTextMarshaler) MarshalText() ([]byte, error) { + return []byte(strings.ToUpper(string(n))), nil +} diff --git a/tests/text_marshaler_test.go b/tests/text_marshaler_test.go new file mode 100644 index 00000000..33a63eeb --- /dev/null +++ b/tests/text_marshaler_test.go @@ -0,0 +1,40 @@ +package tests + +import ( + "testing" + + "github.com/mailru/easyjson" +) + +func TestStructWithTextMarshalerMarshal(t *testing.T) { + tests := []struct { + name string + input StructWrappedTextMarshaler + expected string + }{ + { + name: "Filled struct", + input: StructWrappedTextMarshaler{ + Value: namedWithTextMarshaler("test"), + }, + expected: `{"Value":"TEST"}`, + }, + { + name: "Empty struct", + input: StructWrappedTextMarshaler{}, + expected: `{"Value":""}`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + marshaled, err := easyjson.Marshal(test.input) + if err != nil { + t.Errorf("easyjson.Marshal failed: %v", err) + } + if string(marshaled) != test.expected { + t.Errorf("easyjson.Marshal output incorrect: got %s, want %s", string(marshaled), test.expected) + } + }) + } +} From ffa0b23ade4e69fc97bcd872e5b120d7418a5b85 Mon Sep 17 00:00:00 2001 From: Vladislav Bulagakov Date: Mon, 15 Sep 2025 11:14:50 +0300 Subject: [PATCH 14/16] feat: Add version and commit information to easyjson generator Closes #415 --- Makefile | 6 +++++- easyjson/main.go | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cc5ebbad..44d34348 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +VERSION := $(shell git describe --tags --always --dirty) +COMMIT := $(shell git rev-parse --short HEAD) + all: test clean: @@ -6,7 +9,8 @@ clean: rm -rf benchmark/*_easyjson.go build: - go build -o ./bin/easyjson ./easyjson + go build -ldflags="-s -w -X 'main.Version=$(VERSION)' -X 'main.Commit=$(COMMIT)'" -o ./bin/easyjson ./easyjson + generate: build bin/easyjson -stubs \ diff --git a/easyjson/main.go b/easyjson/main.go index d337c846..55be0ac9 100644 --- a/easyjson/main.go +++ b/easyjson/main.go @@ -16,6 +16,11 @@ import ( "github.com/mailru/easyjson/parser" ) +var ( + Version = "dev" // + Commit = "none" +) + var buildTags = flag.String("build_tags", "", "build tags to add to generated file") var genBuildFlags = flag.String("gen_build_flags", "", "build flags when running the generator while bootstrapping") var snakeCase = flag.Bool("snake_case", false, "use snake_case names instead of CamelCase by default") @@ -31,6 +36,7 @@ var specifiedName = flag.String("output_filename", "", "specify the filename of var processPkg = flag.Bool("pkg", false, "process the whole package instead of just the given file") var disallowUnknownFields = flag.Bool("disallow_unknown_fields", false, "return error if any unknown field in json appeared") var skipMemberNameUnescaping = flag.Bool("disable_members_unescape", false, "don't perform unescaping of member names to improve performance") +var showVersion = flag.Bool("version", false, "print version and exit") func generate(fname string) (err error) { fInfo, err := os.Stat(fname) @@ -98,6 +104,11 @@ func main() { files := flag.Args() + if *showVersion { + fmt.Printf("easyjson generator\nversion: %s\ncommit: %s\n", Version, Commit) + os.Exit(0) + } + gofile := os.Getenv("GOFILE") if *processPkg { gofile = filepath.Dir(gofile) From a50dd0f30cf540167e75b18e9ce00d0280072fa5 Mon Sep 17 00:00:00 2001 From: jeeves-form3 Date: Tue, 5 Apr 2022 11:32:43 +0000 Subject: [PATCH 15/16] [skip ci] Adding CODEOWNERS file --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..a9e4945e --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# automatically generated by terraform - please do not edit here +* @form3tech-oss/Contributors-codeowners From d041e25ff46e330a9fad85cd50725a82726e1d73 Mon Sep 17 00:00:00 2001 From: jeeves-form3 Date: Tue, 21 Jun 2022 08:54:04 +0000 Subject: [PATCH 16/16] [skip ci] Updating CODEOWNERS file --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a9e4945e..efc73c3e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # automatically generated by terraform - please do not edit here -* @form3tech-oss/Contributors-codeowners +* @form3tech-oss/contributors-codeowners