deduplicate static strings

buffer.go

@@ -18,7 +18,10 @@ type (
 
 		Vars []Variable
 
-		arr []Off
+		Static []Off
+		arr    []Off
+
+		Flags BufferFlags
 	}
 
 	// BufferReader is a thin structure on top of Buffer to logically separate reading methods.
@@ -31,6 +34,14 @@ type (
 	BufferWriter struct {
 		*Buffer
 	}
+
+	BufferFlags int
+)
+
+const (
+	BufferStatic BufferFlags = 1 << iota
+
+	BufferDefault = BufferStatic
 )
 
 var (
@@ -72,20 +83,23 @@ func MakeBuffer() Buffer {
 	return Buffer{
 		Encoder: MakeEncoder(),
 		Decoder: MakeDecoder(),
+		Flags:   BufferDefault,
 	}
 }
 
 func MakeBufferFrom(buf []byte) Buffer {
 	return Buffer{
+		B:       buf[:0],
 		Encoder: MakeEncoder(),
 		Decoder: MakeDecoder(),
-		B:       buf[:0],
+		Flags:   BufferDefault,
 	}
 }
 
 func (b *Buffer) Reset() {
 	b.B = b.B[:0]
 	b.Vars = b.Vars[:0]
+	b.Static = b.Static[:0]
 }
 
 func (b *Buffer) Reader() BufferReader { return BufferReader{b} }
@@ -154,3 +168,15 @@ func (b *Buffer) Unwrap() []byte {
 func (b *Buffer) Dump() string {
 	return NewDumper(nil).Dump(b)
 }
+
+func (f BufferFlags) Is(x BufferFlags) bool {
+	return f&x == x
+}
+
+func (f *BufferFlags) Set(x BufferFlags) {
+	*f |= x
+}
+
+func (f *BufferFlags) Unset(x BufferFlags) {
+	*f &^= x
+}

buffer_writer.go

@@ -4,6 +4,7 @@ import (
 	"io"
 
 	"nikand.dev/go/cbor"
+	"nikand.dev/go/skip"
 )
 
 func (b BufferWriter) Off() Off {
@@ -34,9 +35,15 @@ func (b BufferWriter) Raw(raw []byte) Off {
 		}
 	}
 
+	if off := b.GetStatic(raw); off != None {
+		return off
+	}
+
 	off := b.Off()
 	b.B = append(b.B, raw...)
 
+	b.AddStatic(off)
+
 	return off
 }
 
@@ -73,12 +80,28 @@ func (b BufferWriter) String(v string) Off {
 
 	off := b.Off()
 	b.B = b.Encoder.AppendString(b.B, v)
+
+	if static := b.GetStatic(b.B[off:]); static != None {
+		b.Reset(off)
+		return static
+	}
+
+	b.AddStatic(off)
+
 	return off
 }
 
 func (b BufferWriter) Bytes(v []byte) Off {
 	off := b.Off()
 	b.B = b.Encoder.AppendBytes(b.B, v)
+
+	if static := b.GetStatic(b.B[off:]); static != None {
+		b.Reset(off)
+		return static
+	}
+
+	b.AddStatic(off)
+
 	return off
 }
 
@@ -89,6 +112,14 @@ func (b BufferWriter) TagString(tag Tag, v string) Off {
 
 	off := b.Off()
 	b.B = b.Encoder.AppendTagString(b.B, tag, v)
+
+	if static := b.GetStatic(b.B[off:]); static != None {
+		b.Reset(off)
+		return static
+	}
+
+	b.AddStatic(off)
+
 	return off
 }
 
@@ -99,6 +130,14 @@ func (b BufferWriter) TagBytes(tag Tag, v []byte) Off {
 
 	off := b.Off()
 	b.B = b.Encoder.AppendTagBytes(b.B, tag, v)
+
+	if static := b.GetStatic(b.B[off:]); static != None {
+		b.Reset(off)
+		return static
+	}
+
+	b.AddStatic(off)
+
 	return off
 }
 
@@ -186,3 +225,35 @@ func (w bufferIOWriter) Write(p []byte) (int, error) {
 
 	return len(p), nil
 }
+
+func (b BufferWriter) GetStatic(raw []byte) Off {
+	if !b.Flags.Is(BufferStatic) {
+		return None
+	}
+
+	if len(raw) <= 2 {
+		return None
+	}
+
+	for _, off := range b.Static {
+		if len(raw) <= 4 && b.Off()-off >= 0x10000-offReserve {
+			continue
+		}
+
+		if !skip.Prefix(b.B[off:], raw) {
+			continue
+		}
+
+		return off
+	}
+
+	return None
+}
+
+func (b BufferWriter) AddStatic(off Off) {
+	if !b.Flags.Is(BufferStatic) {
+		return
+	}
+
+	b.Static = append(b.Static, off)
+}

go.mod

@@ -4,6 +4,6 @@ go 1.22
 
 require (
 	nikand.dev/go/cbor v0.0.0-20240927214417-98af9ca7872f
-	nikand.dev/go/json v0.8.1-0.20241010232239-b0a0bc26a37c
-	nikand.dev/go/skip v0.0.0-20241011141638-a03a00e577bf
+	nikand.dev/go/json v0.8.1-0.20241012140807-b6f5d5d75038
+	nikand.dev/go/skip v0.0.0-20241012140729-20cd863c178e
 )

go.sum

@@ -8,7 +8,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 nikand.dev/go/cbor v0.0.0-20240927214417-98af9ca7872f h1:ni1YmHBWF8YI22c1TW7uTBG9nfyezWSC4n+9zZT7RHQ=
 nikand.dev/go/cbor v0.0.0-20240927214417-98af9ca7872f/go.mod h1:KU6mV3VAVKdPXXlzE9BtpUaVtV+Z/N/jFEeC5jXMkDg=
-nikand.dev/go/json v0.8.1-0.20241010232239-b0a0bc26a37c h1:tkn3oSY9oKIwVwxC/sRAs8kA0j1tGqxKUser9f9yYns=
-nikand.dev/go/json v0.8.1-0.20241010232239-b0a0bc26a37c/go.mod h1:/8k1AZBdWiVk+7HRNv3Dw045aX8mL4nNERp5N6MH42c=
-nikand.dev/go/skip v0.0.0-20241011141638-a03a00e577bf h1:2rRsIxaqV1tS6unZ3Pzxi47HDNWDfn9jVncZkSp1wQU=
-nikand.dev/go/skip v0.0.0-20241011141638-a03a00e577bf/go.mod h1:sxT9iqxrhIQGtoTQWbhSy7yhd5mmtenoHpG0buF6p1w=
+nikand.dev/go/json v0.8.1-0.20241012140807-b6f5d5d75038 h1:YiO2bePPZltav2PeQsxs26hQg/u92ZzhT5MFDVP2QJs=
+nikand.dev/go/json v0.8.1-0.20241012140807-b6f5d5d75038/go.mod h1:VVY1cDuJoQ9uTAU7O3MHrLqY70zV/8I91GPKGEcxGjI=
+nikand.dev/go/skip v0.0.0-20241012140729-20cd863c178e h1:IJLCWfEVwuFm09YuKdC52oCE+7/CvSU7A3VmWnTaLqg=
+nikand.dev/go/skip v0.0.0-20241012140729-20cd863c178e/go.mod h1:sxT9iqxrhIQGtoTQWbhSy7yhd5mmtenoHpG0buF6p1w=

jq.go

@@ -162,17 +162,6 @@ func (f Empty) ApplyTo(b *Buffer, off Off, next bool) (Off, bool, error) {
 	return None, false, nil
 }
 
-func NewHalt(err error) Halt { return Halt{Err: err} }
-
-func (f Halt) ApplyTo(b *Buffer, off Off, next bool) (Off, bool, error) {
-	err := f.Err
-	if err == nil {
-		err = ErrHalt
-	}
-
-	return off, false, err
-}
-
 func NewLiteral(x any) Literal {
 	var e Encoder
 
@@ -198,6 +187,17 @@ func (f Literal) ApplyTo(b *Buffer, off Off, next bool) (Off, bool, error) {
 	return b.Writer().Raw(f.Raw), false, nil
 }
 
+func NewHalt(err error) Halt { return Halt{Err: err} }
+
+func (f Halt) ApplyTo(b *Buffer, off Off, next bool) (Off, bool, error) {
+	err := f.Err
+	if err == nil {
+		err = ErrHalt
+	}
+
+	return off, false, err
+}
+
 func (f Off) String() string { return fmt.Sprintf("%x", int(f)) }
 func (f Off) Format(s fmt.State, v rune) {
 	if v == 'v' && f < 0 {

jqjson/decoder.go

@@ -168,6 +168,13 @@ func (d *Decoder) decode(b *jq.Buffer, r []byte, st int, key bool) (off jq.Off,
 			return off, st, err
 		}
 
+		if static := bw.GetStatic(b.B[off:]); static != jq.None {
+			bw.Reset(off)
+			off = static
+		} else {
+			bw.AddStatic(off)
+		}
+
 		return off, i, nil
 	}
 

jqurl/decoder.go

@@ -64,17 +64,6 @@ func (d *Decoder) Decode(b *jq.Buffer, r []byte, st int) (off jq.Off, i int, err
 		}
 
 		key := bw.TagBytes(cbor.String, d.b[br:])
-
-		for j := 0; j < len(d.arr); j += 2 {
-			if !b.Equal(d.arr[j], key) {
-				continue
-			}
-
-			bw.Reset(key)
-			key = d.arr[j]
-			break
-		}
-
 		val := jq.Null
 		br = len(d.b)
 
@@ -100,45 +89,27 @@ func (d *Decoder) Decode(b *jq.Buffer, r []byte, st int) (off jq.Off, i int, err
 		return bw.Map(d.arr), i, nil
 	}
 
-	dups := func(arr []jq.Off, j int) []jq.Off {
-		for k := j; k < len(arr); k += 2 {
-			if arr[k] == arr[j] {
-				arr = append(arr, arr[k+1])
-			}
-		}
-
-		return arr
-	}
+	for j := 0; j < len(d.arr); j += 2 {
+		l := len(d.arr)
+		d.arr = append(d.arr, d.arr[j+1])
 
-	move := func(arr []jq.Off, j int) []jq.Off {
 		w := j + 2
-		r := j + 2
 
-		for ; r < len(arr); r += 2 {
-			if arr[r] == arr[j] {
+		for r := w; r < l; r += 2 {
+			if d.arr[r] == d.arr[j] {
+				d.arr = append(d.arr, d.arr[r+1])
 				continue
 			}
 
-			arr[w], arr[w+1] = arr[r], arr[r+1]
+			d.arr[w], d.arr[w+1] = d.arr[r], d.arr[r+1]
 			w += 2
 		}
 
-		return arr[:w]
-	}
-
-	for j := 0; j < len(d.arr)-2; j += 2 {
-		l := len(d.arr)
-
-		d.arr = dups(d.arr, j)
-
-		if len(d.arr) == l+1 {
-			d.arr = d.arr[:l]
-			continue
+		if w != l {
+			d.arr[j+1] = bw.Array(d.arr[l:])
 		}
 
-		d.arr[j+1] = bw.Array(d.arr[l:])
-
-		d.arr = move(d.arr[:l], j)
+		d.arr = d.arr[:w]
 	}
 
 	return bw.Map(d.arr), i, nil

jqurl/url_test.go

@@ -43,6 +43,7 @@ func TestDecoderEncoder(tb *testing.T) {
 		"a=b&a=c&a=d",
 	} {
 		b.Reset()
+		tb.Logf("test case: (%s)", x)
 
 		off, i, err := d.Decode(b, []byte(x), 0)
 		assertNoError(tb, err)