Merge pull request #178 from SevereCloud/dev-v2.13.0

v2.13.0

Author: SevereCloud

README.md

@@ -17,9 +17,10 @@ Version API 5.131.
 
 - [API](https://pkg.go.dev/github.com/SevereCloud/vksdk/v2/api)
   - 500+ methods
-  - Ability to change the request handler
   - Ability to modify HTTP client
   - Request Limiter
+  - Support [zstd](https://pkg.go.dev/github.com/SevereCloud/vksdk/v2/api#VK.EnableZstd)
+and [MessagePack](https://pkg.go.dev/github.com/SevereCloud/vksdk/v2/api#VK.EnableMessagePack)
   - Token pool
   - [OAuth](https://pkg.go.dev/github.com/SevereCloud/vksdk/v2/api/oauth)
 - [Callback API](https://pkg.go.dev/github.com/SevereCloud/vksdk/v2/callback)

api/README.md

@@ -80,6 +80,54 @@ if errors.As(err, &e) {
 
 Для Execute существует отдельная ошибка `ExecuteErrors`
 
+### Поддержка MessagePack и zstd
+
+> Результат перехода с gzip (JSON) на zstd (msgpack):
+>
+> - в 7 раз быстрее сжатие (–1 мкс);
+> - на 10% меньше размер данных (8 Кбайт вместо 9 Кбайт);
+> - продуктовый эффект не статзначимый :(
+>
+> [Как мы отказались от JPEG, JSON, TCP и ускорили ВКонтакте в два раза](https://habr.com/ru/company/vk/blog/594633/)
+
+VK API способно возвращать ответ в виде [MessagePack](https://msgpack.org/).
+Это эффективный формат двоичной сериализации, похожий на JSON, только быстрее
+и меньше по размеру.
+
+ВНИМАНИЕ, C MessagePack НЕКОТОРЫЕ МЕТОДЫ МОГУТ ВОЗВРАЩАТЬ
+СЛОМАННУЮ КОДИРОВКУ.
+
+Для сжатия, вместо классического gzip, можно использовать
+[zstd](https://github.com/facebook/zstd). Сейчас vksdk поддерживает zstd без
+словаря. Если кто знает как получать словарь,
+[отпишитесь сюда](https://github.com/SevereCloud/vksdk/issues/180).
+
+```go
+vk := api.NewVK(os.Getenv("USER_TOKEN"))
+
+method := "store.getStickersKeywords"
+params := api.Params{
+	"aliases":       true,
+	"all_products":  true,
+	"need_stickers": true,
+}
+
+r, err := vk.Request(method, params) // Content-Length: 44758
+if err != nil {
+	log.Fatal(err)
+}
+log.Println("json:", len(r)) // json: 814231
+
+vk.EnableMessagePack() // Включаем поддержку MessagePack
+vk.EnableZstd() // Включаем поддержку zstd
+
+r, err = vk.Request(method, params) // Content-Length: 35755
+if err != nil {
+	log.Fatal(err)
+}
+log.Println("msgpack:", len(r)) // msgpack: 650775
+```
+
 ### Запрос любого метода
 
 Пример запроса [users.get](https://vk.com/dev/users.get)

api/ads.go

@@ -1,9 +1,11 @@
 package api // import "github.com/SevereCloud/vksdk/v2/api"
 
 import (
+	"bytes"
 	"encoding/json"
 
 	"github.com/SevereCloud/vksdk/v2/object"
+	"github.com/vmihailenco/msgpack/v5"
 )
 
 // AdsAddOfficeUsersItem struct.
@@ -21,6 +23,23 @@ func (r *AdsAddOfficeUsersItem) UnmarshalJSON(data []byte) (err error) {
 	return
 }
 
+// DecodeMsgpack func.
+func (r *AdsAddOfficeUsersItem) DecodeMsgpack(dec *msgpack.Decoder) error {
+	data, err := dec.DecodeRaw()
+	if err != nil {
+		return err
+	}
+
+	if msgpack.Unmarshal(data, &r.OK) != nil {
+		d := msgpack.NewDecoder(bytes.NewReader(data))
+		d.SetCustomStructTag("json")
+
+		return d.Decode(&r.Error)
+	}
+
+	return nil
+}
+
 // AdsAddOfficeUsersResponse struct.
 type AdsAddOfficeUsersResponse []AdsAddOfficeUsersItem
 

api/ads_test.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/SevereCloud/vksdk/v2/api"
 	"github.com/stretchr/testify/assert"
+	"github.com/vmihailenco/msgpack/v5"
+	"github.com/vmihailenco/msgpack/v5/msgpcode"
 )
 
 func TestAdsResponse_UnmarshalJSON(t *testing.T) {
@@ -32,6 +34,46 @@ func TestAdsResponse_UnmarshalJSON(t *testing.T) {
 	)
 }
 
+func TestAdsResponse_DecodeMsgpack(t *testing.T) {
+	t.Parallel()
+
+	f := func(data []byte, expected api.AdsAddOfficeUsersItem, wantErr string) {
+		var r api.AdsAddOfficeUsersItem
+
+		err := msgpack.Unmarshal(data, &r)
+		if err != nil || wantErr != "" {
+			assert.EqualError(t, err, wantErr)
+		}
+
+		assert.Equal(t, expected, r)
+	}
+
+	f([]byte{msgpcode.False}, api.AdsAddOfficeUsersItem{OK: false}, "")
+	f([]byte{msgpcode.True}, api.AdsAddOfficeUsersItem{OK: true}, "")
+	f(
+		[]byte{
+			0x82, 0xAA, 0x65, 0x72, 0x72, 0x6F, 0x72, 0x5F, 0x63, 0x6F, 0x64,
+			0x65, 0x64, 0xAA, 0x65, 0x72, 0x72, 0x6F, 0x72, 0x5F, 0x64, 0x65,
+			0x73, 0x63, 0xD9, 0x48, 0x4F, 0x6E, 0x65, 0x20, 0x6F, 0x66, 0x20,
+			0x74, 0x68, 0x65, 0x20, 0x70, 0x61, 0x72, 0x61, 0x6D, 0x65, 0x74,
+			0x65, 0x72, 0x73, 0x20, 0x73, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69,
+			0x65, 0x64, 0x20, 0x77, 0x61, 0x73, 0x20, 0x6D, 0x69, 0x73, 0x73,
+			0x69, 0x6E, 0x67, 0x20, 0x6F, 0x72, 0x20, 0x69, 0x6E, 0x76, 0x61,
+			0x6C, 0x69, 0x64, 0x3A, 0x20, 0x64, 0x61, 0x74, 0x61, 0x5B, 0x31,
+			0x5D, 0x5B, 0x75, 0x73, 0x65, 0x72, 0x5F, 0x69, 0x64, 0x5D,
+		},
+		api.AdsAddOfficeUsersItem{
+			OK: false,
+			Error: api.AdsError{
+				Code: 100,
+				Desc: "One of the parameters specified was missing or invalid: data[1][user_id]",
+			},
+		},
+		"",
+	)
+	f(nil, api.AdsAddOfficeUsersItem{}, "EOF")
+}
+
 func TestVK_AdsGetAccounts(t *testing.T) {
 	t.Parallel()
 

api/api.go

@@ -7,9 +7,11 @@ package api // import "github.com/SevereCloud/vksdk/v2/api"
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"mime"
 	"net/http"
 	"net/url"
@@ -21,6 +23,8 @@ import (
 	"github.com/SevereCloud/vksdk/v2"
 	"github.com/SevereCloud/vksdk/v2/internal"
 	"github.com/SevereCloud/vksdk/v2/object"
+	"github.com/klauspost/compress/zstd"
+	"github.com/vmihailenco/msgpack/v5"
 )
 
 // Api constants.
@@ -91,16 +95,19 @@ type VK struct {
 	UserAgent    string
 	Handler      func(method string, params ...Params) (Response, error)
 
+	msgpack bool
+	zstd    bool
+
 	mux      sync.Mutex
 	lastTime time.Time
 	rps      int
 }
 
 // Response struct.
 type Response struct {
-	Response      json.RawMessage `json:"response"`
-	Error         Error           `json:"error"`
-	ExecuteErrors ExecuteErrors   `json:"execute_errors"`
+	Response      object.RawMessage `json:"response"`
+	Error         Error             `json:"error"`
+	ExecuteErrors ExecuteErrors     `json:"execute_errors"`
 }
 
 // NewVK returns a new VK.
@@ -243,24 +250,52 @@ func (vk *VK) DefaultHandler(method string, sliceParams ...Params) (Response, er
 			return response, err
 		}
 
+		acceptEncoding := "gzip"
+		if vk.zstd {
+			acceptEncoding = "zstd"
+		}
+
 		req.Header.Set("User-Agent", vk.UserAgent)
 		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
+		req.Header.Set("Accept-Encoding", acceptEncoding)
+
+		var reader io.Reader
+
 		resp, err := vk.Client.Do(req)
 		if err != nil {
 			return response, err
 		}
 
-		mediatype, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type"))
-		if mediatype != "application/json" {
-			_ = resp.Body.Close()
-			return response, &InvalidContentType{mediatype}
+		switch resp.Header.Get("Content-Encoding") {
+		case "zstd":
+			reader, _ = zstd.NewReader(resp.Body)
+		case "gzip":
+			reader, _ = gzip.NewReader(resp.Body)
+		default:
+			reader = resp.Body
 		}
 
-		err = json.NewDecoder(resp.Body).Decode(&response)
-		if err != nil {
+		mediatype, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type"))
+		switch mediatype {
+		case "application/json":
+			err = json.NewDecoder(reader).Decode(&response)
+			if err != nil {
+				_ = resp.Body.Close()
+				return response, err
+			}
+		case "application/x-msgpack":
+			dec := msgpack.NewDecoder(reader)
+			dec.SetCustomStructTag("json")
+
+			err = dec.Decode(&response)
+			if err != nil {
+				_ = resp.Body.Close()
+				return response, err
+			}
+		default:
 			_ = resp.Body.Close()
-			return response, err
+			return response, &InvalidContentType{mediatype}
 		}
 
 		_ = resp.Body.Close()
@@ -291,6 +326,10 @@ func (vk *VK) Request(method string, sliceParams ...Params) ([]byte, error) {
 
 	sliceParams = append(sliceParams, reqParams)
 
+	if vk.msgpack {
+		method += ".msgpack"
+	}
+
 	resp, err := vk.Handler(method, sliceParams...)
 
 	return resp.Response, err
@@ -303,7 +342,32 @@ func (vk *VK) RequestUnmarshal(method string, obj interface{}, sliceParams ...Pa
 		return err
 	}
 
-	return json.Unmarshal(rawResponse, &obj)
+	if vk.msgpack {
+		dec := msgpack.NewDecoder(bytes.NewReader(rawResponse))
+		dec.SetCustomStructTag("json")
+
+		err = dec.Decode(&obj)
+	} else {
+		err = json.Unmarshal(rawResponse, &obj)
+	}
+
+	return err
+}
+
+// EnableMessagePack enable using MessagePack instead of JSON.
+//
+// THIS IS EXPERIMENTAL FUNCTION! Broken encoding returned in some methods.
+//
+// See https://msgpack.org
+func (vk *VK) EnableMessagePack() {
+	vk.msgpack = true
+}
+
+// EnableZstd enable using zstd instead of gzip.
+//
+// This not use dict.
+func (vk *VK) EnableZstd() {
+	vk.zstd = true
 }
 
 func fmtReflectValue(value reflect.Value, depth int) string {

api/api_test.go

@@ -322,3 +322,23 @@ func TestContext(t *testing.T) {
 	_, err := vkUser.UsersGet(p)
 	assert.EqualError(t, err, "Post \"https://api.vk.com/method/users.get\": context deadline exceeded")
 }
+
+func TestVK_EnableMessagePack(t *testing.T) {
+	t.Parallel()
+
+	vk := api.NewVK("")
+	vk.EnableMessagePack()
+
+	_, err := vk.UsersGet(nil)
+	assert.ErrorIs(t, err, api.ErrAuth)
+}
+
+func TestVK_EnableZstd(t *testing.T) {
+	t.Parallel()
+
+	vk := api.NewVK("")
+	vk.EnableZstd()
+
+	_, err := vk.UsersGet(nil)
+	assert.ErrorIs(t, err, api.ErrAuth)
+}