Skip to content

Commit 388ee25

Browse files
AnnaShalevaAnnaShaleva
authored
Merge pull request #3725 from nspcc-dev/neofs-sdk-update
*: update `neofs-sdk-go`
2 parents f82088b + 9e3f75e commit 388ee25

File tree

7 files changed

+150
-140
lines changed

7 files changed

+150
-140
lines changed

‎cli/util/upload_bin.go

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,17 @@ package util
33
import (
44
"context"
55
"fmt"
6-
"slices"
76
"strconv"
87
"sync"
98
"time"
109

1110
"github.com/nspcc-dev/neo-go/cli/cmdargs"
1211
"github.com/nspcc-dev/neo-go/cli/options"
1312
"github.com/nspcc-dev/neo-go/pkg/core/block"
13+
"github.com/nspcc-dev/neo-go/pkg/crypto/keys"
1414
"github.com/nspcc-dev/neo-go/pkg/io"
1515
"github.com/nspcc-dev/neo-go/pkg/rpcclient"
1616
"github.com/nspcc-dev/neo-go/pkg/services/helpers/neofs"
17-
"github.com/nspcc-dev/neo-go/pkg/util"
18-
"github.com/nspcc-dev/neo-go/pkg/wallet"
1917
"github.com/nspcc-dev/neofs-sdk-go/client"
2018
"github.com/nspcc-dev/neofs-sdk-go/container"
2119
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
@@ -107,12 +105,12 @@ func uploadBin(ctx *cli.Context) error {
107105
return cli.Exit(fmt.Sprintf("failed to get current block height from RPC: %v", err), 1)
108106
}
109107
fmt.Fprintln(ctx.App.Writer, "Chain block height:", currentBlockHeight)
110-
i, buf, err := searchIndexFile(ctx, pWrapper, containerID, acc, signer, indexFileSize, attr, indexAttrKey, maxParallelSearches, maxRetries, debug)
108+
i, buf, err := searchIndexFile(ctx, pWrapper, containerID, acc.PrivateKey(), signer, indexFileSize, attr, indexAttrKey, maxParallelSearches, maxRetries, debug)
111109
if err != nil {
112110
return cli.Exit(fmt.Errorf("failed to find objects: %w", err), 1)
113111
}
114112

115-
err = uploadBlocksAndIndexFiles(ctx, pWrapper, rpc, signer, containerID, acc, attr, indexAttrKey, buf, i, indexFileSize, uint(currentBlockHeight), numWorkers, maxRetries, debug)
113+
err = uploadBlocksAndIndexFiles(ctx, pWrapper, rpc, signer, containerID, user.NewFromScriptHash(acc.ScriptHash()), attr, indexAttrKey, buf, i, indexFileSize, uint(currentBlockHeight), numWorkers, maxRetries, debug)
116114
if err != nil {
117115
return cli.Exit(fmt.Errorf("failed to upload objects: %w", err), 1)
118116
}
@@ -140,7 +138,7 @@ func retry(action func() error, maxRetries uint, debug bool) error {
140138
}
141139

142140
// uploadBlocksAndIndexFiles uploads the blocks and index files to the container using the pool.
143-
func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.Client, signer user.Signer, containerID cid.ID, acc *wallet.Account, attr, indexAttributeKey string, buf []byte, currentIndexFileID, indexFileSize, currentBlockHeight uint, numWorkers, maxRetries uint, debug bool) error {
141+
func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.Client, signer user.Signer, containerID cid.ID, ownerID user.ID, attr, indexAttributeKey string, buf []byte, currentIndexFileID, indexFileSize, currentBlockHeight uint, numWorkers, maxRetries uint, debug bool) error {
144142
if currentIndexFileID*indexFileSize >= currentBlockHeight {
145143
fmt.Fprintf(ctx.App.Writer, "No new blocks to upload. Need to upload starting from %d, current height %d\n", currentIndexFileID*indexFileSize, currentBlockHeight)
146144
return nil
@@ -152,15 +150,14 @@ func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.C
152150
errCh = make(chan error)
153151
doneCh = make(chan struct{})
154152
wg sync.WaitGroup
155-
emptyOID = make([]byte, neofs.OIDSize)
156153
)
157154
fmt.Fprintf(ctx.App.Writer, "Processing batch from %d to %d\n", indexFileStart, indexFileEnd-1)
158155
wg.Add(int(numWorkers))
159156
for i := range numWorkers {
160157
go func(i uint) {
161158
defer wg.Done()
162159
for blockIndex := indexFileStart + i; blockIndex < indexFileEnd; blockIndex += numWorkers {
163-
if slices.Compare(buf[blockIndex%indexFileSize*neofs.OIDSize:blockIndex%indexFileSize*neofs.OIDSize+neofs.OIDSize], emptyOID) != 0 {
160+
if !oid.ID(buf[blockIndex%indexFileSize*oid.Size : blockIndex%indexFileSize*oid.Size+oid.Size]).IsZero() {
164161
if debug {
165162
fmt.Fprintf(ctx.App.Writer, "Block %d is already uploaded\n", blockIndex)
166163
}
@@ -207,7 +204,7 @@ func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.C
207204
)
208205
errRetr := retry(func() error {
209206
var errUpload error
210-
resOid, errUpload = uploadObj(ctx.Context, p, signer, acc.PrivateKey().GetScriptHash(), containerID, objBytes, attrs)
207+
resOid, errUpload = uploadObj(ctx.Context, p, signer, ownerID, containerID, objBytes, attrs)
211208
if errUpload != nil {
212209
return errUpload
213210
}
@@ -223,7 +220,7 @@ func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.C
223220
}
224221
return
225222
}
226-
resOid.Encode(buf[blockIndex%indexFileSize*neofs.OIDSize:])
223+
copy(buf[blockIndex%indexFileSize*oid.Size:], resOid[:])
227224
}
228225
}(i)
229226
}
@@ -241,9 +238,9 @@ func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.C
241238
fmt.Fprintf(ctx.App.Writer, "Successfully processed batch of blocks: from %d to %d\n", indexFileStart, indexFileEnd-1)
242239

243240
// Additional check for empty OIDs in the buffer.
244-
for k := uint(0); k < (indexFileEnd-indexFileStart)*neofs.OIDSize; k += neofs.OIDSize {
245-
if slices.Compare(buf[k:k+neofs.OIDSize], emptyOID) == 0 {
246-
return fmt.Errorf("empty OID found in index file %d at position %d (block index %d)", indexFileStart/indexFileSize, k/neofs.OIDSize, indexFileStart/indexFileSize*indexFileSize+k/neofs.OIDSize)
241+
for k := uint(0); k < (indexFileEnd-indexFileStart)*oid.Size; k += oid.Size {
242+
if oid.ID(buf[k : k+oid.Size]).IsZero() {
243+
return fmt.Errorf("empty OID found in index file %d at position %d (block index %d)", indexFileStart/indexFileSize, k/oid.Size, indexFileStart/indexFileSize*indexFileSize+k/oid.Size)
247244
}
248245
}
249246
if indexFileEnd-indexFileStart == indexFileSize {
@@ -254,7 +251,7 @@ func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.C
254251
}
255252
err := retry(func() error {
256253
var errUpload error
257-
_, errUpload = uploadObj(ctx.Context, p, signer, acc.PrivateKey().GetScriptHash(), containerID, buf, attrs)
254+
_, errUpload = uploadObj(ctx.Context, p, signer, ownerID, containerID, buf, attrs)
258255
return errUpload
259256
}, maxRetries, debug)
260257
if err != nil {
@@ -268,10 +265,10 @@ func uploadBlocksAndIndexFiles(ctx *cli.Context, p poolWrapper, rpc *rpcclient.C
268265
}
269266

270267
// searchIndexFile returns the ID and buffer for the next index file to be uploaded.
271-
func searchIndexFile(ctx *cli.Context, p poolWrapper, containerID cid.ID, account *wallet.Account, signer user.Signer, indexFileSize uint, blockAttributeKey, attributeKey string, maxParallelSearches, maxRetries uint, debug bool) (uint, []byte, error) {
268+
func searchIndexFile(ctx *cli.Context, p poolWrapper, containerID cid.ID, privKeys *keys.PrivateKey, signer user.Signer, indexFileSize uint, blockAttributeKey, attributeKey string, maxParallelSearches, maxRetries uint, debug bool) (uint, []byte, error) {
272269
var (
273270
// buf is used to store OIDs of the uploaded blocks.
274-
buf = make([]byte, indexFileSize*neofs.OIDSize)
271+
buf = make([]byte, indexFileSize*oid.Size)
275272
doneCh = make(chan struct{})
276273
errCh = make(chan error)
277274

@@ -283,7 +280,7 @@ func searchIndexFile(ctx *cli.Context, p poolWrapper, containerID cid.ID, accoun
283280
// Search for existing index files.
284281
filters.AddFilter("IndexSize", fmt.Sprintf("%d", indexFileSize), object.MatchStringEqual)
285282
for i := 0; ; i++ {
286-
indexIDs := searchObjects(ctx.Context, p, containerID, account, attributeKey, uint(i), uint(i+1), 1, maxRetries, debug, errCh, filters)
283+
indexIDs := searchObjects(ctx.Context, p, containerID, privKeys, attributeKey, uint(i), uint(i+1), 1, maxRetries, debug, errCh, filters)
287284
count := 0
288285
for range indexIDs {
289286
count++
@@ -338,14 +335,14 @@ func searchIndexFile(ctx *cli.Context, p poolWrapper, containerID cid.ID, accoun
338335
}
339336
pos := uint(blockIndex) % indexFileSize
340337
if _, ok := processedIndices.LoadOrStore(pos, blockIndex); !ok {
341-
id.Encode(buf[pos*neofs.OIDSize:])
338+
copy(buf[pos*oid.Size:], id[:])
342339
}
343340
}
344341
}()
345342
}
346343

347344
// Search for blocks within the index file range.
348-
objIDs := searchObjects(ctx.Context, p, containerID, account, blockAttributeKey, existingIndexCount*indexFileSize, (existingIndexCount+1)*indexFileSize, maxParallelSearches, maxRetries, debug, errCh)
345+
objIDs := searchObjects(ctx.Context, p, containerID, privKeys, blockAttributeKey, existingIndexCount*indexFileSize, (existingIndexCount+1)*indexFileSize, maxParallelSearches, maxRetries, debug, errCh)
349346
for id := range objIDs {
350347
oidCh <- id
351348
}
@@ -364,7 +361,7 @@ func searchIndexFile(ctx *cli.Context, p poolWrapper, containerID cid.ID, accoun
364361
// searchObjects searches in parallel for objects with attribute GE startIndex and LT
365362
// endIndex. It returns a buffered channel of resulting object IDs and closes it once
366363
// OID search is finished. Errors are sent to errCh in a non-blocking way.
367-
func searchObjects(ctx context.Context, p poolWrapper, containerID cid.ID, account *wallet.Account, blockAttributeKey string, startIndex, endIndex, maxParallelSearches, maxRetries uint, debug bool, errCh chan error, additionalFilters ...object.SearchFilters) chan oid.ID {
364+
func searchObjects(ctx context.Context, p poolWrapper, containerID cid.ID, privKeys *keys.PrivateKey, blockAttributeKey string, startIndex, endIndex, maxParallelSearches, maxRetries uint, debug bool, errCh chan error, additionalFilters ...object.SearchFilters) chan oid.ID {
368365
var res = make(chan oid.ID, 2*neofs.DefaultSearchBatchSize)
369366
go func() {
370367
var wg sync.WaitGroup
@@ -402,7 +399,7 @@ func searchObjects(ctx context.Context, p poolWrapper, containerID cid.ID, accou
402399
var objIDs []oid.ID
403400
err := retry(func() error {
404401
var errBlockSearch error
405-
objIDs, errBlockSearch = neofs.ObjectSearch(ctx, p, account.PrivateKey(), containerID.String(), prm)
402+
objIDs, errBlockSearch = neofs.ObjectSearch(ctx, p, privKeys, containerID.String(), prm)
406403
return errBlockSearch
407404
}, maxRetries, debug)
408405
if err != nil {
@@ -426,18 +423,16 @@ func searchObjects(ctx context.Context, p poolWrapper, containerID cid.ID, accou
426423
}
427424

428425
// uploadObj uploads object to the container using provided settings.
429-
func uploadObj(ctx context.Context, p poolWrapper, signer user.Signer, owner util.Uint160, containerID cid.ID, objData []byte, attrs []object.Attribute) (oid.ID, error) {
426+
func uploadObj(ctx context.Context, p poolWrapper, signer user.Signer, ownerID user.ID, containerID cid.ID, objData []byte, attrs []object.Attribute) (oid.ID, error) {
430427
var (
431-
ownerID user.ID
432428
hdr object.Object
433429
prmObjectPutInit client.PrmObjectPutInit
434430
resOID = oid.ID{}
435431
)
436432

437-
ownerID.SetScriptHash(owner)
438433
hdr.SetPayload(objData)
439434
hdr.SetContainerID(containerID)
440-
hdr.SetOwnerID(&ownerID)
435+
hdr.SetOwner(ownerID)
441436
hdr.SetAttributes(attrs...)
442437

443438
writer, err := p.ObjectPutInit(ctx, hdr, signer, prmObjectPutInit)
@@ -455,7 +450,7 @@ func uploadObj(ctx context.Context, p poolWrapper, signer user.Signer, owner uti
455450
}
456451
res := writer.GetResult()
457452
resOID = res.StoredObjectID()
458-
if resOID.Equals(oid.ID{}) {
453+
if resOID.IsZero() {
459454
return resOID, fmt.Errorf("object ID is empty")
460455
}
461456
return resOID, nil

‎go.mod

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ require (
1616
github.com/nspcc-dev/dbft v0.3.1
1717
github.com/nspcc-dev/go-ordered-json v0.0.0-20240830112754-291b000d1f3b
1818
github.com/nspcc-dev/neo-go/pkg/interop v0.0.0-20241212130705-ea0a6114d2d6
19-
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.12
19+
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.12.0.20241205083504-335d9fe90f24
2020
github.com/nspcc-dev/rfc6979 v0.2.3
2121
github.com/pierrec/lz4 v2.6.1+incompatible
2222
github.com/prometheus/client_golang v1.20.2
@@ -30,12 +30,12 @@ require (
3030
golang.org/x/term v0.27.0
3131
golang.org/x/text v0.21.0
3232
golang.org/x/tools v0.24.0
33-
google.golang.org/grpc v1.62.0
33+
google.golang.org/grpc v1.65.0
3434
gopkg.in/yaml.v3 v3.0.1
3535
)
3636

3737
require (
38-
github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20221202181307-76fa05c21b12 // indirect
38+
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
3939
github.com/beorn7/perks v1.0.1 // indirect
4040
github.com/bits-and-blooms/bitset v1.14.2 // indirect
4141
github.com/blang/semver/v4 v4.0.0 // indirect
@@ -45,7 +45,6 @@ require (
4545
github.com/davecgh/go-spew v1.1.1 // indirect
4646
github.com/frankban/quicktest v1.14.5 // indirect
4747
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
48-
github.com/golang/protobuf v1.5.3 // indirect
4948
github.com/golang/snappy v0.0.1 // indirect
5049
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
5150
github.com/ingonyama-zk/icicle v1.1.0 // indirect
@@ -55,9 +54,9 @@ require (
5554
github.com/mattn/go-isatty v0.0.20 // indirect
5655
github.com/mmcloughlin/addchain v0.4.0 // indirect
5756
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
58-
github.com/nspcc-dev/hrw/v2 v2.0.1 // indirect
59-
github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240305074711-35bc78d84dc4 // indirect
60-
github.com/nspcc-dev/tzhash v1.7.2 // indirect
57+
github.com/nspcc-dev/hrw/v2 v2.0.2 // indirect
58+
github.com/nspcc-dev/neofs-api-go/v2 v2.14.1-0.20240827150555-5ce597aa14ea // indirect
59+
github.com/nspcc-dev/tzhash v1.8.2 // indirect
6160
github.com/pmezard/go-difflib v1.0.0 // indirect
6261
github.com/prometheus/client_model v0.6.1 // indirect
6362
github.com/prometheus/common v0.55.0 // indirect
@@ -73,7 +72,7 @@ require (
7372
golang.org/x/net v0.28.0 // indirect
7473
golang.org/x/sync v0.10.0 // indirect
7574
golang.org/x/sys v0.28.0 // indirect
76-
google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect
75+
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
7776
google.golang.org/protobuf v1.34.2 // indirect
7877
rsc.io/tmplfunc v0.0.3 // indirect
7978
)

0 commit comments

Comments
 (0)