Skip to content

Commit 336ff77

Browse files
istaeistae
authored
perf: sampler optimization (#4882)
1 parent 78bea5c commit 336ff77

File tree

3 files changed

+99
-68
lines changed

3 files changed

+99
-68
lines changed

‎pkg/soc/soc.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,22 @@ func (s *SOC) Sign(signer crypto.Signer) (swarm.Chunk, error) {
130130
return s.Chunk()
131131
}
132132

133+
// UnwrapCAC extracts the CAC inside the SOC.
134+
func UnwrapCAC(sch swarm.Chunk) (swarm.Chunk, error) {
135+
chunkData := sch.Data()
136+
if len(chunkData) < swarm.SocMinChunkSize {
137+
return nil, errWrongChunkSize
138+
}
139+
140+
cursor := swarm.HashSize + swarm.SocSignatureSize
141+
ch, err := cac.NewWithDataSpan(chunkData[cursor:])
142+
if err != nil {
143+
return nil, err
144+
}
145+
146+
return ch, nil
147+
}
148+
133149
// FromChunk recreates a SOC representation from swarm.Chunk data.
134150
func FromChunk(sch swarm.Chunk) (*SOC, error) {
135151
chunkData := sch.Data()

‎pkg/soc/soc_test.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,15 @@ func TestFromChunk(t *testing.T) {
327327
if !ch.Equal(recoveredSOC.WrappedChunk()) {
328328
t.Fatalf("wrapped chunk mismatch. got %s want %s", recoveredSOC.WrappedChunk().Address(), ch.Address())
329329
}
330+
331+
unwrapped, err := soc.UnwrapCAC(sch)
332+
if err != nil {
333+
t.Fatal(err)
334+
}
335+
336+
if !ch.Equal(unwrapped) {
337+
t.Fatalf("wrapped chunk mismatch. got %s want %s", recoveredSOC.WrappedChunk().Address(), ch.Address())
338+
}
330339
}
331340

332341
func TestCreateAddress(t *testing.T) {

‎pkg/storer/sample.go

Lines changed: 74 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"fmt"
1212
"hash"
1313
"math/big"
14+
"runtime"
1415
"sort"
1516
"sync"
1617
"testing"
@@ -41,67 +42,6 @@ type Sample struct {
4142
Items []SampleItem
4243
}
4344

44-
// RandSample returns Sample with random values.
45-
func RandSample(t *testing.T, anchor []byte) Sample {
46-
t.Helper()
47-
48-
chunks := make([]swarm.Chunk, SampleSize)
49-
for i := 0; i < SampleSize; i++ {
50-
ch := chunk.GenerateTestRandomChunk()
51-
if i%3 == 0 {
52-
ch = chunk.GenerateTestRandomSoChunk(t, ch)
53-
}
54-
chunks[i] = ch
55-
}
56-
57-
sample, err := MakeSampleUsingChunks(chunks, anchor)
58-
if err != nil {
59-
t.Fatal(err)
60-
}
61-
62-
return sample
63-
}
64-
65-
// MakeSampleUsingChunks returns Sample constructed using supplied chunks.
66-
func MakeSampleUsingChunks(chunks []swarm.Chunk, anchor []byte) (Sample, error) {
67-
prefixHasherFactory := func() hash.Hash {
68-
return swarm.NewPrefixHasher(anchor)
69-
}
70-
items := make([]SampleItem, len(chunks))
71-
for i, ch := range chunks {
72-
tr, err := transformedAddress(bmt.NewHasher(prefixHasherFactory), ch, getChunkType(ch))
73-
if err != nil {
74-
return Sample{}, err
75-
}
76-
77-
items[i] = SampleItem{
78-
TransformedAddress: tr,
79-
ChunkAddress: ch.Address(),
80-
ChunkData: ch.Data(),
81-
Stamp: newStamp(ch.Stamp()),
82-
}
83-
}
84-
85-
sort.Slice(items, func(i, j int) bool {
86-
return items[i].TransformedAddress.Compare(items[j].TransformedAddress) == -1
87-
})
88-
89-
return Sample{Items: items}, nil
90-
}
91-
92-
func newStamp(s swarm.Stamp) *postage.Stamp {
93-
return postage.NewStamp(s.BatchID(), s.Index(), s.Timestamp(), s.Sig())
94-
}
95-
96-
func getChunkType(chunk swarm.Chunk) swarm.ChunkType {
97-
if cac.Valid(chunk) {
98-
return swarm.ChunkTypeContentAddressed
99-
} else if soc.Valid(chunk) {
100-
return swarm.ChunkTypeSingleOwner
101-
}
102-
return swarm.ChunkTypeUnspecified
103-
}
104-
10545
// ReserveSample generates the sample of reserve storage of a node required for the
10646
// storage incentives agent to participate in the lottery round. In order to generate
10747
// this sample we need to iterate through all the chunks in the node's reserve and
@@ -125,8 +65,9 @@ func (db *DB) ReserveSample(
12565
consensusTime uint64,
12666
minBatchBalance *big.Int,
12767
) (Sample, error) {
68+
12869
g, ctx := errgroup.WithContext(ctx)
129-
chunkC := make(chan *reserve.ChunkBinItem, 64)
70+
13071
allStats := &SampleStats{}
13172
statsLock := sync.Mutex{}
13273
addStats := func(stats SampleStats) {
@@ -144,6 +85,8 @@ func (db *DB) ReserveSample(
14485

14586
allStats.BatchesBelowValueDuration = time.Since(t)
14687

88+
chunkC := make(chan *reserve.ChunkBinItem)
89+
14790
// Phase 1: Iterate chunk addresses
14891
g.Go(func() error {
14992
start := time.Now()
@@ -170,13 +113,14 @@ func (db *DB) ReserveSample(
170113
})
171114

172115
// Phase 2: Get the chunk data and calculate transformed hash
173-
sampleItemChan := make(chan SampleItem, 64)
116+
sampleItemChan := make(chan SampleItem)
174117

175118
prefixHasherFactory := func() hash.Hash {
176119
return swarm.NewPrefixHasher(anchor)
177120
}
178121

179-
const workers = 6
122+
workers := max(4, runtime.NumCPU())
123+
db.logger.Debug("reserve sampler workers", "count", workers)
180124

181125
for i := 0; i < workers; i++ {
182126
g.Go(func() error {
@@ -241,6 +185,7 @@ func (db *DB) ReserveSample(
241185
}()
242186

243187
sampleItems := make([]SampleItem, 0, SampleSize)
188+
244189
// insert function will insert the new item in its correct place. If the sample
245190
// size goes beyond what we need we omit the last item.
246191
insert := func(item SampleItem) {
@@ -376,20 +321,20 @@ func transformedAddressCAC(hasher *bmt.Hasher, chunk swarm.Chunk) (swarm.Address
376321
return swarm.NewAddress(taddr), nil
377322
}
378323

379-
func transformedAddressSOC(hasher *bmt.Hasher, chunk swarm.Chunk) (swarm.Address, error) {
324+
func transformedAddressSOC(hasher *bmt.Hasher, socChunk swarm.Chunk) (swarm.Address, error) {
380325
// Calculate transformed address from wrapped chunk
381-
sChunk, err := soc.FromChunk(chunk)
326+
cacChunk, err := soc.UnwrapCAC(socChunk)
382327
if err != nil {
383328
return swarm.ZeroAddress, err
384329
}
385-
taddrCac, err := transformedAddressCAC(hasher, sChunk.WrappedChunk())
330+
taddrCac, err := transformedAddressCAC(hasher, cacChunk)
386331
if err != nil {
387332
return swarm.ZeroAddress, err
388333
}
389334

390335
// Hash address and transformed address to make transformed address for this SOC
391336
sHasher := swarm.NewHasher()
392-
if _, err := sHasher.Write(chunk.Address().Bytes()); err != nil {
337+
if _, err := sHasher.Write(socChunk.Address().Bytes()); err != nil {
393338
return swarm.ZeroAddress, err
394339
}
395340
if _, err := sHasher.Write(taddrCac.Bytes()); err != nil {
@@ -432,3 +377,64 @@ func (s *SampleStats) add(other SampleStats) {
432377
s.ChunkLoadFailed += other.ChunkLoadFailed
433378
s.StampLoadFailed += other.StampLoadFailed
434379
}
380+
381+
// RandSample returns Sample with random values.
382+
func RandSample(t *testing.T, anchor []byte) Sample {
383+
t.Helper()
384+
385+
chunks := make([]swarm.Chunk, SampleSize)
386+
for i := 0; i < SampleSize; i++ {
387+
ch := chunk.GenerateTestRandomChunk()
388+
if i%3 == 0 {
389+
ch = chunk.GenerateTestRandomSoChunk(t, ch)
390+
}
391+
chunks[i] = ch
392+
}
393+
394+
sample, err := MakeSampleUsingChunks(chunks, anchor)
395+
if err != nil {
396+
t.Fatal(err)
397+
}
398+
399+
return sample
400+
}
401+
402+
// MakeSampleUsingChunks returns Sample constructed using supplied chunks.
403+
func MakeSampleUsingChunks(chunks []swarm.Chunk, anchor []byte) (Sample, error) {
404+
prefixHasherFactory := func() hash.Hash {
405+
return swarm.NewPrefixHasher(anchor)
406+
}
407+
items := make([]SampleItem, len(chunks))
408+
for i, ch := range chunks {
409+
tr, err := transformedAddress(bmt.NewHasher(prefixHasherFactory), ch, getChunkType(ch))
410+
if err != nil {
411+
return Sample{}, err
412+
}
413+
414+
items[i] = SampleItem{
415+
TransformedAddress: tr,
416+
ChunkAddress: ch.Address(),
417+
ChunkData: ch.Data(),
418+
Stamp: newStamp(ch.Stamp()),
419+
}
420+
}
421+
422+
sort.Slice(items, func(i, j int) bool {
423+
return items[i].TransformedAddress.Compare(items[j].TransformedAddress) == -1
424+
})
425+
426+
return Sample{Items: items}, nil
427+
}
428+
429+
func newStamp(s swarm.Stamp) *postage.Stamp {
430+
return postage.NewStamp(s.BatchID(), s.Index(), s.Timestamp(), s.Sig())
431+
}
432+
433+
func getChunkType(chunk swarm.Chunk) swarm.ChunkType {
434+
if cac.Valid(chunk) {
435+
return swarm.ChunkTypeContentAddressed
436+
} else if soc.Valid(chunk) {
437+
return swarm.ChunkTypeSingleOwner
438+
}
439+
return swarm.ChunkTypeUnspecified
440+
}

0 commit comments

Comments
 (0)