From 0b4e3cba1da01cc46a480509603fc36db424bf17 Mon Sep 17 00:00:00 2001 From: Elias Naur Date: Sat, 24 Feb 2024 21:18:52 +0000 Subject: [PATCH 1/4] Optimize ReconstructSome for leopard 8+16 --- leopard.go | 20 ++++++++++++-------- leopard8.go | 22 +++++++++++++--------- reedsolomon_test.go | 7 +++---- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/leopard.go b/leopard.go index 6b4c8018..3fcecd6f 100644 --- a/leopard.go +++ b/leopard.go @@ -334,17 +334,17 @@ func (r *leopardFF16) Split(data []byte) ([][]byte, error) { func (r *leopardFF16) ReconstructSome(shards [][]byte, required []bool) error { if len(required) == r.totalShards { - return r.reconstruct(shards, true) + return r.reconstruct(shards, true, required) } - return r.reconstruct(shards, false) + return r.reconstruct(shards, false, required) } func (r *leopardFF16) Reconstruct(shards [][]byte) error { - return r.reconstruct(shards, true) + return r.reconstruct(shards, true, nil) } func (r *leopardFF16) ReconstructData(shards [][]byte) error { - return r.reconstruct(shards, false) + return r.reconstruct(shards, false, nil) } func (r *leopardFF16) Verify(shards [][]byte) (bool, error) { @@ -375,8 +375,8 @@ func (r *leopardFF16) Verify(shards [][]byte) (bool, error) { return true, nil } -func (r *leopardFF16) reconstruct(shards [][]byte, recoverAll bool) error { - if len(shards) != r.totalShards { +func (r *leopardFF16) reconstruct(shards [][]byte, recoverAll bool, required []bool) error { + if len(shards) != r.totalShards || required != nil && len(required) < r.dataShards { return ErrTooFewShards } @@ -388,15 +388,19 @@ func (r *leopardFF16) reconstruct(shards [][]byte, recoverAll bool) error { // nothing to do. numberPresent := 0 dataPresent := 0 + missingRequired := 0 for i := 0; i < r.totalShards; i++ { if len(shards[i]) != 0 { numberPresent++ if i < r.dataShards { dataPresent++ } + } else if required != nil && required[i] { + missingRequired++ } } - if numberPresent == r.totalShards || !recoverAll && dataPresent == r.dataShards { + if numberPresent == r.totalShards || !recoverAll && dataPresent == r.dataShards || + required != nil && missingRequired == 0 { // Cool. All of the shards have data. We don't // need to do anything. return nil @@ -541,7 +545,7 @@ func (r *leopardFF16) reconstruct(shards [][]byte, recoverAll bool) error { end = r.totalShards } for i := 0; i < end; i++ { - if len(shards[i]) != 0 { + if len(shards[i]) != 0 || required != nil && !required[i] { continue } if cap(shards[i]) >= shardSize { diff --git a/leopard8.go b/leopard8.go index cd863a13..2ba1e569 100644 --- a/leopard8.go +++ b/leopard8.go @@ -370,17 +370,17 @@ func (r *leopardFF8) Split(data []byte) ([][]byte, error) { func (r *leopardFF8) ReconstructSome(shards [][]byte, required []bool) error { if len(required) == r.totalShards { - return r.reconstruct(shards, true) + return r.reconstruct(shards, true, required) } - return r.reconstruct(shards, false) + return r.reconstruct(shards, false, required) } func (r *leopardFF8) Reconstruct(shards [][]byte) error { - return r.reconstruct(shards, true) + return r.reconstruct(shards, true, nil) } func (r *leopardFF8) ReconstructData(shards [][]byte) error { - return r.reconstruct(shards, false) + return r.reconstruct(shards, false, nil) } func (r *leopardFF8) Verify(shards [][]byte) (bool, error) { @@ -411,8 +411,8 @@ func (r *leopardFF8) Verify(shards [][]byte) (bool, error) { return true, nil } -func (r *leopardFF8) reconstruct(shards [][]byte, recoverAll bool) error { - if len(shards) != r.totalShards { +func (r *leopardFF8) reconstruct(shards [][]byte, recoverAll bool, required []bool) error { + if len(shards) != r.totalShards || required != nil && len(required) < r.dataShards { return ErrTooFewShards } @@ -424,15 +424,19 @@ func (r *leopardFF8) reconstruct(shards [][]byte, recoverAll bool) error { // nothing to do. numberPresent := 0 dataPresent := 0 + missingRequired := 0 for i := 0; i < r.totalShards; i++ { if len(shards[i]) != 0 { numberPresent++ if i < r.dataShards { dataPresent++ } + } else if required != nil && required[i] { + missingRequired++ } } - if numberPresent == r.totalShards || !recoverAll && dataPresent == r.dataShards { + if numberPresent == r.totalShards || !recoverAll && dataPresent == r.dataShards || + required != nil && missingRequired == 0 { // Cool. All of the shards have data. We don't // need to do anything. return nil @@ -566,7 +570,7 @@ func (r *leopardFF8) reconstruct(shards [][]byte, recoverAll bool) error { // Add output for i, sh := range shards { - if !recoverAll && i >= r.dataShards { + if !recoverAll && i >= r.dataShards || required != nil && !required[i] { continue } if len(sh) == 0 { @@ -657,7 +661,7 @@ func (r *leopardFF8) reconstruct(shards [][]byte, recoverAll bool) error { } // Restore for i := 0; i < end; i++ { - if len(sh[i]) != 0 { + if len(sh[i]) != 0 || required != nil && !required[i] { continue } diff --git a/reedsolomon_test.go b/reedsolomon_test.go index a7f7ab25..e1dd40d3 100644 --- a/reedsolomon_test.go +++ b/reedsolomon_test.go @@ -877,8 +877,7 @@ func testReconstructData(t *testing.T, o ...Option) { } if shardsCopy[2] != nil || shardsCopy[5] != nil || shardsCopy[6] != nil { - // This is expected in some cases. - t.Log("ReconstructSome reconstructed extra shards") + t.Error("ReconstructSome reconstructed extra shards") } // Reconstruct with 10 shards present. Use pre-allocated memory for one of them. @@ -1490,12 +1489,12 @@ func BenchmarkReconstruct10x4x1M(b *testing.B) { benchmarkReconstruct(b, 10, 4, 1024*1024) } -// Benchmark 5 data slices with 2 parity slices holding 1MB bytes each +// Benchmark 50 data slices with 20 parity slices holding 1MB bytes each func BenchmarkReconstruct50x20x1M(b *testing.B) { benchmarkReconstruct(b, 50, 20, 1024*1024) } -// Benchmark 5 data slices with 2 parity slices holding 1MB bytes each +// Benchmark 50 data slices with 20 parity slices holding 1MB bytes each func BenchmarkReconstructLeopard50x20x1M(b *testing.B) { benchmarkReconstruct(b, 50, 20, 1024*1024, WithLeopardGF(true), WithInversionCache(true)) } From 08a1be0b7e10f15363667f8fd14476c9b7e45589 Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Mon, 4 Mar 2024 15:13:51 +0100 Subject: [PATCH 2/4] Try optimization: only add required shards errorBits/errLocs --- leopard.go | 6 +++--- leopard8.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/leopard.go b/leopard.go index 3fcecd6f..098c1b3b 100644 --- a/leopard.go +++ b/leopard.go @@ -430,19 +430,19 @@ func (r *leopardFF16) reconstruct(shards [][]byte, recoverAll bool, required []b for i := 0; i < r.parityShards; i++ { if len(shards[i+r.dataShards]) == 0 { errLocs[i] = 1 - if LEO_ERROR_BITFIELD_OPT && recoverAll { + if LEO_ERROR_BITFIELD_OPT && (recoverAll || (required != nil && required[i])) { errorBits.set(i) } } } for i := r.parityShards; i < m; i++ { errLocs[i] = 1 - if LEO_ERROR_BITFIELD_OPT && recoverAll { + if LEO_ERROR_BITFIELD_OPT && (recoverAll || (required != nil && required[i])) { errorBits.set(i) } } for i := 0; i < r.dataShards; i++ { - if len(shards[i]) == 0 { + if len(shards[i]) == 0 && (recoverAll || (required != nil && required[i])) { errLocs[i+m] = 1 if LEO_ERROR_BITFIELD_OPT { errorBits.set(i + m) diff --git a/leopard8.go b/leopard8.go index 2ba1e569..0efb3bff 100644 --- a/leopard8.go +++ b/leopard8.go @@ -467,19 +467,19 @@ func (r *leopardFF8) reconstruct(shards [][]byte, recoverAll bool, required []bo for i := 0; i < r.parityShards; i++ { if len(shards[i+r.dataShards]) == 0 { errLocs[i] = 1 - if LEO_ERROR_BITFIELD_OPT && recoverAll { + if LEO_ERROR_BITFIELD_OPT && (recoverAll || (required != nil && required[i])) { errorBits.set(i) } } } for i := r.parityShards; i < m; i++ { errLocs[i] = 1 - if LEO_ERROR_BITFIELD_OPT && recoverAll { + if LEO_ERROR_BITFIELD_OPT && (recoverAll || (required != nil && required[i])) { errorBits.set(i) } } for i := 0; i < r.dataShards; i++ { - if len(shards[i]) == 0 { + if len(shards[i]) == 0 && (recoverAll || (required != nil && required[i])) { errLocs[i+m] = 1 if LEO_ERROR_BITFIELD_OPT { errorBits.set(i + m) From 2d78890c1322b3ef347a1428750bec55fafe7bf3 Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Mon, 4 Mar 2024 15:58:24 +0100 Subject: [PATCH 3/4] Fix tests --- leopard.go | 2 +- leopard8.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/leopard.go b/leopard.go index 098c1b3b..2ae1b5b3 100644 --- a/leopard.go +++ b/leopard.go @@ -442,7 +442,7 @@ func (r *leopardFF16) reconstruct(shards [][]byte, recoverAll bool, required []b } } for i := 0; i < r.dataShards; i++ { - if len(shards[i]) == 0 && (recoverAll || (required != nil && required[i])) { + if len(shards[i]) == 0 /*&& (recoverAll || (required != nil && required[i]))*/ { errLocs[i+m] = 1 if LEO_ERROR_BITFIELD_OPT { errorBits.set(i + m) diff --git a/leopard8.go b/leopard8.go index 0efb3bff..9f2f5e00 100644 --- a/leopard8.go +++ b/leopard8.go @@ -479,7 +479,7 @@ func (r *leopardFF8) reconstruct(shards [][]byte, recoverAll bool, required []bo } } for i := 0; i < r.dataShards; i++ { - if len(shards[i]) == 0 && (recoverAll || (required != nil && required[i])) { + if len(shards[i]) == 0 /*&& (recoverAll || (required != nil && required[i]))*/ { errLocs[i+m] = 1 if LEO_ERROR_BITFIELD_OPT { errorBits.set(i + m) From ad607095fffd2dec561b8df70b8cbd5178095849 Mon Sep 17 00:00:00 2001 From: Ismail Khoffi Date: Mon, 4 Mar 2024 20:39:43 +0100 Subject: [PATCH 4/4] should be investigated properly instead: make errorLocs dependent on `required` --- leopard.go | 4 +++- leopard8.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/leopard.go b/leopard.go index 2ae1b5b3..46606b53 100644 --- a/leopard.go +++ b/leopard.go @@ -429,7 +429,9 @@ func (r *leopardFF16) reconstruct(shards [][]byte, recoverAll bool, required []b var errLocs [order]ffe for i := 0; i < r.parityShards; i++ { if len(shards[i+r.dataShards]) == 0 { - errLocs[i] = 1 + if recoverAll || (required != nil && required[i+r.dataShards]) { + errLocs[i] = 1 + } if LEO_ERROR_BITFIELD_OPT && (recoverAll || (required != nil && required[i])) { errorBits.set(i) } diff --git a/leopard8.go b/leopard8.go index 9f2f5e00..6f90c5b1 100644 --- a/leopard8.go +++ b/leopard8.go @@ -466,7 +466,9 @@ func (r *leopardFF8) reconstruct(shards [][]byte, recoverAll bool, required []bo var errLocs [order8]ffe8 for i := 0; i < r.parityShards; i++ { if len(shards[i+r.dataShards]) == 0 { - errLocs[i] = 1 + if recoverAll || (required != nil && required[i+r.dataShards]) { + errLocs[i] = 1 + } if LEO_ERROR_BITFIELD_OPT && (recoverAll || (required != nil && required[i])) { errorBits.set(i) }