Skip to content

Commit

Permalink
feat(shwap): add Q1Q4 file (#3522)
Browse files Browse the repository at this point in the history
  • Loading branch information
@walldiss
walldiss authored Jun 27, 2024
1 parent ab4f860 commit 218bdc7
Show file tree
Hide file tree
Showing 5 changed files with 368 additions and 52 deletions.
4 changes: 4 additions & 0 deletions share/new_eds/testing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,10 @@ func BenchGetHalfAxisFromAccessor(
for _, squareHalf := range []int{0, 1} {
name := fmt.Sprintf("Size:%v/ProofType:%s/squareHalf:%s", size, axisType, strconv.Itoa(squareHalf))
b.Run(name, func(b *testing.B) {
// warm up cache
_, err := f.AxisHalf(ctx, axisType, f.Size(ctx)/2*(squareHalf))
require.NoError(b, err)

b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := f.AxisHalf(ctx, axisType, f.Size(ctx)/2*(squareHalf))
Expand Down
56 changes: 32 additions & 24 deletions store/file/ods.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ type ODSFile struct {
// Storing the square in memory allows for efficient single-read operations, avoiding the need for
// piecemeal reads by rows or columns, and facilitates quick access to data for these operations.
ods square
// disableCache is a flag that, when set to true, disables the in-memory cache of the original data
// Used for testing and benchmarking purposes, this flag allows for the evaluation of the
// performance.
disableCache bool
}

// OpenODSFile opens an existing file. File has to be closed after usage.
Expand Down Expand Up @@ -153,12 +157,12 @@ func (f *ODSFile) AxisHalf(_ context.Context, axisType rsmt2d.Axis, axisIdx int)
}

// if axis is from the second half of the square, read full ODS and compute the axis half
err := f.readODS()
ods, err := f.readODS()
if err != nil {
return eds.AxisHalf{}, err
}

half, err := f.ods.computeAxisHalf(axisType, axisIdx)
half, err := ods.computeAxisHalf(axisType, axisIdx)
if err != nil {
return eds.AxisHalf{}, fmt.Errorf("computing axis half: %w", err)
}
Expand All @@ -180,11 +184,11 @@ func (f *ODSFile) RowNamespaceData(

// Shares returns data shares extracted from the Accessor.
func (f *ODSFile) Shares(context.Context) ([]share.Share, error) {
err := f.readODS()
ods, err := f.readODS()
if err != nil {
return nil, err
}
return f.ods.shares()
return ods.shares()
}

func (f *ODSFile) readAxisHalf(axisType rsmt2d.Axis, axisIdx int) (eds.AxisHalf, error) {
Expand All @@ -197,13 +201,13 @@ func (f *ODSFile) readAxisHalf(axisType rsmt2d.Axis, axisIdx int) (eds.AxisHalf,

switch axisType {
case rsmt2d.Col:
col, err := f.readCol(axisIdx, 0)
col, err := readCol(f.fl, f.hdr, axisIdx, 0)
return eds.AxisHalf{
Shares: col,
IsParity: false,
}, err
case rsmt2d.Row:
row, err := f.readRow(axisIdx)
row, err := readRow(f.fl, f.hdr, axisIdx, 0)
return eds.AxisHalf{
Shares: row,
IsParity: false,
Expand All @@ -212,38 +216,42 @@ func (f *ODSFile) readAxisHalf(axisType rsmt2d.Axis, axisIdx int) (eds.AxisHalf,
return eds.AxisHalf{}, fmt.Errorf("unknown axis")
}

func (f *ODSFile) readODS() error {
func (f *ODSFile) readODS() (square, error) {
f.lock.Lock()
defer f.lock.Unlock()
if f.ods != nil {
return nil
return f.ods, nil
}

// reset file pointer to the beginning of the file shares data
_, err := f.fl.Seek(int64(f.hdr.Size()), io.SeekStart)
if err != nil {
return fmt.Errorf("discarding header: %w", err)
return nil, fmt.Errorf("discarding header: %w", err)
}

square, err := readSquare(f.fl, share.Size, f.size())
if err != nil {
return fmt.Errorf("reading ODS: %w", err)
return nil, fmt.Errorf("reading ODS: %w", err)
}
f.ods = square
return nil

if !f.disableCache {
f.ods = square
}
return square, nil
}

func (f *ODSFile) readRow(idx int) ([]share.Share, error) {
shrLn := int(f.hdr.shareSize)
odsLn := f.size() / 2
func readRow(fl io.ReaderAt, hdr *headerV0, rowIdx, quadrantIdx int) ([]share.Share, error) {
shrLn := int(hdr.shareSize)
odsLn := int(hdr.squareSize / 2)
quadrantOffset := quadrantIdx * odsLn * odsLn * shrLn

shares := make([]share.Share, odsLn)

pos := idx * odsLn
offset := f.hdr.Size() + pos*shrLn
pos := rowIdx * odsLn
offset := hdr.Size() + quadrantOffset + pos*shrLn

axsData := make([]byte, odsLn*shrLn)
if _, err := f.fl.ReadAt(axsData, int64(offset)); err != nil {
if _, err := fl.ReadAt(axsData, int64(offset)); err != nil {
return nil, err
}

Expand All @@ -253,18 +261,18 @@ func (f *ODSFile) readRow(idx int) ([]share.Share, error) {
return shares, nil
}

func (f *ODSFile) readCol(axisIdx, quadrantIdx int) ([]share.Share, error) {
shrLn := int(f.hdr.shareSize)
odsLn := f.size() / 2
func readCol(fl io.ReaderAt, hdr *headerV0, colIdx, quadrantIdx int) ([]share.Share, error) {
shrLn := int(hdr.shareSize)
odsLn := int(hdr.squareSize / 2)
quadrantOffset := quadrantIdx * odsLn * odsLn * shrLn

shares := make([]share.Share, odsLn)
for i := range shares {
pos := axisIdx + i*odsLn
offset := f.hdr.Size() + quadrantOffset + pos*shrLn
pos := colIdx + i*odsLn
offset := hdr.Size() + quadrantOffset + pos*shrLn

shr := make(share.Share, shrLn)
if _, err := f.fl.ReadAt(shr, int64(offset)); err != nil {
if _, err := fl.ReadAt(shr, int64(offset)); err != nil {
return nil, err
}
shares[i] = shr
Expand Down
111 changes: 83 additions & 28 deletions store/file/ods_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,9 @@ func TestReadODSFromFile(t *testing.T) {
f, err := CreateODSFile(path, []byte{}, eds)
require.NoError(t, err)

err = f.readODS()
ods, err := f.readODS()
require.NoError(t, err)
for i, row := range f.ods {
for i, row := range ods {
original := eds.Row(uint(i))[:eds.Width()/2]
require.True(t, len(original) == len(row))
require.Equal(t, original, row)
Expand All @@ -65,19 +65,18 @@ func TestODSFile(t *testing.T) {
eds.TestSuiteAccessor(ctx, t, createODSFile, ODSSize)
}

// ReconstructSome, default codec
// BenchmarkAxisFromODSFile/Size:32/Axis:row/squareHalf:first(original)-10 455848 2588 ns/op
// BenchmarkAxisFromODSFile/Size:32/Axis:row/squareHalf:second(extended)-10 9015 203950 ns/op
// BenchmarkAxisFromODSFile/Size:32/Axis:col/squareHalf:first(original)-10 52734 21178 ns/op
// BenchmarkAxisFromODSFile/Size:32/Axis:col/squareHalf:second(extended)-10 8830 127452 ns/op
// BenchmarkAxisFromODSFile/Size:64/Axis:row/squareHalf:first(original)-10 303834 4763 ns/op
// BenchmarkAxisFromODSFile/Size:64/Axis:row/squareHalf:second(extended)-10 2940 426246 ns/op
// BenchmarkAxisFromODSFile/Size:64/Axis:col/squareHalf:first(original)-10 27758 42842 ns/op
// BenchmarkAxisFromODSFile/Size:64/Axis:col/squareHalf:second(extended)-10 3385 353868 ns/op
// BenchmarkAxisFromODSFile/Size:128/Axis:row/squareHalf:first(original)-10 172086 6455 ns/op
// BenchmarkAxisFromODSFile/Size:128/Axis:row/squareHalf:second(extended)-10 672 1550386 ns/op
// BenchmarkAxisFromODSFile/Size:128/Axis:col/squareHalf:first(original)-10 14202 84316 ns/op
// BenchmarkAxisFromODSFile/Size:128/Axis:col/squareHalf:second(extended)-10 978 1230980 ns/op
// BenchmarkAxisFromODSFile/Size:32/ProofType:row/squareHalf:0-10 460231 2555 ns/op
// BenchmarkAxisFromODSFile/Size:32/ProofType:row/squareHalf:1-10 5320 218609 ns/op
// BenchmarkAxisFromODSFile/Size:32/ProofType:col/squareHalf:0-10 4572247 256.7 ns/op
// BenchmarkAxisFromODSFile/Size:32/ProofType:col/squareHalf:1-10 5170 212567 ns/op
// BenchmarkAxisFromODSFile/Size:64/ProofType:row/squareHalf:0-10 299281 3777 ns/op
// BenchmarkAxisFromODSFile/Size:64/ProofType:row/squareHalf:1-10 1646 661930 ns/op
// BenchmarkAxisFromODSFile/Size:64/ProofType:col/squareHalf:0-10 3318733 359.1 ns/op
// BenchmarkAxisFromODSFile/Size:64/ProofType:col/squareHalf:1-10 1600 648482 ns/op
// BenchmarkAxisFromODSFile/Size:128/ProofType:row/squareHalf:0-10 170642 6347 ns/op
// BenchmarkAxisFromODSFile/Size:128/ProofType:row/squareHalf:1-10 328 3194674 ns/op
// BenchmarkAxisFromODSFile/Size:128/ProofType:col/squareHalf:0-10 1931910 640.9 ns/op
// BenchmarkAxisFromODSFile/Size:128/ProofType:col/squareHalf:1-10 387 3304090 ns/op
func BenchmarkAxisFromODSFile(b *testing.B) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
b.Cleanup(cancel)
Expand All @@ -90,19 +89,43 @@ func BenchmarkAxisFromODSFile(b *testing.B) {
eds.BenchGetHalfAxisFromAccessor(ctx, b, newFile, minSize, maxSize)
}

// BenchmarkShareFromODSFile/Size:32/Axis:row/squareHalf:first(original)-10 10339 111328 ns/op
// BenchmarkShareFromODSFile/Size:32/Axis:row/squareHalf:second(extended)-10 3392 359180 ns/op
// BenchmarkShareFromODSFile/Size:32/Axis:col/squareHalf:first(original)-10 8925 131352 ns/op
// BenchmarkShareFromODSFile/Size:32/Axis:col/squareHalf:second(extended)-10 3447 346218 ns/op
// BenchmarkShareFromODSFile/Size:64/Axis:row/squareHalf:first(original)-10 5503 215833 ns/op
// BenchmarkShareFromODSFile/Size:64/Axis:row/squareHalf:second(extended)-10 1231 1001053 ns/op
// BenchmarkShareFromODSFile/Size:64/Axis:col/squareHalf:first(original)-10 4711 250001 ns/op
// BenchmarkShareFromODSFile/Size:64/Axis:col/squareHalf:second(extended)-10 1315 910079 ns/op
// BenchmarkShareFromODSFile/Size:128/Axis:row/squareHalf:first(original)-10 2364 435748 ns/op
// BenchmarkShareFromODSFile/Size:128/Axis:row/squareHalf:second(extended)-10 358 3330620 ns/op
// BenchmarkShareFromODSFile/Size:128/Axis:col/squareHalf:first(original)-10 2114 514642 ns/op
// BenchmarkShareFromODSFile/Size:128/Axis:col/squareHalf:second(extended)-10 373 3068104 ns/op
func BenchmarkShareFromODSFile(b *testing.B) {
// BenchmarkAxisFromODSFileDisabledCache/Size:32/ProofType:row/squareHalf:0-10 481326 2447 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:32/ProofType:row/squareHalf:1-10 5134 218191 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:32/ProofType:col/squareHalf:0-10 56260 21109 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:32/ProofType:col/squareHalf:1-10 5608 217877 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:64/ProofType:row/squareHalf:0-10 321994 3941 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:64/ProofType:row/squareHalf:1-10 1237 919419 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:64/ProofType:col/squareHalf:0-10 28233 43209 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:64/ProofType:col/squareHalf:1-10 1334 898654 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:128/ProofType:row/squareHalf:0-10 179788 6839 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:128/ProofType:row/squareHalf:1-10 310 3935097 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:128/ProofType:col/squareHalf:0-10 13867 85854 ns/op
// BenchmarkAxisFromODSFileDisabledCache/Size:128/ProofType:col/squareHalf:1-10 298 3900021 ns/op
func BenchmarkAxisFromODSFileDisabledCache(b *testing.B) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
b.Cleanup(cancel)

minSize, maxSize := 32, 128
newFile := func(size int) eds.Accessor {
eds := edstest.RandEDS(b, size)
return createODSFileDisabledCache(b, eds)
}
eds.BenchGetHalfAxisFromAccessor(ctx, b, newFile, minSize, maxSize)
}

// BenchmarkSampleFromODSFile/Size:32/quadrant:1-10 10908 104872 ns/op
// BenchmarkSampleFromODSFile/Size:32/quadrant:2-10 9906 104641 ns/op
// BenchmarkSampleFromODSFile/Size:32/quadrant:3-10 8983 123384 ns/op
// BenchmarkSampleFromODSFile/Size:32/quadrant:4-10 3476 343850 ns/op
// BenchmarkSampleFromODSFile/Size:64/quadrant:1-10 5835 200151 ns/op
// BenchmarkSampleFromODSFile/Size:64/quadrant:2-10 5401 201271 ns/op
// BenchmarkSampleFromODSFile/Size:64/quadrant:3-10 4648 239045 ns/op
// BenchmarkSampleFromODSFile/Size:64/quadrant:4-10 1263 895983 ns/op
// BenchmarkSampleFromODSFile/Size:128/quadrant:1-10 2475 409687 ns/op
// BenchmarkSampleFromODSFile/Size:128/quadrant:2-10 2790 411153 ns/op
// BenchmarkSampleFromODSFile/Size:128/quadrant:3-10 2286 487123 ns/op
// BenchmarkSampleFromODSFile/Size:128/quadrant:4-10 321 3698735 ns/op
func BenchmarkSampleFromODSFile(b *testing.B) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
b.Cleanup(cancel)

Expand All @@ -114,9 +137,41 @@ func BenchmarkShareFromODSFile(b *testing.B) {
eds.BenchGetSampleFromAccessor(ctx, b, newFile, minSize, maxSize)
}

// BenchmarkSampleFromODSFileDisabledCache/Size:32/quadrant:1-10 11040 106378 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:32/quadrant:2-10 9936 106403 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:32/quadrant:3-10 8635 124142 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:32/quadrant:4-10 1940 596330 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:64/quadrant:1-10 5930 199782 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:64/quadrant:2-10 5494 201658 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:64/quadrant:3-10 4756 237897 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:64/quadrant:4-10 638 1874038 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:128/quadrant:1-10 2500 408092 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:128/quadrant:2-10 2696 410861 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:128/quadrant:3-10 2290 490488 ns/op
// BenchmarkSampleFromODSFileDisabledCache/Size:128/quadrant:4-10 159 7660843 ns/op
func BenchmarkSampleFromODSFileDisabledCache(b *testing.B) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
b.Cleanup(cancel)

minSize, maxSize := 32, 128
newFile := func(size int) eds.Accessor {
eds := edstest.RandEDS(b, size)
return createODSFileDisabledCache(b, eds)
}
eds.BenchGetSampleFromAccessor(ctx, b, newFile, minSize, maxSize)
}

func createODSFile(t testing.TB, eds *rsmt2d.ExtendedDataSquare) eds.Accessor {
path := t.TempDir() + "/" + strconv.Itoa(rand.Intn(1000))
fl, err := CreateODSFile(path, []byte{}, eds)
require.NoError(t, err)
return fl
}

func createODSFileDisabledCache(t testing.TB, eds *rsmt2d.ExtendedDataSquare) eds.Accessor {
path := t.TempDir() + "/" + strconv.Itoa(rand.Intn(1000))
fl, err := CreateODSFile(path, []byte{}, eds)
require.NoError(t, err)
fl.disableCache = true
return fl
}
Loading

0 comments on commit 218bdc7

Please sign in to comment.