From 204efdb629fa5b90c00718fcfd9fe9adc942765d Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Thu, 27 Aug 2020 10:42:52 -0400 Subject: [PATCH 01/11] Getting started --- hamt_bench_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/hamt_bench_test.go b/hamt_bench_test.go index deab914..775f60e 100644 --- a/hamt_bench_test.go +++ b/hamt_bench_test.go @@ -53,6 +53,7 @@ func BenchmarkSerializeNode(b *testing.B) { type benchSetCase struct { kcount int bitwidth int + // flushInterval int } var benchSetCaseTable []benchSetCase @@ -76,6 +77,9 @@ func init() { 7, 8, } + // flushIntervals := []int{ + // 1, + // } // bucketsize-aka-arraywidth? maybe someday. for _, c := range kCounts { for _, bw := range bitwidths { @@ -150,9 +154,9 @@ func BenchmarkFill(b *testing.B) { // // The number of *additional* blocks per entry is reported. // This number is usually less than one, because the bulk flush means changes might be amortized. -func BenchmarkSetBulk(b *testing.B) { - doBenchmarkSetSuite(b, false) -} +// func BenchmarkSetBulk(b *testing.B) { +// doBenchmarkSetSuite(b, false) +// } // BenchmarkSetIndividual is the same as BenchmarkSetBulk, but flushes more. // Flush happens per insert. @@ -166,8 +170,9 @@ func BenchmarkSetIndividual(b *testing.B) { } func doBenchmarkSetSuite(b *testing.B, flushPer bool) { - for _, t := range benchSetCaseTable { + for j, t := range benchSetCaseTable { b.Run(fmt.Sprintf("n=%dk/bitwidth=%d", t.kcount, t.bitwidth), func(b *testing.B) { + fmt.Printf("Case: %d, b.N=%d\n", j, b.N) for i := 0; i < b.N; i++ { r := rander{rand.New(rand.NewSource(int64(i)))} blockstore := newMockBlocks() From 641930aab4e0bf5839e8ed84bab4d455fea11c1e Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Fri, 28 Aug 2020 10:50:02 -0400 Subject: [PATCH 02/11] WIP --- hamt_bench_test.go | 84 +++++++++++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/hamt_bench_test.go b/hamt_bench_test.go index 775f60e..2bde8c7 100644 --- a/hamt_bench_test.go +++ b/hamt_bench_test.go @@ -51,12 +51,28 @@ func BenchmarkSerializeNode(b *testing.B) { } type benchSetCase struct { + kcount int + bitwidth int + datasize int + flushInterval int +} + +type benchFillCase struct { + kcount int + bitwidth int + datasize int + flushInterval int +} + +type benchFindCase struct { kcount int bitwidth int - // flushInterval int + datasize int } var benchSetCaseTable []benchSetCase +var benchFillCaseTable []benchFillCase +var benchFindCaseTable []benchFindCase func init() { kCounts := []int{ @@ -77,13 +93,40 @@ func init() { 7, 8, } - // flushIntervals := []int{ - // 1, - // } + flushIntervals := []int{ + 1, + 1000, + } + dataSize := []int{ + 1, + } // bucketsize-aka-arraywidth? maybe someday. for _, c := range kCounts { - for _, bw := range bitwidths { - benchSetCaseTable = append(benchSetCaseTable, benchSetCase{kcount: c, bitwidth: bw}) + for _, d := range dataSize { + for _, bw := range bitwidths { + benchFindCaseTable = append(benchFindCaseTable, + benchFindCase{ + kcount: c, + bitwidth: bw, + datasize: d, + }) + for _, f := range flushIntervals { + benchFillCaseTable = append(benchFillCaseTable, + benchFillCase{ + kcount: c, + bitwidth: bw, + datasize: d, + flushInterval: f, + }) + benchSetCaseTable = append(benchSetCaseTable, + benchSetCase{ + kcount: c, + bitwidth: bw, + datasize: d, + flushInterval: f, + }) + } + } } } } @@ -114,7 +157,7 @@ func init() { // See "BenchmarkSet*" for a probe of how long it takes to set additional entries in an already-large hamt // (this gives a more interesting and useful nanoseconds-per-op indicators). func BenchmarkFill(b *testing.B) { - for _, t := range benchSetCaseTable { + for _, t := range benchFillCaseTable { b.Run(fmt.Sprintf("n=%dk/bitwidth=%d", t.kcount, t.bitwidth), func(b *testing.B) { for i := 0; i < b.N; i++ { r := rander{rand.New(rand.NewSource(int64(i)))} @@ -148,32 +191,24 @@ func BenchmarkFill(b *testing.B) { } } -// BenchmarkSetBulk creates a large HAMT, then resets the timer, and does another 1000 inserts, +// BenchmarkSet creates a large HAMT, then starts the timer, and does another 1000 inserts, // measuring the time taken for this second batch of inserts. -// Flushing happens once after all 1000 inserts. -// -// The number of *additional* blocks per entry is reported. -// This number is usually less than one, because the bulk flush means changes might be amortized. -// func BenchmarkSetBulk(b *testing.B) { -// doBenchmarkSetSuite(b, false) -// } - -// BenchmarkSetIndividual is the same as BenchmarkSetBulk, but flushes more. -// Flush happens per insert. +// Flushing rate is parameterized. // // The number of *additional* blocks per entry is reported. -// Since we flush each insert individually, this number should be at least 1 -- +// This number is usually less than one with high flush interval means changes might be amortized. +// For flush interval one this number should be at least 1 -- // however, since we choose random keys, it can still turn out lower if keys happen to collide. // (The Set method does not make it possible to adjust our denominator to compensate for this: it does not yield previous values nor indicators of prior presense.) -func BenchmarkSetIndividual(b *testing.B) { - doBenchmarkSetSuite(b, true) +func BenchmarkSet(b *testing.B) { + doBenchmarkSetSuite(b) } -func doBenchmarkSetSuite(b *testing.B, flushPer bool) { +func doBenchmarkSetSuite(b *testing.B) { for j, t := range benchSetCaseTable { b.Run(fmt.Sprintf("n=%dk/bitwidth=%d", t.kcount, t.bitwidth), func(b *testing.B) { - fmt.Printf("Case: %d, b.N=%d\n", j, b.N) for i := 0; i < b.N; i++ { + b.StopTimer() r := rander{rand.New(rand.NewSource(int64(i)))} blockstore := newMockBlocks() n := NewNode(cbor.NewCborStore(blockstore), UseTreeBitWidth(t.bitwidth)) @@ -187,10 +222,11 @@ func doBenchmarkSetSuite(b *testing.B, flushPer bool) { b.Fatal(err) } initalBlockstoreSize := len(blockstore.data) - b.ResetTimer() + // b.ResetTimer() blockstore.stats = blockstoreStats{} // Additional inserts: b.ReportAllocs() + b.StartTimer() for j := 0; j < 1000; j++ { if err := n.Set(context.Background(), r.randString(), r.randValue()); err != nil { b.Fatal(err) From de6fc0f9e61ef9660eb3b4e98605a478974c1a8d Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Fri, 6 Nov 2020 15:40:51 -0500 Subject: [PATCH 03/11] Update benchmarks further --- hamt_bench_test.go | 206 ++++++++++++++++++++++++--------------------- hamt_test.go | 2 + 2 files changed, 112 insertions(+), 96 deletions(-) diff --git a/hamt_bench_test.go b/hamt_bench_test.go index 2bde8c7..a018416 100644 --- a/hamt_bench_test.go +++ b/hamt_bench_test.go @@ -15,14 +15,14 @@ type rander struct { r *rand.Rand } -func (r *rander) randString() string { - buf := make([]byte, 18) +func (r *rander) randString(stringSize int) string { + buf := make([]byte, stringSize) rand.Read(buf) return hex.EncodeToString(buf) } -func (r *rander) randValue() []byte { - buf := make([]byte, 30) +func (r *rander) randValue(datasize int) []byte { + buf := make([]byte, datasize) rand.Read(buf) return buf } @@ -34,7 +34,7 @@ func BenchmarkSerializeNode(b *testing.B) { n := NewNode(cs) for i := 0; i < 50; i++ { - if err := n.Set(context.TODO(), r.randString(), r.randValue()); err != nil { + if err := n.Set(context.TODO(), r.randString(18), r.randValue(30)); err != nil { b.Fatal(err) } } @@ -50,42 +50,28 @@ func BenchmarkSerializeNode(b *testing.B) { } } -type benchSetCase struct { - kcount int - bitwidth int - datasize int - flushInterval int -} - -type benchFillCase struct { - kcount int - bitwidth int - datasize int - flushInterval int +type hamtParams struct { + id string + count int + datasize int + keysize int } -type benchFindCase struct { - kcount int +type benchCase struct { + id string + count int bitwidth int datasize int + keysize int } -var benchSetCaseTable []benchSetCase -var benchFillCaseTable []benchFillCase -var benchFindCaseTable []benchFindCase +var caseTable []benchCase func init() { - kCounts := []int{ - 1, - 5, - 10, - 50, - 100, - 500, - 1000, // aka 1M - //10000, // aka 10M -- you'll need a lot of RAM for this. Also, some patience. - } + bitwidths := []int{ + 1, + 2, 3, 4, 5, @@ -93,40 +79,63 @@ func init() { 7, 8, } - flushIntervals := []int{ - 1, - 1000, - } - dataSize := []int{ - 1, + + hamts := []hamtParams{ + hamtParams{ + id: "init.AddressMap", + count: 55649, + datasize: 3, + keysize: 26, + }, + hamtParams{ + id: "market.PendingProposals", + count: 40713, + datasize: 151, + keysize: 38, + }, + hamtParams{ + id: "market.EscrowWTable", + count: 2113, + datasize: 7, + keysize: 4, + }, + hamtParams{ + id: "market.LockedTable", + count: 2098, + datasize: 4, + keysize: 4, + }, + hamtParams{ + id: "market.DealOpsByEpoch", + count: 16558, + datasize: 43, + keysize: 3, + }, + hamtParams{ + id: "power.CronEventQueue", + count: 60, + datasize: 43, + keysize: 3, + }, + hamtParams{ + id: "power.CLaims", + count: 15610, + datasize: 5, + keysize: 3, + }, } + // bucketsize-aka-arraywidth? maybe someday. - for _, c := range kCounts { - for _, d := range dataSize { - for _, bw := range bitwidths { - benchFindCaseTable = append(benchFindCaseTable, - benchFindCase{ - kcount: c, - bitwidth: bw, - datasize: d, - }) - for _, f := range flushIntervals { - benchFillCaseTable = append(benchFillCaseTable, - benchFillCase{ - kcount: c, - bitwidth: bw, - datasize: d, - flushInterval: f, - }) - benchSetCaseTable = append(benchSetCaseTable, - benchSetCase{ - kcount: c, - bitwidth: bw, - datasize: d, - flushInterval: f, - }) - } - } + for _, h := range hamts { + for _, bw := range bitwidths { + caseTable = append(caseTable, + benchCase{ + id: fmt.Sprintf("%s -- bw=%d", h.id, bw), + count: h.count, + bitwidth: bw, + datasize: h.datasize, + keysize: h.keysize, + }) } } } @@ -157,15 +166,15 @@ func init() { // See "BenchmarkSet*" for a probe of how long it takes to set additional entries in an already-large hamt // (this gives a more interesting and useful nanoseconds-per-op indicators). func BenchmarkFill(b *testing.B) { - for _, t := range benchFillCaseTable { - b.Run(fmt.Sprintf("n=%dk/bitwidth=%d", t.kcount, t.bitwidth), func(b *testing.B) { + for _, t := range caseTable { + b.Run(fmt.Sprintf("%s", t.id), func(b *testing.B) { for i := 0; i < b.N; i++ { r := rander{rand.New(rand.NewSource(int64(i)))} blockstore := newMockBlocks() n := NewNode(cbor.NewCborStore(blockstore), UseTreeBitWidth(t.bitwidth)) //b.ResetTimer() - for j := 0; j < t.kcount*1000; j++ { - if err := n.Set(context.Background(), r.randString(), r.randValue()); err != nil { + for j := 0; j < t.count; j++ { + if err := n.Set(context.Background(), r.randString(t.keysize), r.randValue(t.datasize)); err != nil { b.Fatal(err) } } @@ -179,56 +188,61 @@ func BenchmarkFill(b *testing.B) { if blockstore.stats.evtcntPutDup > 0 { b.Logf("on round N=%d: blockstore stats: %#v\n", b.N, blockstore.stats) // note: must refer to this before doing `n.checkSize`; that function has many effects. } - b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(t.kcount*1000), "getEvts/entry") - b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(t.kcount*1000), "putEvts/entry") - b.ReportMetric(float64(len(blockstore.data))/float64(t.kcount*1000), "blocks/entry") + b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(t.count), "getEvts/entry") + b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(t.count), "putEvts/entry") + b.ReportMetric(float64(len(blockstore.data))/float64(t.count), "blocks/entry") binarySize, _ := n.checkSize(context.Background()) - b.ReportMetric(float64(binarySize)/float64(t.kcount*1000), "bytes(hamtAccnt)/entry") - b.ReportMetric(float64(blockstore.totalBlockSizes())/float64(t.kcount*1000), "bytes(blockstoreAccnt)/entry") + b.ReportMetric(float64(binarySize)/float64(t.count), "bytes(hamtAccnt)/entry") + b.ReportMetric(float64(blockstore.totalBlockSizes())/float64(t.count), "bytes(blockstoreAccnt)/entry") b.StartTimer() } }) } } -// BenchmarkSet creates a large HAMT, then starts the timer, and does another 1000 inserts, +// BenchmarkSetBulk creates a large HAMT, then starts the timer, and does another 1000 inserts, // measuring the time taken for this second batch of inserts. -// Flushing rate is parameterized. +// Flushing happens once after all 1000 inserts. // // The number of *additional* blocks per entry is reported. // This number is usually less than one with high flush interval means changes might be amortized. -// For flush interval one this number should be at least 1 -- -// however, since we choose random keys, it can still turn out lower if keys happen to collide. -// (The Set method does not make it possible to adjust our denominator to compensate for this: it does not yield previous values nor indicators of prior presense.) -func BenchmarkSet(b *testing.B) { - doBenchmarkSetSuite(b) +func BenchmarkSetBulk(b *testing.B) { + doBenchmarkSetSuite(b, false) +} + +// BenchmarkSetIndividual is the same as BenchmarkSetBulk, but flushes more. +// Flush happens per insert. +// +// The number of *additional* blocks per entry is reported. +// Since we flush each insert individually, this number should be at least 1. +func BenchmarkSetIndividual(b *testing.B) { + doBenchmarkSetSuite(b, true) } -func doBenchmarkSetSuite(b *testing.B) { - for j, t := range benchSetCaseTable { - b.Run(fmt.Sprintf("n=%dk/bitwidth=%d", t.kcount, t.bitwidth), func(b *testing.B) { +func doBenchmarkSetSuite(b *testing.B, flushPer bool) { + for _, t := range caseTable { + b.Run(fmt.Sprintf("%s", t.id), func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() r := rander{rand.New(rand.NewSource(int64(i)))} blockstore := newMockBlocks() n := NewNode(cbor.NewCborStore(blockstore), UseTreeBitWidth(t.bitwidth)) // Initial fill: - for j := 0; j < t.kcount*1000; j++ { - if err := n.Set(context.Background(), r.randString(), r.randValue()); err != nil { + for j := 0; j < t.count; j++ { + if err := n.Set(context.Background(), r.randString(t.keysize), r.randValue(t.datasize)); err != nil { b.Fatal(err) } } if err := n.Flush(context.Background()); err != nil { b.Fatal(err) } - initalBlockstoreSize := len(blockstore.data) // b.ResetTimer() blockstore.stats = blockstoreStats{} // Additional inserts: b.ReportAllocs() b.StartTimer() for j := 0; j < 1000; j++ { - if err := n.Set(context.Background(), r.randString(), r.randValue()); err != nil { + if err := n.Set(context.Background(), r.randString(t.keysize), r.randValue(t.datasize)); err != nil { b.Fatal(err) } if flushPer { @@ -249,9 +263,9 @@ func doBenchmarkSetSuite(b *testing.B) { if blockstore.stats.evtcntPutDup > 0 { b.Logf("on round N=%d: blockstore stats: %#v\n", b.N, blockstore.stats) } - b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(t.kcount*1000), "getEvts/entry") - b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(t.kcount*1000), "putEvts/entry") - b.ReportMetric(float64(len(blockstore.data)-initalBlockstoreSize)/float64(1000), "addntlBlocks/addntlEntry") + b.ReportMetric(float64(blockstore.stats.evtcntGet)/1000, "getEvts/set") + b.ReportMetric(float64(blockstore.stats.evtcntPut)/1000, "putEvts/set") + b.ReportMetric(float64(blockstore.stats.bytesPut)/1000, "bytesPut/set") b.StartTimer() } }) @@ -259,13 +273,13 @@ func doBenchmarkSetSuite(b *testing.B) { } func BenchmarkFind(b *testing.B) { - for _, t := range benchSetCaseTable { - b.Run(fmt.Sprintf("n=%dk/bitwidth=%d", t.kcount, t.bitwidth), - doBenchmarkEntriesCount(t.kcount*1000, t.bitwidth)) + for _, t := range caseTable { + b.Run(fmt.Sprintf("%s", t.id), + doBenchmarkEntriesCount(t.count, t.bitwidth, t.datasize, t.keysize)) } } -func doBenchmarkEntriesCount(num int, bitWidth int) func(b *testing.B) { +func doBenchmarkEntriesCount(num int, bitWidth int, datasize int, keysize int) func(b *testing.B) { r := rander{rand.New(rand.NewSource(int64(num)))} return func(b *testing.B) { blockstore := newMockBlocks() @@ -274,8 +288,8 @@ func doBenchmarkEntriesCount(num int, bitWidth int) func(b *testing.B) { var keys []string for i := 0; i < num; i++ { - k := r.randString() - if err := n.Set(context.TODO(), k, r.randValue()); err != nil { + k := r.randString(keysize) + if err := n.Set(context.TODO(), k, r.randValue(datasize)); err != nil { b.Fatal(err) } keys = append(keys, k) diff --git a/hamt_test.go b/hamt_test.go index c67288d..18765be 100644 --- a/hamt_test.go +++ b/hamt_test.go @@ -41,6 +41,7 @@ func (mb *mockBlocks) Get(c cid.Cid) (block.Block, error) { func (mb *mockBlocks) Put(b block.Block) error { mb.stats.evtcntPut++ + mb.stats.bytesPut += len(b.RawData()) if _, exists := mb.data[b.Cid()]; exists { mb.stats.evtcntPutDup++ } @@ -51,6 +52,7 @@ func (mb *mockBlocks) Put(b block.Block) error { type blockstoreStats struct { evtcntGet int evtcntPut int + bytesPut int evtcntPutDup int } From 01c83fa2b6f996c99db994ddc9c1b7d81ddcdcd1 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Wed, 18 Nov 2020 21:52:15 -0500 Subject: [PATCH 04/11] Reset benchmark and fixes --- hamt_bench_test.go | 153 ++++++++++++++++++++++++++++++++------------- 1 file changed, 110 insertions(+), 43 deletions(-) diff --git a/hamt_bench_test.go b/hamt_bench_test.go index a018416..9c4ffbc 100644 --- a/hamt_bench_test.go +++ b/hamt_bench_test.go @@ -27,6 +27,11 @@ func (r *rander) randValue(datasize int) []byte { return buf } +func (r *rander) selectKey(keys []string) string { + i := rand.Int() % len(keys) + return keys[i] +} + func BenchmarkSerializeNode(b *testing.B) { r := rander{rand.New(rand.NewSource(1234))} @@ -87,42 +92,42 @@ func init() { datasize: 3, keysize: 26, }, - hamtParams{ - id: "market.PendingProposals", - count: 40713, - datasize: 151, - keysize: 38, - }, - hamtParams{ - id: "market.EscrowWTable", - count: 2113, - datasize: 7, - keysize: 4, - }, - hamtParams{ - id: "market.LockedTable", - count: 2098, - datasize: 4, - keysize: 4, - }, - hamtParams{ - id: "market.DealOpsByEpoch", - count: 16558, - datasize: 43, - keysize: 3, - }, - hamtParams{ - id: "power.CronEventQueue", - count: 60, - datasize: 43, - keysize: 3, - }, - hamtParams{ - id: "power.CLaims", - count: 15610, - datasize: 5, - keysize: 3, - }, + // hamtParams{ + // id: "market.PendingProposals", + // count: 40713, + // datasize: 151, + // keysize: 38, + // }, + // hamtParams{ + // id: "market.EscrowWTable", + // count: 2113, + // datasize: 7, + // keysize: 4, + // }, + // hamtParams{ + // id: "market.LockedTable", + // count: 2098, + // datasize: 4, + // keysize: 4, + // }, + // hamtParams{ + // id: "market.DealOpsByEpoch", + // count: 16558, + // datasize: 43, + // keysize: 3, + // }, + // hamtParams{ + // id: "power.CronEventQueue", + // count: 60, + // datasize: 43, + // keysize: 3, + // }, + // hamtParams{ + // id: "power.CLaims", + // count: 15610, + // datasize: 5, + // keysize: 3, + // }, } // bucketsize-aka-arraywidth? maybe someday. @@ -305,21 +310,83 @@ func doBenchmarkEntriesCount(num int, bitWidth int, datasize int, keysize int) f } runtime.GC() - blockstore.stats = blockstoreStats{} b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - nd, err := LoadNode(context.TODO(), cs, c, UseTreeBitWidth(bitWidth)) - if err != nil { - b.Fatal(err) + blockstore.stats = blockstoreStats{} + for j := 0; j < 1000; j++ { + nd, err := LoadNode(context.TODO(), cs, c, UseTreeBitWidth(bitWidth)) + if err != nil { + b.Fatal(err) + } + if err = nd.Find(context.TODO(), r.selectKey(keys), nil); err != nil { + b.Fatal(err) + } } + b.StopTimer() + b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(1000), "getEvts/find") + b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(1000), "putEvts/find") // surely this is zero, but for completeness. + b.StartTimer() + } + } +} - if err = nd.Find(context.TODO(), keys[i%num], nil); err != nil { +func BenchmarkReset(b *testing.B) { + for _, t := range caseTable { + b.Run(fmt.Sprintf("%s", t.id), + doBenchmarkResetSuite(t.count, t.bitwidth, t.datasize, t.keysize)) + } +} + +func doBenchmarkResetSuite(num int, bitWidth int, datasize int, keysize int) func(b *testing.B) { + r := rander{rand.New(rand.NewSource(int64(num)))} + return func(b *testing.B) { + blockstore := newMockBlocks() + cs := cbor.NewCborStore(blockstore) + n := NewNode(cs, UseTreeBitWidth(bitWidth)) + + var keys []string + for i := 0; i < num; i++ { + k := r.randString(keysize) + if err := n.Set(context.TODO(), k, r.randValue(datasize)); err != nil { b.Fatal(err) } + keys = append(keys, k) + } + + if err := n.Flush(context.TODO()); err != nil { + b.Fatal(err) + } + + c, err := cs.Put(context.TODO(), n) + if err != nil { + b.Fatal(err) + } + + runtime.GC() + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + blockstore.stats = blockstoreStats{} + for j := 0; j < 1000; j++ { + nd, err := LoadNode(context.TODO(), cs, c, UseTreeBitWidth(bitWidth)) + if err != nil { + b.Fatal(err) + } + if err := nd.Set(context.Background(), r.selectKey(keys), r.randValue(datasize)); err != nil { + b.Fatal(err) + } + if err := nd.Flush(context.Background()); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + b.ReportMetric(float64(blockstore.stats.evtcntGet)/1000, "getEvts/set") + b.ReportMetric(float64(blockstore.stats.evtcntPut)/1000, "putEvts/set") + b.ReportMetric(float64(blockstore.stats.bytesPut)/1000, "bytesPut/set") + b.StartTimer() } - b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(b.N), "getEvts/find") - b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(b.N), "putEvts/find") // surely this is zero, but for completeness. } } From e1fb0daa9f3016a1c02d729caf561e7a2ff4d2e4 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Mon, 23 Nov 2020 18:23:06 -0500 Subject: [PATCH 05/11] Add benchparse tool --- benchparse/main.go | 233 +++++++++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 8 ++ hamt_bench_test.go | 94 +++++++++--------- hamt_test.go | 16 ++++ 5 files changed, 305 insertions(+), 47 deletions(-) create mode 100644 benchparse/main.go diff --git a/benchparse/main.go b/benchparse/main.go new file mode 100644 index 0000000..551040a --- /dev/null +++ b/benchparse/main.go @@ -0,0 +1,233 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "os" + "sort" + "strconv" + "strings" + + "golang.org/x/perf/benchstat" +) + +func main() { + if len(os.Args) < 2 { + panic("need to specify input file") + } + fIn, err := os.Open(os.Args[1]) + if err != nil { + panic(err) + } + + c := benchstat.Collection{} + if err := c.AddFile("config1", fIn); err != nil { + panic(err) + } + tables := c.Tables() + + reports, err := genReports(tables) + if err != nil { + panic(err) + } + if err := processReports(reports); err != nil { + panic(err) + } + + fOut, err := os.Create("out.csv") + if err != nil { + panic(err) + } + defer fOut.Close() + if err := writeReports(reports, fOut); err != nil { + panic(err) + } +} + +// All metrics for all benchmarks for one HAMT configuration +type HAMTReport struct { + ID string + Benchmarks map[string]Benchmark +} + +func (hr *HAMTReport) Write(datawriter *bufio.Writer) error { + // Write title + if _, err := datawriter.WriteString(hr.ID + "\n"); err != nil { + return err + } + + // Write benchmarks + for _, benchmark := range hr.Benchmarks { + if err := benchmark.Write(datawriter); err != nil { + return err + } + } + return nil +} + +type Benchmark struct { + Name string + Metrics map[string]struct{} + Sweep map[int]map[string]float64 // map[Bitwidth]MetricsMap +} + +func (b *Benchmark) Write(datawriter *bufio.Writer) error { + // Get sorted keys + metrics := make([]string, 0) + for metric := range b.Metrics { + metrics = append(metrics, metric) + } + sort.Strings(metrics) + + bws := make([]int, 0) + for bw := range b.Sweep { + bws = append(bws, bw) + } + sort.Ints(bws) + + // Write headers + headers := []string{"Benchmark", "Bitwidth"} + headers = append(headers, metrics...) + if _, err := datawriter.WriteString(strings.Join(headers, ",")); err != nil { + return err + } + if _, err := datawriter.WriteString("\n"); err != nil { + return err + } + + // Write sweeps + for _, bw := range bws { + if _, err := datawriter.WriteString(b.Name); err != nil { + return err + } + if _, err := datawriter.WriteString("," + strconv.Itoa(bw)); err != nil { + return err + } + for _, metric := range metrics { + m, found := b.Sweep[bw][metric] + if !found { + // empty column + if _, err := datawriter.WriteString(","); err != nil { + return err + } + continue + } + if _, err := datawriter.WriteString(fmt.Sprintf(",%.3f", m)); err != nil { + return err + } + } + if _, err := datawriter.WriteString("\n"); err != nil { + return err + } + } + + return nil +} + +func writeReports(reports map[string]*HAMTReport, w io.Writer) error { + datawriter := bufio.NewWriter(w) + for _, report := range reports { + if err := report.Write(datawriter); err != nil { + return err + } + if _, err := datawriter.WriteString("\n\n"); err != nil { + return err + } + } + return datawriter.Flush() +} + +func genReports(tables []*benchstat.Table) (map[string]*HAMTReport, error) { + reports := make(map[string]*HAMTReport) + for _, table := range tables { + for _, row := range table.Rows { + if len(row.Metrics) == 0 { + continue + } + if len(row.Metrics) >= 2 { + panic("don't know how to handle lots of metrics") + } + good, bname, hamtID, bw := parseBench(row.Benchmark) + if !good { + continue + } + fmt.Printf("bname: %s, hamtID: %s, bw: %d\n", bname, hamtID, bw) + + if _, found := reports[hamtID]; !found { + reports[hamtID] = &HAMTReport{ + ID: hamtID, + Benchmarks: make(map[string]Benchmark), + } + } + if _, found := reports[hamtID].Benchmarks[bname]; !found { + reports[hamtID].Benchmarks[bname] = Benchmark{ + Name: bname, + Metrics: make(map[string]struct{}), + Sweep: make(map[int]map[string]float64), + } + } + if _, found := reports[hamtID].Benchmarks[bname].Sweep[bw]; !found { + reports[hamtID].Benchmarks[bname].Sweep[bw] = make(map[string]float64) + } + reports[hamtID].Benchmarks[bname].Metrics[table.Metric] = struct{}{} + reports[hamtID].Benchmarks[bname].Sweep[bw][table.Metric] = row.Metrics[0].Mean + } + } + return reports, nil +} + +const GasCostGet = 114617 +const GasCostPut = 353640 +const GasCostPerBytePut = 1300 + +const EffectiveTimeGet = 11462 +const EffectiveTimePut = 35364 + +const getsKey = "getEvts" +const putsKey = "putsEvts" +const bytesPutKey = "bytesPut" +const timeKey = "time/op" + +// Add GasCost, Time/1000, and EffectiveRuntime +func processReports(reports map[string]*HAMTReport) error { + for _, report := range reports { + for _, bench := range report.Benchmarks { + bench.Metrics["gasCost"] = struct{}{} + bench.Metrics["time/1000"] = struct{}{} + bench.Metrics["effectiveRuntime"] = struct{}{} + + for _, row := range bench.Sweep { + row["gasCost"] = row[getsKey]*GasCostGet + row[putsKey]*GasCostPut + row[bytesPutKey]*GasCostPerBytePut + row["time/1000"] = row[timeKey] / float64(1000) + row["effectiveRuntime"] = row["time/1000"] + row[getsKey]*EffectiveTimeGet + row[putsKey]*EffectiveTimePut + } + } + } + return nil +} + +// Unedited hamt benchmark strings are ugly but consistent +// BenchmarkName/hamtID_--_bw=Bitwidth-8 +// This can be simplified once we clean up generation +func parseBench(bstatBench string) (bool, string, string, int) { + slashSeps := strings.Split(bstatBench, "/") + if len(slashSeps) != 2 { + return false, "", "", 0 + } + bname := slashSeps[0] + + uScoreSeps := strings.Split(slashSeps[1], "_") + if len(uScoreSeps) != 3 { + return false, "", "", 0 + } + hamtID := uScoreSeps[0] + + bitwidthStr := strings.TrimPrefix(uScoreSeps[2], "bw=") + bitwidthStr = strings.TrimSuffix(bitwidthStr, "-8") + bw, err := strconv.Atoi(bitwidthStr) + if err != nil { + panic(err) + } + return true, bname, hamtID, bw +} diff --git a/go.mod b/go.mod index 1cf7aa0..9136fba 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.6.1 github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488 + golang.org/x/perf v0.0.0-20200918155509-d949658356f9 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 ) diff --git a/go.sum b/go.sum index 4132630..1c21099 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/googleapis/gax-go v0.0.0-20161107002406-da06d194a00e/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyFSs7UnsU= @@ -22,6 +23,7 @@ github.com/ipfs/go-ipld-format v0.0.1 h1:HCu4eB/Gh+KD/Q0M8u888RFkorTWNIL3da4oc5d github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/mattn/go-sqlite3 v0.0.0-20161215041557-2d44decb4941/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16 h1:5W7KhL8HVF3XCFOweFD3BNESdnO8ewyYTFT2R+/b8FQ= @@ -75,8 +77,14 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20170207211851-4464e7848382/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/perf v0.0.0-20200918155509-d949658356f9 h1:yVBHF5pcQLKR9B+y+dOJ6y68nqJBDWaZ9DhB1Ohg0qE= +golang.org/x/perf v0.0.0-20200918155509-d949658356f9/go.mod h1:FrqOtQDO3iMDVUtw5nNTDFpR1HUCGh00M3kj2wiSzLQ= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d h1:Z0Ahzd7HltpJtjAHHxX8QFP3j1yYgiuvjbjRzDj/KH0= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/hamt_bench_test.go b/hamt_bench_test.go index 9c4ffbc..af948ac 100644 --- a/hamt_bench_test.go +++ b/hamt_bench_test.go @@ -92,42 +92,42 @@ func init() { datasize: 3, keysize: 26, }, - // hamtParams{ - // id: "market.PendingProposals", - // count: 40713, - // datasize: 151, - // keysize: 38, - // }, - // hamtParams{ - // id: "market.EscrowWTable", - // count: 2113, - // datasize: 7, - // keysize: 4, - // }, - // hamtParams{ - // id: "market.LockedTable", - // count: 2098, - // datasize: 4, - // keysize: 4, - // }, - // hamtParams{ - // id: "market.DealOpsByEpoch", - // count: 16558, - // datasize: 43, - // keysize: 3, - // }, - // hamtParams{ - // id: "power.CronEventQueue", - // count: 60, - // datasize: 43, - // keysize: 3, - // }, - // hamtParams{ - // id: "power.CLaims", - // count: 15610, - // datasize: 5, - // keysize: 3, - // }, + hamtParams{ + id: "market.PendingProposals", + count: 40713, + datasize: 151, + keysize: 38, + }, + hamtParams{ + id: "market.EscrowWTable", + count: 2113, + datasize: 7, + keysize: 4, + }, + hamtParams{ + id: "market.LockedTable", + count: 2098, + datasize: 4, + keysize: 4, + }, + hamtParams{ + id: "market.DealOpsByEpoch", + count: 16558, + datasize: 43, + keysize: 3, + }, + hamtParams{ + id: "power.CronEventQueue", + count: 60, + datasize: 43, + keysize: 3, + }, + hamtParams{ + id: "power.CLaims", + count: 15610, + datasize: 5, + keysize: 3, + }, } // bucketsize-aka-arraywidth? maybe someday. @@ -193,9 +193,9 @@ func BenchmarkFill(b *testing.B) { if blockstore.stats.evtcntPutDup > 0 { b.Logf("on round N=%d: blockstore stats: %#v\n", b.N, blockstore.stats) // note: must refer to this before doing `n.checkSize`; that function has many effects. } - b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(t.count), "getEvts/entry") - b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(t.count), "putEvts/entry") - b.ReportMetric(float64(len(blockstore.data))/float64(t.count), "blocks/entry") + b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(t.count), "getEvts") + b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(t.count), "putEvts") + b.ReportMetric(float64(len(blockstore.data))/float64(t.count), "blocks") binarySize, _ := n.checkSize(context.Background()) b.ReportMetric(float64(binarySize)/float64(t.count), "bytes(hamtAccnt)/entry") b.ReportMetric(float64(blockstore.totalBlockSizes())/float64(t.count), "bytes(blockstoreAccnt)/entry") @@ -268,9 +268,9 @@ func doBenchmarkSetSuite(b *testing.B, flushPer bool) { if blockstore.stats.evtcntPutDup > 0 { b.Logf("on round N=%d: blockstore stats: %#v\n", b.N, blockstore.stats) } - b.ReportMetric(float64(blockstore.stats.evtcntGet)/1000, "getEvts/set") - b.ReportMetric(float64(blockstore.stats.evtcntPut)/1000, "putEvts/set") - b.ReportMetric(float64(blockstore.stats.bytesPut)/1000, "bytesPut/set") + b.ReportMetric(float64(blockstore.stats.evtcntGet)/1000, "getEvts") + b.ReportMetric(float64(blockstore.stats.evtcntPut)/1000, "putEvts") + b.ReportMetric(float64(blockstore.stats.bytesPut)/1000, "bytesPut") b.StartTimer() } }) @@ -325,8 +325,8 @@ func doBenchmarkEntriesCount(num int, bitWidth int, datasize int, keysize int) f } } b.StopTimer() - b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(1000), "getEvts/find") - b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(1000), "putEvts/find") // surely this is zero, but for completeness. + b.ReportMetric(float64(blockstore.stats.evtcntGet)/float64(1000), "getEvts") + b.ReportMetric(float64(blockstore.stats.evtcntPut)/float64(1000), "putEvts") // surely this is zero, but for completeness. b.StartTimer() } } @@ -383,9 +383,9 @@ func doBenchmarkResetSuite(num int, bitWidth int, datasize int, keysize int) fun } } b.StopTimer() - b.ReportMetric(float64(blockstore.stats.evtcntGet)/1000, "getEvts/set") - b.ReportMetric(float64(blockstore.stats.evtcntPut)/1000, "putEvts/set") - b.ReportMetric(float64(blockstore.stats.bytesPut)/1000, "bytesPut/set") + b.ReportMetric(float64(blockstore.stats.evtcntGet)/1000, "getEvts") + b.ReportMetric(float64(blockstore.stats.evtcntPut)/1000, "putEvts") + b.ReportMetric(float64(blockstore.stats.bytesPut)/1000, "bytesPut") b.StartTimer() } } diff --git a/hamt_test.go b/hamt_test.go index 18765be..7572837 100644 --- a/hamt_test.go +++ b/hamt_test.go @@ -148,11 +148,13 @@ var shortIdentityHash = func(k []byte) []byte { } func TestCanonicalStructure(t *testing.T) { + t.Skip() addAndRemoveKeys(t, []string{"K"}, []string{"B"}, UseHashFunction(identityHash)) addAndRemoveKeys(t, []string{"K0", "K1", "KAA1", "KAA2", "KAA3"}, []string{"KAA4"}) } func TestCanonicalStructureAlternateBitWidth(t *testing.T) { + t.Skip() addAndRemoveKeys(t, []string{"K"}, []string{"B"}, UseTreeBitWidth(7), UseHashFunction(identityHash)) addAndRemoveKeys(t, []string{"K0", "K1", "KAA1", "KAA2", "KAA3"}, []string{"KAA4"}, UseTreeBitWidth(7), UseHashFunction(identityHash)) addAndRemoveKeys(t, []string{"K"}, []string{"B"}, UseTreeBitWidth(6), UseHashFunction(identityHash)) @@ -162,6 +164,7 @@ func TestCanonicalStructureAlternateBitWidth(t *testing.T) { } func TestOverflow(t *testing.T) { + t.Skip() keys := make([]string, 4) for i := range keys { keys[i] = strings.Repeat("A", 32) + fmt.Sprintf("%d", i) @@ -193,6 +196,7 @@ func TestOverflow(t *testing.T) { } func TestFillAndCollapse(t *testing.T) { + t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) root := NewNode(cs, UseTreeBitWidth(8), UseHashFunction(identityHash)) @@ -524,6 +528,7 @@ func statsrec(n *Node, st *hamtStats) { } func TestHash(t *testing.T) { + t.Skip() h1 := defaultHashFunction([]byte("abcd")) h2 := defaultHashFunction([]byte("abce")) if h1[0] == h2[0] && h1[1] == h2[1] && h1[3] == h2[3] { @@ -532,10 +537,12 @@ func TestHash(t *testing.T) { } func TestBasic(t *testing.T) { + t.Skip() testBasic(t) } func TestSha256(t *testing.T) { + t.Skip() testBasic(t, UseHashFunction(func(in []byte) []byte { out := sha256.Sum256(in) return out[:] @@ -582,6 +589,7 @@ func testBasic(t *testing.T, options ...Option) { } func TestDelete(t *testing.T) { + t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) begn := NewNode(cs) @@ -621,6 +629,7 @@ func TestDelete(t *testing.T) { } func TestSetGet(t *testing.T) { + t.Skip() ctx := context.Background() vals := make(map[string][]byte) var keys []string @@ -744,6 +753,7 @@ func nodesEqual(t *testing.T, store cbor.IpldStore, n1, n2 *Node) bool { } func TestReloadEmpty(t *testing.T) { + t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) @@ -764,6 +774,7 @@ func TestReloadEmpty(t *testing.T) { } func TestCopy(t *testing.T) { + t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) @@ -788,6 +799,7 @@ func TestCopy(t *testing.T) { } func TestCopyCopiesNilSlices(t *testing.T) { + t.Skip() cs := cbor.NewCborStore(newMockBlocks()) n := NewNode(cs) @@ -806,6 +818,7 @@ func TestCopyCopiesNilSlices(t *testing.T) { } func TestCopyWithoutFlush(t *testing.T) { + t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) @@ -843,6 +856,7 @@ func TestCopyWithoutFlush(t *testing.T) { } func TestValueLinking(t *testing.T) { + t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) @@ -884,6 +898,7 @@ func TestValueLinking(t *testing.T) { } func TestSetNilValues(t *testing.T) { + t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) @@ -933,6 +948,7 @@ func TestSetNilValues(t *testing.T) { // nodes to test whether the implementation will reject malformed encoded nodes // on load. func TestMalformedHamt(t *testing.T) { + t.Skip() ctx := context.Background() blocks := newMockBlocks() cs := cbor.NewCborStore(blocks) From 8a40499f3757912692c1ff3d7a90b2bb2908c41c Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Tue, 24 Nov 2020 08:36:44 -0500 Subject: [PATCH 06/11] Benchparse output undo skips --- benchparse/main.go | 6 +++--- hamt_test.go | 16 ---------------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/benchparse/main.go b/benchparse/main.go index 551040a..95cb0b7 100644 --- a/benchparse/main.go +++ b/benchparse/main.go @@ -13,8 +13,8 @@ import ( ) func main() { - if len(os.Args) < 2 { - panic("need to specify input file") + if len(os.Args) < 3 { + panic("need to specify input and output file") } fIn, err := os.Open(os.Args[1]) if err != nil { @@ -35,7 +35,7 @@ func main() { panic(err) } - fOut, err := os.Create("out.csv") + fOut, err := os.Create(os.Args[2]) if err != nil { panic(err) } diff --git a/hamt_test.go b/hamt_test.go index 7572837..18765be 100644 --- a/hamt_test.go +++ b/hamt_test.go @@ -148,13 +148,11 @@ var shortIdentityHash = func(k []byte) []byte { } func TestCanonicalStructure(t *testing.T) { - t.Skip() addAndRemoveKeys(t, []string{"K"}, []string{"B"}, UseHashFunction(identityHash)) addAndRemoveKeys(t, []string{"K0", "K1", "KAA1", "KAA2", "KAA3"}, []string{"KAA4"}) } func TestCanonicalStructureAlternateBitWidth(t *testing.T) { - t.Skip() addAndRemoveKeys(t, []string{"K"}, []string{"B"}, UseTreeBitWidth(7), UseHashFunction(identityHash)) addAndRemoveKeys(t, []string{"K0", "K1", "KAA1", "KAA2", "KAA3"}, []string{"KAA4"}, UseTreeBitWidth(7), UseHashFunction(identityHash)) addAndRemoveKeys(t, []string{"K"}, []string{"B"}, UseTreeBitWidth(6), UseHashFunction(identityHash)) @@ -164,7 +162,6 @@ func TestCanonicalStructureAlternateBitWidth(t *testing.T) { } func TestOverflow(t *testing.T) { - t.Skip() keys := make([]string, 4) for i := range keys { keys[i] = strings.Repeat("A", 32) + fmt.Sprintf("%d", i) @@ -196,7 +193,6 @@ func TestOverflow(t *testing.T) { } func TestFillAndCollapse(t *testing.T) { - t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) root := NewNode(cs, UseTreeBitWidth(8), UseHashFunction(identityHash)) @@ -528,7 +524,6 @@ func statsrec(n *Node, st *hamtStats) { } func TestHash(t *testing.T) { - t.Skip() h1 := defaultHashFunction([]byte("abcd")) h2 := defaultHashFunction([]byte("abce")) if h1[0] == h2[0] && h1[1] == h2[1] && h1[3] == h2[3] { @@ -537,12 +532,10 @@ func TestHash(t *testing.T) { } func TestBasic(t *testing.T) { - t.Skip() testBasic(t) } func TestSha256(t *testing.T) { - t.Skip() testBasic(t, UseHashFunction(func(in []byte) []byte { out := sha256.Sum256(in) return out[:] @@ -589,7 +582,6 @@ func testBasic(t *testing.T, options ...Option) { } func TestDelete(t *testing.T) { - t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) begn := NewNode(cs) @@ -629,7 +621,6 @@ func TestDelete(t *testing.T) { } func TestSetGet(t *testing.T) { - t.Skip() ctx := context.Background() vals := make(map[string][]byte) var keys []string @@ -753,7 +744,6 @@ func nodesEqual(t *testing.T, store cbor.IpldStore, n1, n2 *Node) bool { } func TestReloadEmpty(t *testing.T) { - t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) @@ -774,7 +764,6 @@ func TestReloadEmpty(t *testing.T) { } func TestCopy(t *testing.T) { - t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) @@ -799,7 +788,6 @@ func TestCopy(t *testing.T) { } func TestCopyCopiesNilSlices(t *testing.T) { - t.Skip() cs := cbor.NewCborStore(newMockBlocks()) n := NewNode(cs) @@ -818,7 +806,6 @@ func TestCopyCopiesNilSlices(t *testing.T) { } func TestCopyWithoutFlush(t *testing.T) { - t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) @@ -856,7 +843,6 @@ func TestCopyWithoutFlush(t *testing.T) { } func TestValueLinking(t *testing.T) { - t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) @@ -898,7 +884,6 @@ func TestValueLinking(t *testing.T) { } func TestSetNilValues(t *testing.T) { - t.Skip() ctx := context.Background() cs := cbor.NewCborStore(newMockBlocks()) @@ -948,7 +933,6 @@ func TestSetNilValues(t *testing.T) { // nodes to test whether the implementation will reject malformed encoded nodes // on load. func TestMalformedHamt(t *testing.T) { - t.Skip() ctx := context.Background() blocks := newMockBlocks() cs := cbor.NewCborStore(blocks) From 3b66b9d69ae44f3b66c1b939b32591d2ac148faf Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Tue, 24 Nov 2020 11:15:45 -0500 Subject: [PATCH 07/11] Add benchparse README --- benchparse/README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 benchparse/README.md diff --git a/benchparse/README.md b/benchparse/README.md new file mode 100644 index 0000000..6ec79fb --- /dev/null +++ b/benchparse/README.md @@ -0,0 +1,24 @@ +# Benchparse + +`benchparse` is a lightweight tool for processing the raw output of golang benchmark runs into useful csvs. + +The expected workflow: +```sh +go test -bench=. -count=2 -run=^a > all-benchmark-outputs.out +benchparse all-benchmark-outputs.out out.csv +``` + +Three derivative values useful for analysis are generated from the benchmark raw outputs: gas cost, effective runtime and time / 1000. Gas cost is derived from constants kept in this package reflecting current network gas pricing. time/1000 gives the benchmark runtime of one operation since each benchmark value is taken from 1000 iterations of a benchmark. Effective runtime is the sum of one operation's benchmark runtime and the expected runtime of the gets and puts geneated by that operation when persisting to disk. Get and put runtime estimations are just gas cost / 10 since 10 gas correspond to 1 nanosecond. + +The output csv is of the form +``` +hamt-id-a +header1,header2,... +data1,data2,... + +hamt-id-b +header1,header2,... +data1,data2,... + +... +``` \ No newline at end of file From f8ef45e73836a71601308463e0cdb29ceffbad81 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Tue, 24 Nov 2020 12:35:11 -0500 Subject: [PATCH 08/11] Correctly account for puts in gas cost and eff runtime --- benchparse/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchparse/main.go b/benchparse/main.go index 95cb0b7..60dc862 100644 --- a/benchparse/main.go +++ b/benchparse/main.go @@ -185,7 +185,7 @@ const EffectiveTimeGet = 11462 const EffectiveTimePut = 35364 const getsKey = "getEvts" -const putsKey = "putsEvts" +const putsKey = "putEvts" const bytesPutKey = "bytesPut" const timeKey = "time/op" From 1c8d0008b35832d75d1096832e792c86355bbf6d Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Sun, 6 Dec 2020 17:21:49 -0500 Subject: [PATCH 09/11] Move benchparse from this branch --- benchparse/README.md | 24 ----- benchparse/main.go | 233 ------------------------------------------- go.mod | 14 ++- go.sum | 37 +++++-- 4 files changed, 41 insertions(+), 267 deletions(-) delete mode 100644 benchparse/README.md delete mode 100644 benchparse/main.go diff --git a/benchparse/README.md b/benchparse/README.md deleted file mode 100644 index 6ec79fb..0000000 --- a/benchparse/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Benchparse - -`benchparse` is a lightweight tool for processing the raw output of golang benchmark runs into useful csvs. - -The expected workflow: -```sh -go test -bench=. -count=2 -run=^a > all-benchmark-outputs.out -benchparse all-benchmark-outputs.out out.csv -``` - -Three derivative values useful for analysis are generated from the benchmark raw outputs: gas cost, effective runtime and time / 1000. Gas cost is derived from constants kept in this package reflecting current network gas pricing. time/1000 gives the benchmark runtime of one operation since each benchmark value is taken from 1000 iterations of a benchmark. Effective runtime is the sum of one operation's benchmark runtime and the expected runtime of the gets and puts geneated by that operation when persisting to disk. Get and put runtime estimations are just gas cost / 10 since 10 gas correspond to 1 nanosecond. - -The output csv is of the form -``` -hamt-id-a -header1,header2,... -data1,data2,... - -hamt-id-b -header1,header2,... -data1,data2,... - -... -``` \ No newline at end of file diff --git a/benchparse/main.go b/benchparse/main.go deleted file mode 100644 index 60dc862..0000000 --- a/benchparse/main.go +++ /dev/null @@ -1,233 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "io" - "os" - "sort" - "strconv" - "strings" - - "golang.org/x/perf/benchstat" -) - -func main() { - if len(os.Args) < 3 { - panic("need to specify input and output file") - } - fIn, err := os.Open(os.Args[1]) - if err != nil { - panic(err) - } - - c := benchstat.Collection{} - if err := c.AddFile("config1", fIn); err != nil { - panic(err) - } - tables := c.Tables() - - reports, err := genReports(tables) - if err != nil { - panic(err) - } - if err := processReports(reports); err != nil { - panic(err) - } - - fOut, err := os.Create(os.Args[2]) - if err != nil { - panic(err) - } - defer fOut.Close() - if err := writeReports(reports, fOut); err != nil { - panic(err) - } -} - -// All metrics for all benchmarks for one HAMT configuration -type HAMTReport struct { - ID string - Benchmarks map[string]Benchmark -} - -func (hr *HAMTReport) Write(datawriter *bufio.Writer) error { - // Write title - if _, err := datawriter.WriteString(hr.ID + "\n"); err != nil { - return err - } - - // Write benchmarks - for _, benchmark := range hr.Benchmarks { - if err := benchmark.Write(datawriter); err != nil { - return err - } - } - return nil -} - -type Benchmark struct { - Name string - Metrics map[string]struct{} - Sweep map[int]map[string]float64 // map[Bitwidth]MetricsMap -} - -func (b *Benchmark) Write(datawriter *bufio.Writer) error { - // Get sorted keys - metrics := make([]string, 0) - for metric := range b.Metrics { - metrics = append(metrics, metric) - } - sort.Strings(metrics) - - bws := make([]int, 0) - for bw := range b.Sweep { - bws = append(bws, bw) - } - sort.Ints(bws) - - // Write headers - headers := []string{"Benchmark", "Bitwidth"} - headers = append(headers, metrics...) - if _, err := datawriter.WriteString(strings.Join(headers, ",")); err != nil { - return err - } - if _, err := datawriter.WriteString("\n"); err != nil { - return err - } - - // Write sweeps - for _, bw := range bws { - if _, err := datawriter.WriteString(b.Name); err != nil { - return err - } - if _, err := datawriter.WriteString("," + strconv.Itoa(bw)); err != nil { - return err - } - for _, metric := range metrics { - m, found := b.Sweep[bw][metric] - if !found { - // empty column - if _, err := datawriter.WriteString(","); err != nil { - return err - } - continue - } - if _, err := datawriter.WriteString(fmt.Sprintf(",%.3f", m)); err != nil { - return err - } - } - if _, err := datawriter.WriteString("\n"); err != nil { - return err - } - } - - return nil -} - -func writeReports(reports map[string]*HAMTReport, w io.Writer) error { - datawriter := bufio.NewWriter(w) - for _, report := range reports { - if err := report.Write(datawriter); err != nil { - return err - } - if _, err := datawriter.WriteString("\n\n"); err != nil { - return err - } - } - return datawriter.Flush() -} - -func genReports(tables []*benchstat.Table) (map[string]*HAMTReport, error) { - reports := make(map[string]*HAMTReport) - for _, table := range tables { - for _, row := range table.Rows { - if len(row.Metrics) == 0 { - continue - } - if len(row.Metrics) >= 2 { - panic("don't know how to handle lots of metrics") - } - good, bname, hamtID, bw := parseBench(row.Benchmark) - if !good { - continue - } - fmt.Printf("bname: %s, hamtID: %s, bw: %d\n", bname, hamtID, bw) - - if _, found := reports[hamtID]; !found { - reports[hamtID] = &HAMTReport{ - ID: hamtID, - Benchmarks: make(map[string]Benchmark), - } - } - if _, found := reports[hamtID].Benchmarks[bname]; !found { - reports[hamtID].Benchmarks[bname] = Benchmark{ - Name: bname, - Metrics: make(map[string]struct{}), - Sweep: make(map[int]map[string]float64), - } - } - if _, found := reports[hamtID].Benchmarks[bname].Sweep[bw]; !found { - reports[hamtID].Benchmarks[bname].Sweep[bw] = make(map[string]float64) - } - reports[hamtID].Benchmarks[bname].Metrics[table.Metric] = struct{}{} - reports[hamtID].Benchmarks[bname].Sweep[bw][table.Metric] = row.Metrics[0].Mean - } - } - return reports, nil -} - -const GasCostGet = 114617 -const GasCostPut = 353640 -const GasCostPerBytePut = 1300 - -const EffectiveTimeGet = 11462 -const EffectiveTimePut = 35364 - -const getsKey = "getEvts" -const putsKey = "putEvts" -const bytesPutKey = "bytesPut" -const timeKey = "time/op" - -// Add GasCost, Time/1000, and EffectiveRuntime -func processReports(reports map[string]*HAMTReport) error { - for _, report := range reports { - for _, bench := range report.Benchmarks { - bench.Metrics["gasCost"] = struct{}{} - bench.Metrics["time/1000"] = struct{}{} - bench.Metrics["effectiveRuntime"] = struct{}{} - - for _, row := range bench.Sweep { - row["gasCost"] = row[getsKey]*GasCostGet + row[putsKey]*GasCostPut + row[bytesPutKey]*GasCostPerBytePut - row["time/1000"] = row[timeKey] / float64(1000) - row["effectiveRuntime"] = row["time/1000"] + row[getsKey]*EffectiveTimeGet + row[putsKey]*EffectiveTimePut - } - } - } - return nil -} - -// Unedited hamt benchmark strings are ugly but consistent -// BenchmarkName/hamtID_--_bw=Bitwidth-8 -// This can be simplified once we clean up generation -func parseBench(bstatBench string) (bool, string, string, int) { - slashSeps := strings.Split(bstatBench, "/") - if len(slashSeps) != 2 { - return false, "", "", 0 - } - bname := slashSeps[0] - - uScoreSeps := strings.Split(slashSeps[1], "_") - if len(uScoreSeps) != 3 { - return false, "", "", 0 - } - hamtID := uScoreSeps[0] - - bitwidthStr := strings.TrimPrefix(uScoreSeps[2], "bw=") - bitwidthStr = strings.TrimSuffix(bitwidthStr, "-8") - bw, err := strconv.Atoi(bitwidthStr) - if err != nil { - panic(err) - } - return true, bname, hamtID, bw -} diff --git a/go.mod b/go.mod index 9136fba..9ef94aa 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,26 @@ module github.com/filecoin-project/go-hamt-ipld/v3 require ( + github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect github.com/ipfs/go-block-format v0.0.2 - github.com/ipfs/go-cid v0.0.6 + github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-ipld-cbor v0.0.4 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.6.1 github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488 golang.org/x/perf v0.0.0-20200918155509-d949658356f9 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 + github.com/ipfs/go-ipld-format v0.0.2 // indirect + github.com/minio/sha256-simd v0.1.1 // indirect + github.com/multiformats/go-multihash v0.0.14 // indirect + github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a // indirect + github.com/smartystreets/assertions v1.0.1 // indirect + github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect + github.com/spaolacci/murmur3 v1.1.0 + github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 // indirect + github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c + golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 ) go 1.13 diff --git a/go.sum b/go.sum index 1c21099..838158c 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,10 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/googleapis/gax-go v0.0.0-20161107002406-da06d194a00e/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f h1:KMlcu9X58lhTA/KrfX8Bi1LQSO4pzoVjTiL3h4Jk+Zk= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyFSs7UnsU= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJu4ZbSc= @@ -15,6 +16,8 @@ github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.6 h1:go0y+GcDOGeJIV01FeBsta4FHngoA4Wz7KMeLkXAhMs= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= +github.com/ipfs/go-cid v0.0.7 h1:ysQJVJA3fNDF1qigJbsSQOdjhVLsOEoPdh0+R97k3jY= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipld-cbor v0.0.4 h1:Aw3KPOKXjvrm6VjwJvFf1F1ekR/BH3jdof3Bk7OTiSA= @@ -23,13 +26,16 @@ github.com/ipfs/go-ipld-format v0.0.1 h1:HCu4eB/Gh+KD/Q0M8u888RFkorTWNIL3da4oc5d github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/mattn/go-sqlite3 v0.0.0-20161215041557-2d44decb4941/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16 h1:5W7KhL8HVF3XCFOweFD3BNESdnO8ewyYTFT2R+/b8FQ= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 h1:MHkK1uRtFbVqvAgvWxafZe54+5uBxLluGylDiKgdhwo= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mr-tron/base58 v1.1.0 h1:Y51FGVJ91WBqCEabAi5OPUz38eAx8DakuAm5svLcsfQ= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= @@ -50,16 +56,24 @@ github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6w github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I= +github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-varint v0.0.5 h1:XVZwSo04Cs3j/jS0uAEPpT3JY6DzMcVLLoWOSnCxOjg= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992 h1:bzMe+2coZJYHnhGgVlcQKuRy4FSny4ds8dLQjw5P1XE= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr2us7vdy04YWz3LVAirzP7reh8+M= +github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= +github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= @@ -68,32 +82,37 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= +github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 h1:WXhVOwj2USAXB5oMDwRl3piOux2XMV9TANaYxXHdkoE= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= -github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488 h1:P/Q9QT99FpyHtFke7ERUqX7yYtZ/KigO880L+TKFyTQ= -github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c h1:otRnI08JoahNBxUFqX3372Ab9GnTj8L5J9iP5ImyxGU= +github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/oauth2 v0.0.0-20170207211851-4464e7848382/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/perf v0.0.0-20200918155509-d949658356f9 h1:yVBHF5pcQLKR9B+y+dOJ6y68nqJBDWaZ9DhB1Ohg0qE= -golang.org/x/perf v0.0.0-20200918155509-d949658356f9/go.mod h1:FrqOtQDO3iMDVUtw5nNTDFpR1HUCGh00M3kj2wiSzLQ= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190219092855-153ac476189d h1:Z0Ahzd7HltpJtjAHHxX8QFP3j1yYgiuvjbjRzDj/KH0= golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +<<<<<<< HEAD gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +======= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +>>>>>>> Move benchparse from this branch From a4e9e845ca26a7615eb406a050fa4972e55c5c42 Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Sun, 6 Dec 2020 17:43:35 -0500 Subject: [PATCH 10/11] Post rebase cleanup --- go.mod | 6 +----- go.sum | 10 +++++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 9ef94aa..58bb25f 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,6 @@ require ( github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-cid v0.0.7 github.com/ipfs/go-ipld-cbor v0.0.4 - github.com/spaolacci/murmur3 v1.1.0 - github.com/stretchr/testify v1.6.1 - github.com/whyrusleeping/cbor-gen v0.0.0-20200806213330-63aa96ca5488 - golang.org/x/perf v0.0.0-20200918155509-d949658356f9 - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 github.com/ipfs/go-ipld-format v0.0.2 // indirect github.com/minio/sha256-simd v0.1.1 // indirect github.com/multiformats/go-multihash v0.0.14 // indirect @@ -17,6 +12,7 @@ require ( github.com/smartystreets/assertions v1.0.1 // indirect github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect github.com/spaolacci/murmur3 v1.1.0 + github.com/stretchr/testify v1.6.1 github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20200812213548-958ddffe352c golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect diff --git a/go.sum b/go.sum index 838158c..4e16286 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmv github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.6 h1:go0y+GcDOGeJIV01FeBsta4FHngoA4Wz7KMeLkXAhMs= @@ -24,6 +25,8 @@ github.com/ipfs/go-ipld-cbor v0.0.4 h1:Aw3KPOKXjvrm6VjwJvFf1F1ekR/BH3jdof3Bk7OTi github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-format v0.0.1 h1:HCu4eB/Gh+KD/Q0M8u888RFkorTWNIL3da4oc5dwc80= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= +github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs= +github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -107,12 +110,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -<<<<<<< HEAD +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -======= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= ->>>>>>> Move benchparse from this branch From abc9af6cd078bfd7a7cf45a707592db60feb8abd Mon Sep 17 00:00:00 2001 From: ZenGround0 Date: Mon, 7 Dec 2020 13:18:10 -0500 Subject: [PATCH 11/11] Entropy management --- hamt_bench_test.go | 74 ++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 42 deletions(-) diff --git a/hamt_bench_test.go b/hamt_bench_test.go index af948ac..7ef4d17 100644 --- a/hamt_bench_test.go +++ b/hamt_bench_test.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "math/rand" + "os" "runtime" "testing" @@ -17,23 +18,36 @@ type rander struct { func (r *rander) randString(stringSize int) string { buf := make([]byte, stringSize) - rand.Read(buf) + r.r.Read(buf) return hex.EncodeToString(buf) } func (r *rander) randValue(datasize int) []byte { buf := make([]byte, datasize) - rand.Read(buf) + r.r.Read(buf) return buf } func (r *rander) selectKey(keys []string) string { - i := rand.Int() % len(keys) + i := r.r.Int() % len(keys) return keys[i] } +var benchSeed = int64(42) + +func TestMain(m *testing.M) { + // Hack to run multiple different benchmark seed values + for benchMarkIter := int64(0); benchMarkIter < 3; benchMarkIter++ { + benchSeed = 42 + (100_000_000_000 * benchMarkIter) // We resample on every b.N so choose step size bigger than b.N values + if code := m.Run(); code != 0 { + os.Exit(code) + } + } + os.Exit(0) +} + func BenchmarkSerializeNode(b *testing.B) { - r := rander{rand.New(rand.NewSource(1234))} + r := rander{rand.New(rand.NewSource(benchSeed + 1234))} cs := cbor.NewCborStore(newMockBlocks()) n := NewNode(cs) @@ -87,46 +101,22 @@ func init() { hamts := []hamtParams{ hamtParams{ - id: "init.AddressMap", - count: 55649, + id: "init.AddressMap.Now", + count: 100000, datasize: 3, keysize: 26, }, hamtParams{ - id: "market.PendingProposals", - count: 40713, - datasize: 151, - keysize: 38, - }, - hamtParams{ - id: "market.EscrowWTable", - count: 2113, - datasize: 7, - keysize: 4, - }, - hamtParams{ - id: "market.LockedTable", - count: 2098, - datasize: 4, - keysize: 4, - }, - hamtParams{ - id: "market.DealOpsByEpoch", - count: 16558, - datasize: 43, - keysize: 3, - }, - hamtParams{ - id: "power.CronEventQueue", - count: 60, - datasize: 43, - keysize: 3, + id: "init.AddressMap.ThreeMonths", + count: 167000, + datasize: 3, + keysize: 26, }, hamtParams{ - id: "power.CLaims", - count: 15610, - datasize: 5, - keysize: 3, + id: "init.AddressMap.SixMonths", + count: 240000, + datasize: 3, + keysize: 26, }, } @@ -174,7 +164,7 @@ func BenchmarkFill(b *testing.B) { for _, t := range caseTable { b.Run(fmt.Sprintf("%s", t.id), func(b *testing.B) { for i := 0; i < b.N; i++ { - r := rander{rand.New(rand.NewSource(int64(i)))} + r := rander{rand.New(rand.NewSource(benchSeed + int64(i)))} blockstore := newMockBlocks() n := NewNode(cbor.NewCborStore(blockstore), UseTreeBitWidth(t.bitwidth)) //b.ResetTimer() @@ -229,7 +219,7 @@ func doBenchmarkSetSuite(b *testing.B, flushPer bool) { b.Run(fmt.Sprintf("%s", t.id), func(b *testing.B) { for i := 0; i < b.N; i++ { b.StopTimer() - r := rander{rand.New(rand.NewSource(int64(i)))} + r := rander{rand.New(rand.NewSource(benchSeed + int64(i)))} blockstore := newMockBlocks() n := NewNode(cbor.NewCborStore(blockstore), UseTreeBitWidth(t.bitwidth)) // Initial fill: @@ -285,7 +275,7 @@ func BenchmarkFind(b *testing.B) { } func doBenchmarkEntriesCount(num int, bitWidth int, datasize int, keysize int) func(b *testing.B) { - r := rander{rand.New(rand.NewSource(int64(num)))} + r := rander{rand.New(rand.NewSource(benchSeed + int64(num)))} return func(b *testing.B) { blockstore := newMockBlocks() cs := cbor.NewCborStore(blockstore) @@ -340,7 +330,7 @@ func BenchmarkReset(b *testing.B) { } func doBenchmarkResetSuite(num int, bitWidth int, datasize int, keysize int) func(b *testing.B) { - r := rander{rand.New(rand.NewSource(int64(num)))} + r := rander{rand.New(rand.NewSource(benchSeed + int64(num)))} return func(b *testing.B) { blockstore := newMockBlocks() cs := cbor.NewCborStore(blockstore)