diff --git a/common/etl/collector.go b/common/etl/collector.go index 48beb9c3f..2ee8f795c 100644 --- a/common/etl/collector.go +++ b/common/etl/collector.go @@ -132,16 +132,10 @@ func loadFilesIntoBucket(db ethdb.Database, bucket []byte, providers []dataProvi return err } } - var currentKeyStr string - if len(k) < 4 { - currentKeyStr = fmt.Sprintf("%x", k) - } else { - currentKeyStr = fmt.Sprintf("%x...", k[:4]) - } batchCh <- struct { db ethdb.DbWithPendingMutations currentKey string - }{db: batch, currentKey: currentKeyStr} + }{db: batch, currentKey: makeCurrentKeyStr(k)} batch = getBatch() } return nil @@ -173,9 +167,12 @@ func loadFilesIntoBucket(db ethdb.Database, bucket []byte, providers []dataProvi return err } } - _, err := batch.Commit() + batchCh <- struct { + db ethdb.DbWithPendingMutations + currentKey string + }{db: batch, currentKey: makeCurrentKeyStr(nil)} - return err + return nil }) wg.Go(func() error { commit := func(batchToCommit struct { @@ -218,3 +215,15 @@ func loadFilesIntoBucket(db ethdb.Database, bucket []byte, providers []dataProvi return wg.Wait() } + +func makeCurrentKeyStr(k []byte) string { + var currentKeyStr string + if k == nil { + currentKeyStr = "final" + } else if len(k) < 4 { + currentKeyStr = fmt.Sprintf("%x", k) + } else { + currentKeyStr = fmt.Sprintf("%x...", k[:4]) + } + return currentKeyStr +} diff --git a/common/etl/dataprovider.go b/common/etl/dataprovider.go index daaa5883c..8fb5f45d7 100644 --- a/common/etl/dataprovider.go +++ b/common/etl/dataprovider.go @@ -50,17 +50,9 @@ func FlushToDisk(encoder Encoder, currentKey []byte, b Buffer, datadir string) ( } var m runtime.MemStats runtime.ReadMemStats(&m) - var currentKeyStr string - if currentKey == nil { - currentKeyStr = "final" - } else if len(currentKey) < 4 { - currentKeyStr = fmt.Sprintf("%x", currentKey) - } else { - currentKeyStr = fmt.Sprintf("%x...", currentKey[:4]) - } log.Info( "Flushed buffer file", - "current key", currentKeyStr, + "current key", makeCurrentKeyStr(currentKey), "name", bufferFile.Name(), "alloc", common.StorageSize(m.Alloc), "sys", common.StorageSize(m.Sys), "numGC", int(m.NumGC)) b.Reset() diff --git a/common/etl/etl_test.go b/common/etl/etl_test.go index 099b490f2..144e0e902 100644 --- a/common/etl/etl_test.go +++ b/common/etl/etl_test.go @@ -176,7 +176,6 @@ func TestTransformOnLoadCommitCustomBatchSize(t *testing.T) { }, ) assert.Nil(t, err) - fmt.Println(numberOfCalls) compareBuckets(t, db, sourceBucket, destBucket, nil) assert.Equal(t, 21, numberOfCalls)