Merge pull request #4153 from Algo-devops-service/relbeta3.8.0

Author: algojohnlee

.circleci/config.yml

@@ -217,10 +217,12 @@ commands:
       - restore_libsodium
       - restore_cache:
           keys:
-            - 'go-mod-1-14-7-v2-{{ arch }}-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}'
+            - 'go-mod-1.17.9-v3-{{ arch }}-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}'
       - restore_cache:
           keys:
-            - 'go-cache-v2-{{ .Environment.CIRCLE_STAGE }}-'
+            - 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}'
+            - 'go-cache-v3-{{ arch }}-{{ .Branch }}-'
+            - 'go-cache-v3-{{ arch }}-'
       - run:
           name: scripts/travis/build.sh --make_debug
           command: |
@@ -233,11 +235,11 @@ commands:
             scripts/travis/build.sh --make_debug
       - cache_libsodium
       - save_cache:
-          key: 'go-mod-1-14-7-v2-{{ arch }}-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}'
+          key: 'go-mod-1.17.9-v3-{{ arch }}-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}'
           paths:
             - << parameters.build_dir >>/go/pkg/mod
       - save_cache:
-          key: 'go-cache-v2-{{ .Environment.CIRCLE_STAGE }}-{{ .Environment.CIRCLE_BUILD_NUM }}'
+          key: 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}'
           paths:
             - tmp/go-cache
       - persist_to_workspace:
@@ -257,7 +259,7 @@ commands:
             mkdir -p tmp
             find crypto/libsodium-fork -type f -exec openssl md5 "{}" + > tmp/libsodium.md5
       - save_cache:
-          key: 'libsodium-fork-v2-{{ .Environment.CIRCLE_STAGE }}-{{ checksum "tmp/libsodium.md5" }}'
+          key: 'libsodium-fork-v2-{{ arch }}-{{ checksum "tmp/libsodium.md5" }}'
           paths:
             - crypto/libs
 
@@ -271,7 +273,7 @@ commands:
             find crypto/libsodium-fork -type f -exec openssl md5 "{}" + > tmp/libsodium.md5
       - restore_cache:
           keys:
-            - 'libsodium-fork-v2-{{ .Environment.CIRCLE_STAGE }}-{{ checksum "tmp/libsodium.md5" }}'
+            - 'libsodium-fork-v2-{{ arch }}-{{ checksum "tmp/libsodium.md5" }}'
 
   generic_test:
     description: Run build tests from build workspace, for re-use by diferent architectures
@@ -301,7 +303,9 @@ commands:
           touch << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json
       - restore_cache:
           keys:
-            - 'go-cache-v2-{{ .Environment.CIRCLE_STAGE }}-'
+            - 'go-cache-v3-{{ arch }}-{{ .Branch }}-{{ .Revision }}'
+            - 'go-cache-v3-{{ arch }}-{{ .Branch }}-'
+            - 'go-cache-v3-{{ arch }}-'
       - run:
           name: Run build tests
           no_output_timeout: << parameters.no_output_timeout >>
@@ -333,10 +337,6 @@ commands:
           root: << parameters.result_path >>
           paths:
             - << parameters.result_subdir >>
-      - save_cache:
-          key: 'go-cache-v2-{{ .Environment.CIRCLE_STAGE }}-{{ .Environment.CIRCLE_BUILD_NUM }}'
-          paths:
-            - tmp/go-cache
 
   upload_coverage:
     description: Collect coverage reports and upload them

buildnumber.dat

@@ -1 +1 @@
-2
+0

cmd/buildtools/genesis.go

@@ -152,8 +152,7 @@ var dumpGenesisHashCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
-		hash := crypto.HashObj(genesis)
-		fmt.Print(hash.String())
+		fmt.Print(genesis.Hash().String())
 	},
 }
 
@@ -237,7 +236,7 @@ func ensureReleaseGenesis(src bookkeeping.Genesis, releaseFile string) (err erro
 		return fmt.Errorf("error saving file: %v", err)
 	}
 
-	hash := crypto.HashObj(releaseGenesis)
+	hash := releaseGenesis.Hash()
 	err = ioutil.WriteFile(releaseFileHash, []byte(hash.String()), 0666)
 	if err != nil {
 		return fmt.Errorf("error saving hash file '%s': %v", releaseFileHash, err)
@@ -261,8 +260,20 @@ func verifyReleaseGenesis(src bookkeeping.Genesis, releaseFile string) (updateGe
 func verifyGenesisHashes(src, release bookkeeping.Genesis, hashFile string) (err error) {
 	src.Timestamp = release.Timestamp
 
-	srcHash := crypto.HashObj(src)
-	releaseHash := crypto.HashObj(release)
+	srcHash := src.Hash()
+	releaseHash := release.Hash()
+
+	srcHashCrypo := crypto.HashObj(src)
+	releaseHashCrypto := crypto.HashObj(release)
+
+	if srcHash != srcHashCrypo {
+		return fmt.Errorf("source hashes differ - genesis.json our hashing function isn't consistent")
+	}
+
+	if releaseHash != releaseHashCrypto {
+		return fmt.Errorf("release hashes differ - genesis.json our hashing function isn't consistent")
+	}
+
 	if srcHash != releaseHash {
 		return fmt.Errorf("source and release hashes differ - genesis.json may have diverge from released version")
 	}

cmd/catchpointdump/net.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -191,85 +192,126 @@ func getRemoteDataStream(url string, hint string) (result io.ReadCloser, ctxCanc
 	return
 }
 
-func downloadCatchpoint(addr string, round int) (tarName string, err error) {
+func doDownloadCatchpoint(url string, wdReader util.WatchdogStreamReader, out io.Writer) error {
+	writeChunkSize := 64 * 1024
+
+	var totalBytes int
+	tempBytes := make([]byte, writeChunkSize)
+	lastProgressUpdate := time.Now()
+	progress := -25
+	printDownloadProgressLine(progress, 50, url, 0)
+
+	for {
+		n, err := wdReader.Read(tempBytes)
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		totalBytes += n
+		_, err = out.Write(tempBytes[:n])
+		if err != nil {
+			return err
+		}
+
+		err = wdReader.Reset()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		if time.Since(lastProgressUpdate) > 50*time.Millisecond {
+			lastProgressUpdate = time.Now()
+			printDownloadProgressLine(progress, 50, url, int64(totalBytes))
+			progress++
+		}
+	}
+}
+
+// Downloads a catchpoint tar file and returns the path to the tar file.
+func downloadCatchpoint(addr string, round int) (string, error) {
 	genesisID := strings.Split(networkName, ".")[0] + "-v1.0"
 	urlTemplate := "http://" + addr + "/v1/" + genesisID + "/%s/" + strconv.FormatUint(uint64(round), 36)
 	catchpointURL := fmt.Sprintf(urlTemplate, "ledger")
 
 	catchpointStream, catchpointCtxCancel, err := getRemoteDataStream(catchpointURL, "catchpoint")
 	defer catchpointCtxCancel()
 	if err != nil {
-		return
+		return "", err
 	}
 	defer catchpointStream.Close()
 
 	dirName := "./" + strings.Split(networkName, ".")[0] + "/" + strings.Split(addr, ".")[0]
 	os.RemoveAll(dirName)
 	err = os.MkdirAll(dirName, 0777)
 	if err != nil && !os.IsExist(err) {
-		return
+		return "", err
 	}
-	tarName = dirName + "/" + strconv.FormatUint(uint64(round), 10) + ".tar"
-	file, err2 := os.Create(tarName) // will create a file with 0666 permission.
-	if err2 != nil {
-		return tarName, err2
+	tarName := dirName + "/" + strconv.FormatUint(uint64(round), 10) + ".tar"
+	file, err := os.Create(tarName) // will create a file with 0666 permission.
+	if err != nil {
+		return "", err
 	}
-	defer func() {
-		err = file.Close()
-	}()
-	writeChunkSize := 64 * 1024
+	defer file.Close()
 
-	wdReader := util.MakeWatchdogStreamReader(catchpointStream, 4096, 4096, 2*time.Second)
-	var totalBytes int
-	tempBytes := make([]byte, writeChunkSize)
-	lastProgressUpdate := time.Now()
-	progress := -25
-	printDownloadProgressLine(progress, 50, catchpointURL, 0)
-	defer printDownloadProgressLine(0, 0, catchpointURL, 0)
-	var n int
-	for {
-		n, err = wdReader.Read(tempBytes)
-		if err != nil && err != io.EOF {
-			return
-		}
-		totalBytes += n
-		writtenBytes, err2 := file.Write(tempBytes[:n])
-		if err2 != nil || n != writtenBytes {
-			return tarName, err2
-		}
+	wdReader := util.MakeWatchdogStreamReader(catchpointStream, 4096, 4096, 5*time.Second)
+	defer wdReader.Close()
 
-		err = wdReader.Reset()
-		if err != nil {
-			if err == io.EOF {
-				return tarName, nil
-			}
-			return
+	err = doDownloadCatchpoint(catchpointURL, wdReader, file)
+	if err != nil {
+		return "", err
+	}
+
+	printDownloadProgressLine(0, 0, catchpointURL, 0)
+
+	err = file.Close()
+	if err != nil {
+		return "", err
+	}
+
+	err = catchpointStream.Close()
+	if err != nil {
+		return "", err
+	}
+
+	return tarName, nil
+}
+
+func deleteLedgerFiles(deleteTracker bool) error {
+	paths := []string{
+		"./ledger.block.sqlite",
+		"./ledger.block.sqlite-shm",
+		"./ledger.block.sqlite-wal",
+	}
+	if deleteTracker {
+		trackerPaths := []string{
+			"./ledger.tracker.sqlite",
+			"./ledger.tracker.sqlite-shm",
+			"./ledger.tracker.sqlite-wal",
 		}
-		if time.Since(lastProgressUpdate) > 50*time.Millisecond {
-			lastProgressUpdate = time.Now()
-			printDownloadProgressLine(progress, 50, catchpointURL, int64(totalBytes))
-			progress++
+		paths = append(paths, trackerPaths...)
+	}
+
+	for _, path := range paths {
+		err := os.Remove(path)
+		if (err != nil) && !errors.Is(err, os.ErrNotExist) {
+			return err
 		}
 	}
+
+	return nil
 }
 
 func loadAndDump(addr string, tarFile string, genesisInitState ledgercore.InitState) error {
-	deleteLedgerFiles := func(deleteTracker bool) {
-		os.Remove("./ledger.block.sqlite")
-		os.Remove("./ledger.block.sqlite-shm")
-		os.Remove("./ledger.block.sqlite-wal")
-		if deleteTracker {
-			os.Remove("./ledger.tracker.sqlite")
-			os.Remove("./ledger.tracker.sqlite-shm")
-			os.Remove("./ledger.tracker.sqlite-wal")
-		}
-	}
 	// delete current ledger files.
 	deleteLedgerFiles(true)
 	cfg := config.GetDefaultLocal()
 	l, err := ledger.OpenLedger(logging.Base(), "./ledger", false, genesisInitState, cfg)
 	if err != nil {
 		reportErrorf("Unable to open ledger : %v", err)
+		return err
 	}
 
 	defer deleteLedgerFiles(!loadOnly)
@@ -279,6 +321,7 @@ func loadAndDump(addr string, tarFile string, genesisInitState ledgercore.InitSt
 	err = catchupAccessor.ResetStagingBalances(context.Background(), true)
 	if err != nil {
 		reportErrorf("Unable to initialize catchup database : %v", err)
+		return err
 	}
 
 	stats, err := os.Stat(tarFile)
@@ -297,6 +340,7 @@ func loadAndDump(addr string, tarFile string, genesisInitState ledgercore.InitSt
 	fileHeader, err = loadCatchpointIntoDatabase(context.Background(), catchupAccessor, reader, tarSize)
 	if err != nil {
 		reportErrorf("Unable to load catchpoint file into in-memory database : %v", err)
+		return err
 	}
 
 	if !loadOnly {