[CLI][Localnet][Bug] - [localnet_client_debug] Error 137 - Issue #607

Makefile

@@ -116,6 +116,7 @@ develop_start: ## Run all of the make commands necessary to develop on the proje
 		make go_clean_deps && \
 		make mockgen && \
 		make generate_rpc_openapi && \
+		make refresh_debug_keybase && \
 		make build
 
 .PHONY: develop_test
@@ -218,6 +219,10 @@ docker_loki_install: docker_check ## Installs the loki docker driver
 docker_loki_check:
 	if [ `docker plugin ls | grep loki: | wc -l` -eq 0 ]; then make docker_loki_install; fi
 
+.PHONY: refresh_debug_keybase
+refresh_debug_keybase: ## Refreshes the debug keybase with the latest keys from the private_keys.yaml file if necessary
+	go run build/debug_keybase/main.go ./build/localnet/manifests/private-keys.yaml ./build/debug_keybase
+
 .PHONY: clean_mocks
 clean_mocks: ## Use `clean_mocks` to delete mocks before recreating them. Also useful to cleanup code that was generated from a different branch
 	$(eval modules_dir = "shared/modules")

app/client/keybase/debug/keystore.go

@@ -4,26 +4,17 @@ package debug
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 
-	"github.com/korovkin/limiter"
+	"github.com/dgraph-io/badger/v3"
 	"github.com/pokt-network/pocket/app/client/keybase"
 	"github.com/pokt-network/pocket/build"
 	"github.com/pokt-network/pocket/logger"
-	cryptoPocket "github.com/pokt-network/pocket/shared/crypto"
-	"gopkg.in/yaml.v2"
 )
 
-const (
-	// NOTE: This is the number of validators in the private-keys.yaml manifest file
-	numValidators      = 999
-	debugKeybaseSuffix = "/.pocket/keys"
-
-	// Increasing this number is linearly proportional to the amount of RAM required for the debug client to start and import
-	// pre-generated keys into the keybase. Beware that might cause OOM and process can exit with 137 status code.
-	// 4 threads takes 350-400MiB from a few tests which sounds acceptable.
-	debugKeybaseImportConcurrencyLimit = 4
-)
+const debugKeybaseSuffix = "/.pocket/keys"
 
 var (
 	// TODO: Allow users to override this value via `datadir` flag or env var or config file
@@ -45,81 +36,17 @@ func init() {
 }
 
 func initializeDebugKeybase() error {
-	var (
-		validatorKeysPairMap map[string]string
-		err                  error
-	)
-
-	validatorKeysPairMap, err = parseValidatorPrivateKeysFromEmbeddedYaml()
-
-	if err != nil {
-		return err
-	}
-
 	// Create/Open the keybase at `$HOME/.pocket/keys`
 	kb, err := keybase.NewKeybase(debugKeybasePath)
 	if err != nil {
 		return err
 	}
 	db := kb.GetBadgerDB()
 
-	// Add the keys if the keybase contains less than 999
-	curAddr, _, err := kb.GetAll()
-	if err != nil {
+	if err := restoreBadgerDB(build.DebugKeybaseBackup, db); err != nil {
 		return err
 	}
 
-	// Add validator addresses if not present
-	if len(curAddr) < numValidators {
-		logger.Global.Debug().Msgf(fmt.Sprintf("Debug keybase initializing... Adding %d validator keys to the keybase", numValidators-len(curAddr)))
-
-		// Use writebatch to speed up bulk insert
-		wb := db.NewWriteBatch()
-
-		// Create a channel to receive errors from goroutines
-		errCh := make(chan error, numValidators)
-
-		limit := limiter.NewConcurrencyLimiter(debugKeybaseImportConcurrencyLimit)
-
-		for _, privHexString := range validatorKeysPairMap {
-			limit.Execute(func() {
-				// Import the keys into the keybase with no passphrase or hint as these are for debug purposes
-				keyPair, err := cryptoPocket.CreateNewKeyFromString(privHexString, "", "")
-				if err != nil {
-					errCh <- err
-					return
-				}
-
-				// Use key address as key in DB
-				addrKey := keyPair.GetAddressBytes()
-
-				// Encode KeyPair into []byte for value
-				keypairBz, err := keyPair.Marshal()
-				if err != nil {
-					errCh <- err
-					return
-				}
-				if err := wb.Set(addrKey, keypairBz); err != nil {
-					errCh <- err
-					return
-				}
-			})
-		}
-
-		limit.WaitAndClose()
-
-		// Check if any goroutines returned an error
-		select {
-		case err := <-errCh:
-			return err
-		default:
-		}
-
-		if err := wb.Flush(); err != nil {
-			return err
-		}
-	}
-
 	// Close DB connection
 	if err := kb.Stop(); err != nil {
 		return err
@@ -128,24 +55,31 @@ func initializeDebugKeybase() error {
 	return nil
 }
 
-// parseValidatorPrivateKeysFromEmbeddedYaml fetches the validator private keys from the embedded build/localnet/manifests/private-keys.yaml manifest file.
-func parseValidatorPrivateKeysFromEmbeddedYaml() (map[string]string, error) {
+func restoreBadgerDB(backupData []byte, db *badger.DB) error {
+	logger.Global.Debug().Msgf(fmt.Sprintf("Debug keybase initializing... Restoring from the embedded backup file..."))
 
-	// Parse the YAML file and load into the config struct
-	var config struct {
-		ApiVersion string            `yaml:"apiVersion"`
-		Kind       string            `yaml:"kind"`
-		MetaData   map[string]string `yaml:"metadata"`
-		Type       string            `yaml:"type"`
-		StringData map[string]string `yaml:"stringData"`
+	// Create a temporary directory to store the backup data
+	tempDir, err := ioutil.TempDir("", "badgerdb-restore")
+	if err != nil {
+		return err
 	}
-	if err := yaml.Unmarshal(build.PrivateKeysFile, &config); err != nil {
-		return nil, err
+	defer os.RemoveAll(tempDir)
+
+	// Write the backup data to a file in the temporary directory
+	backupFilePath := filepath.Join(tempDir, "backup")
+	if err := ioutil.WriteFile(backupFilePath, backupData, 0644); err != nil {
+		return err
 	}
-	validatorKeysMap := make(map[string]string)
 
-	for id, privHexString := range config.StringData {
-		validatorKeysMap[id] = privHexString
+	backupFile, err := os.Open(backupFilePath)
+	if err != nil {
+		return err
 	}
-	return validatorKeysMap, nil
+	defer backupFile.Close()
+
+	if err := db.Load(backupFile, 4); err != nil {
+		return err
+	}
+
+	return nil
 }

build/debug.go

@@ -4,13 +4,13 @@ package build
 
 import _ "embed"
 
-// PrivateKeysFile is the pre-generated manifest file for LocalNet debugging
+// DebugKeybaseBackup is a backup of the pre-loaded debug keybase sourced from the manifest file for LocalNet debugging
 //
-//go:embed localnet/manifests/private-keys.yaml
-var PrivateKeysFile []byte
+//go:embed debug_keybase/debug_keybase.bak
+var DebugKeybaseBackup []byte
 
 func init() {
-	if len(PrivateKeysFile) == 0 {
-		panic("PrivateKeysFile is empty")
+	if len(DebugKeybaseBackup) == 0 {
+		panic("DebugKeybaseBackup is empty")
 	}
 }

build/debug_keybase/cedffe1d784ca397062169b565abc056.md5

@@ -0,0 +1 @@
+This file is used to check if the keybase dump is in sync with the YAML file. Its name is the MD5 hash of the private_keys.yaml

build/debug_keybase/main.go

@@ -0,0 +1,192 @@
+package main
+
+import (
+	"crypto/md5"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/korovkin/limiter"
+	"gopkg.in/yaml.v2"
+
+	"github.com/pokt-network/pocket/app/client/keybase"
+	cryptoPocket "github.com/pokt-network/pocket/shared/crypto"
+	"github.com/pokt-network/pocket/shared/utils"
+)
+
+const (
+	// NOTE: This is the number of validators in the private-keys.yaml manifest file
+	numValidators = 999
+
+	// Increasing this number is linearly proportional to the amount of RAM required for the debug client to start and import
+	// pre-generated keys into the keybase. Beware that might cause OOM and process can exit with 137 status code.
+	// 4 threads takes 350-400MiB from a few tests which sounds acceptable.
+	debugKeybaseImportConcurrencyLimit = 4
+)
+
+func main() {
+	if len(os.Args) < 3 {
+		fmt.Println("Usage: go run main.go <source_yaml> <target_folder>")
+		return
+	}
+	sourceYamlPath := os.Args[1]
+	targetFolderPath := os.Args[2]
+
+	privateKeysYamlBytes, err := ioutil.ReadFile(sourceYamlPath)
+	if err != nil {
+		fmt.Printf("Error reading source_yaml: %v\n", err)
+		return
+	}
+	sourceYamlHash := md5.Sum(privateKeysYamlBytes) // nolint:gosec // Weak hashing function only used to check if the file has been changed
+	hashString := fmt.Sprintf("%x.md5", sourceYamlHash)
+	hashFilePath := filepath.Join(targetFolderPath, hashString)
+	targetFilePath := filepath.Join(targetFolderPath, "debug_keybase.bak")
+	if exists, _ := utils.FileExists(hashFilePath); !exists {
+		cleanupStaleFiles(targetFolderPath)
+		dumpKeybase(privateKeysYamlBytes, targetFilePath)
+		createHashFile(hashFilePath)
+	} else {
+		fmt.Println("✅ Keybase dump already exists and in sync with YAML file")
+	}
+}
+
+func dumpKeybase(privateKeysYamlBytes []byte, targetFilePath string) {
+	fmt.Println("⚙️ Initializing debug Keybase...")
+
+	validatorKeysPairMap, err := parseValidatorPrivateKeysFromEmbeddedYaml(privateKeysYamlBytes)
+	if err != nil {
+		panic(err)
+	}
+
+	tmpDir, err := os.MkdirTemp("/tmp", "pocket_debug_keybase_*")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	kb, err := keybase.NewKeybase(tmpDir)
+	if err != nil {
+		panic(err)
+	}
+	db := kb.GetBadgerDB()
+
+	// Add validator addresses if not present
+	fmt.Println("✍️ Debug keybase initializing... Adding all the validator keys")
+
+	// Use writebatch to speed up bulk insert
+	wb := db.NewWriteBatch()
+
+	// Create a channel to receive errors from goroutines
+	errCh := make(chan error, numValidators)
+
+	limit := limiter.NewConcurrencyLimiter(debugKeybaseImportConcurrencyLimit)
+	for _, privHexString := range validatorKeysPairMap {
+		limit.Execute(func() {
+			// Import the keys into the keybase with no passphrase or hint as these are for debug purposes
+			keyPair, err := cryptoPocket.CreateNewKeyFromString(privHexString, "", "")
+			if err != nil {
+				errCh <- err
+				return
+			}
+
+			// Use key address as key in DB
+			addrKey := keyPair.GetAddressBytes()
+
+			// Encode KeyPair into []byte for value
+			keypairBz, err := keyPair.Marshal()
+			if err != nil {
+				errCh <- err
+				return
+			}
+			if err := wb.Set(addrKey, keypairBz); err != nil {
+				errCh <- err
+				return
+			}
+		})
+	}
+
+	limit.WaitAndClose()
+
+	// Check if any goroutines returned an error
+	select {
+	case err := <-errCh:
+		panic(err)
+	default:
+	}
+
+	if err := wb.Flush(); err != nil {
+		panic(err)
+	}
+
+	fmt.Println("✅ Keybase initialized!")
+
+	fmt.Println("⚙️ Creating a dump of the Keybase...")
+	backupFile, err := os.Create(targetFilePath)
+	if err != nil {
+		panic(err)
+	}
+	defer backupFile.Close()
+	if _, err := db.Backup(backupFile, 0); err != nil {
+		panic(err)
+	}
+
+	// Close DB connection
+	if err := kb.Stop(); err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("✅ Keybase dumped in %s\n", targetFilePath)
+}
+
+func parseValidatorPrivateKeysFromEmbeddedYaml(privateKeysYamlBytes []byte) (map[string]string, error) {
+	// Parse the YAML file and load into the config struct
+	var config struct {
+		ApiVersion string            `yaml:"apiVersion"`
+		Kind       string            `yaml:"kind"`
+		MetaData   map[string]string `yaml:"metadata"`
+		Type       string            `yaml:"type"`
+		StringData map[string]string `yaml:"stringData"`
+	}
+	if err := yaml.Unmarshal(privateKeysYamlBytes, &config); err != nil {
+		return nil, err
+	}
+	validatorKeysMap := make(map[string]string)
+
+	for id, privHexString := range config.StringData {
+		validatorKeysMap[id] = privHexString
+	}
+	return validatorKeysMap, nil
+}
+
+func cleanupStaleFiles(targetFolderPath string) {
+	fmt.Printf("🧹 Cleaning up stale backup files in in %s\n", targetFolderPath)
+
+	if err := filepath.Walk(targetFolderPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			panic(err)
+		}
+
+		if !info.IsDir() && (filepath.Ext(path) == ".bak" || filepath.Ext(path) == ".md5") {
+			if err := os.Remove(path); err != nil {
+				panic(err)
+			}
+			fmt.Println("🚮 Deleted file:", path)
+		}
+
+		return nil
+	}); err != nil {
+		panic(err)
+	}
+}
+
+func createHashFile(hashFilePath string) {
+	fmt.Printf("🔖 Creating the MD5 hash file used to track consistency: %s\n", hashFilePath)
+
+	file, err := os.Create(hashFilePath)
+	if err != nil {
+		panic(err)
+	}
+	defer file.Close()
+	file.WriteString("This file is used to check if the keybase dump is in sync with the YAML file. Its name is the MD5 hash of the private_keys.yaml")
+}