diff --git a/Makefile b/Makefile index e79f26456..c76ea54d2 100644 --- a/Makefile +++ b/Makefile @@ -116,6 +116,7 @@ develop_start: ## Run all of the make commands necessary to develop on the proje make go_clean_deps && \ make mockgen && \ make generate_rpc_openapi && \ + make refresh_debug_keybase && \ make build .PHONY: develop_test @@ -218,6 +219,10 @@ docker_loki_install: docker_check ## Installs the loki docker driver docker_loki_check: if [ `docker plugin ls | grep loki: | wc -l` -eq 0 ]; then make docker_loki_install; fi +.PHONY: refresh_debug_keybase +refresh_debug_keybase: ## Refreshes the debug keybase with the latest keys from the private_keys.yaml file if necessary + go run build/debug_keybase/main.go ./build/localnet/manifests/private-keys.yaml ./build/debug_keybase + .PHONY: clean_mocks clean_mocks: ## Use `clean_mocks` to delete mocks before recreating them. Also useful to cleanup code that was generated from a different branch $(eval modules_dir = "shared/modules") diff --git a/app/client/keybase/debug/keystore.go b/app/client/keybase/debug/keystore.go index e7748664e..a95bc135f 100644 --- a/app/client/keybase/debug/keystore.go +++ b/app/client/keybase/debug/keystore.go @@ -4,26 +4,17 @@ package debug import ( "fmt" + "io/ioutil" "os" + "path/filepath" - "github.com/korovkin/limiter" + "github.com/dgraph-io/badger/v3" "github.com/pokt-network/pocket/app/client/keybase" "github.com/pokt-network/pocket/build" "github.com/pokt-network/pocket/logger" - cryptoPocket "github.com/pokt-network/pocket/shared/crypto" - "gopkg.in/yaml.v2" ) -const ( - // NOTE: This is the number of validators in the private-keys.yaml manifest file - numValidators = 999 - debugKeybaseSuffix = "/.pocket/keys" - - // Increasing this number is linearly proportional to the amount of RAM required for the debug client to start and import - // pre-generated keys into the keybase. Beware that might cause OOM and process can exit with 137 status code. - // 4 threads takes 350-400MiB from a few tests which sounds acceptable. - debugKeybaseImportConcurrencyLimit = 4 -) +const debugKeybaseSuffix = "/.pocket/keys" var ( // TODO: Allow users to override this value via `datadir` flag or env var or config file @@ -45,17 +36,6 @@ func init() { } func initializeDebugKeybase() error { - var ( - validatorKeysPairMap map[string]string - err error - ) - - validatorKeysPairMap, err = parseValidatorPrivateKeysFromEmbeddedYaml() - - if err != nil { - return err - } - // Create/Open the keybase at `$HOME/.pocket/keys` kb, err := keybase.NewKeybase(debugKeybasePath) if err != nil { @@ -63,63 +43,10 @@ func initializeDebugKeybase() error { } db := kb.GetBadgerDB() - // Add the keys if the keybase contains less than 999 - curAddr, _, err := kb.GetAll() - if err != nil { + if err := restoreBadgerDB(build.DebugKeybaseBackup, db); err != nil { return err } - // Add validator addresses if not present - if len(curAddr) < numValidators { - logger.Global.Debug().Msgf(fmt.Sprintf("Debug keybase initializing... Adding %d validator keys to the keybase", numValidators-len(curAddr))) - - // Use writebatch to speed up bulk insert - wb := db.NewWriteBatch() - - // Create a channel to receive errors from goroutines - errCh := make(chan error, numValidators) - - limit := limiter.NewConcurrencyLimiter(debugKeybaseImportConcurrencyLimit) - - for _, privHexString := range validatorKeysPairMap { - limit.Execute(func() { - // Import the keys into the keybase with no passphrase or hint as these are for debug purposes - keyPair, err := cryptoPocket.CreateNewKeyFromString(privHexString, "", "") - if err != nil { - errCh <- err - return - } - - // Use key address as key in DB - addrKey := keyPair.GetAddressBytes() - - // Encode KeyPair into []byte for value - keypairBz, err := keyPair.Marshal() - if err != nil { - errCh <- err - return - } - if err := wb.Set(addrKey, keypairBz); err != nil { - errCh <- err - return - } - }) - } - - limit.WaitAndClose() - - // Check if any goroutines returned an error - select { - case err := <-errCh: - return err - default: - } - - if err := wb.Flush(); err != nil { - return err - } - } - // Close DB connection if err := kb.Stop(); err != nil { return err @@ -128,24 +55,31 @@ func initializeDebugKeybase() error { return nil } -// parseValidatorPrivateKeysFromEmbeddedYaml fetches the validator private keys from the embedded build/localnet/manifests/private-keys.yaml manifest file. -func parseValidatorPrivateKeysFromEmbeddedYaml() (map[string]string, error) { +func restoreBadgerDB(backupData []byte, db *badger.DB) error { + logger.Global.Debug().Msg("Debug keybase initializing... Restoring from the embedded backup file...") - // Parse the YAML file and load into the config struct - var config struct { - ApiVersion string `yaml:"apiVersion"` - Kind string `yaml:"kind"` - MetaData map[string]string `yaml:"metadata"` - Type string `yaml:"type"` - StringData map[string]string `yaml:"stringData"` + // Create a temporary directory to store the backup data + tempDir, err := ioutil.TempDir("", "badgerdb-restore") + if err != nil { + return err } - if err := yaml.Unmarshal(build.PrivateKeysFile, &config); err != nil { - return nil, err + defer os.RemoveAll(tempDir) + + // Write the backup data to a file in the temporary directory + backupFilePath := filepath.Join(tempDir, "backup") + if err := ioutil.WriteFile(backupFilePath, backupData, 0644); err != nil { + return err } - validatorKeysMap := make(map[string]string) - for id, privHexString := range config.StringData { - validatorKeysMap[id] = privHexString + backupFile, err := os.Open(backupFilePath) + if err != nil { + return err } - return validatorKeysMap, nil + defer backupFile.Close() + + if err := db.Load(backupFile, 4); err != nil { + return err + } + + return nil } diff --git a/app/docs/CHANGELOG.md b/app/docs/CHANGELOG.md index 76ea051e9..4a375f4c5 100644 --- a/app/docs/CHANGELOG.md +++ b/app/docs/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.4] - 2023-03-24 + +- Updated debug keystore initialization to use an embedded backup instead of the yaml file that has to be rehydrated every time. + ## [0.0.0.3] - 2023-03-20 - Adds message routing type field labels to debug CLI actions diff --git a/build/debug.go b/build/debug.go index edd106882..062242ca8 100644 --- a/build/debug.go +++ b/build/debug.go @@ -4,13 +4,13 @@ package build import _ "embed" -// PrivateKeysFile is the pre-generated manifest file for LocalNet debugging +// DebugKeybaseBackup is a backup of the pre-loaded debug keybase sourced from the manifest file for LocalNet debugging // -//go:embed localnet/manifests/private-keys.yaml -var PrivateKeysFile []byte +//go:embed debug_keybase/debug_keybase.bak +var DebugKeybaseBackup []byte func init() { - if len(PrivateKeysFile) == 0 { - panic("PrivateKeysFile is empty") + if len(DebugKeybaseBackup) == 0 { + panic("DebugKeybaseBackup is empty") } } diff --git a/build/debug_keybase/cedffe1d784ca397062169b565abc056.md5 b/build/debug_keybase/cedffe1d784ca397062169b565abc056.md5 new file mode 100644 index 000000000..f0003fb9b --- /dev/null +++ b/build/debug_keybase/cedffe1d784ca397062169b565abc056.md5 @@ -0,0 +1 @@ +This file is used to check if the keybase dump is in sync with the YAML file. Its name is the MD5 hash of the private_keys.yaml \ No newline at end of file diff --git a/build/debug_keybase/debug_keybase.bak b/build/debug_keybase/debug_keybase.bak new file mode 100644 index 000000000..ef37c9a76 Binary files /dev/null and b/build/debug_keybase/debug_keybase.bak differ diff --git a/build/debug_keybase/main.go b/build/debug_keybase/main.go new file mode 100644 index 000000000..2d13827c4 --- /dev/null +++ b/build/debug_keybase/main.go @@ -0,0 +1,197 @@ +package main + +import ( + "crypto/md5" // nolint:gosec // Weak hashing function only used to check if the file has been changed + "fmt" + "os" + "path/filepath" + + "github.com/korovkin/limiter" + "gopkg.in/yaml.v2" + + "github.com/pokt-network/pocket/app/client/keybase" + cryptoPocket "github.com/pokt-network/pocket/shared/crypto" + "github.com/pokt-network/pocket/shared/utils" +) + +const ( + // NOTE: This is the number of validators in the private-keys.yaml manifest file + numValidators = 999 + + // Increasing this number is linearly proportional to the amount of RAM required for the debug client to start and import + // pre-generated keys into the keybase. Beware that might cause OOM and process can exit with 137 status code. + // 4 threads takes 350-400MiB from a few tests which sounds acceptable. + debugKeybaseImportConcurrencyLimit = 4 +) + +func main() { + if len(os.Args) < 3 { + fmt.Println("Usage: go run main.go ") + return + } + sourceYamlPath := os.Args[1] + targetFolderPath := os.Args[2] + + privateKeysYamlBytes, err := os.ReadFile(sourceYamlPath) + if err != nil { + fmt.Printf("Error reading source_yaml: %v\n", err) + return + } + sourceYamlHash := md5.Sum(privateKeysYamlBytes) // nolint:gosec // Weak hashing function only used to check if the file has been changed + hashString := fmt.Sprintf("%x.md5", sourceYamlHash) + hashFilePath := filepath.Join(targetFolderPath, hashString) + targetFilePath := filepath.Join(targetFolderPath, "debug_keybase.bak") + if exists, _ := utils.FileExists(hashFilePath); !exists { + cleanupStaleFiles(targetFolderPath) + dumpKeybase(privateKeysYamlBytes, targetFilePath) + createHashFile(hashFilePath) + } else { + fmt.Println("✅ Keybase dump already exists and in sync with YAML file") + } +} + +func dumpKeybase(privateKeysYamlBytes []byte, targetFilePath string) { + fmt.Println("⚙️ Initializing debug Keybase...") + + validatorKeysPairMap, err := parseValidatorPrivateKeysFromEmbeddedYaml(privateKeysYamlBytes) + if err != nil { + panic(err) + } + + tmpDir, err := os.MkdirTemp("/tmp", "pocket_debug_keybase_*") + if err != nil { + panic(err) + } + defer os.RemoveAll(tmpDir) + + kb, err := keybase.NewKeybase(tmpDir) + if err != nil { + panic(err) + } + db := kb.GetBadgerDB() + + // Add validator addresses if not present + fmt.Println("✍️ Debug keybase initializing... Adding all the validator keys") + + // Use writebatch to speed up bulk insert + wb := db.NewWriteBatch() + + // Create a channel to receive errors from goroutines + errCh := make(chan error, numValidators) + + limit := limiter.NewConcurrencyLimiter(debugKeybaseImportConcurrencyLimit) + for _, privHexString := range validatorKeysPairMap { + if _, err := limit.Execute(func() { + // Import the keys into the keybase with no passphrase or hint as these are for debug purposes + keyPair, err := cryptoPocket.CreateNewKeyFromString(privHexString, "", "") + if err != nil { + errCh <- err + return + } + + // Use key address as key in DB + addrKey := keyPair.GetAddressBytes() + + // Encode KeyPair into []byte for value + keypairBz, err := keyPair.Marshal() + if err != nil { + errCh <- err + return + } + if err := wb.Set(addrKey, keypairBz); err != nil { + errCh <- err + return + } + }); err != nil { + panic(err) + } + } + + if err := limit.WaitAndClose(); err != nil { + panic(err) + } + + // Check if any goroutines returned an error + select { + case err := <-errCh: + panic(err) + default: + } + + if err := wb.Flush(); err != nil { + panic(err) + } + + fmt.Println("✅ Keybase initialized!") + + fmt.Println("⚙️ Creating a dump of the Keybase...") + backupFile, err := os.Create(targetFilePath) + if err != nil { + panic(err) + } + defer backupFile.Close() + if _, err := db.Backup(backupFile, 0); err != nil { + panic(err) + } + + // Close DB connection + if err := kb.Stop(); err != nil { + panic(err) + } + + fmt.Printf("✅ Keybase dumped in %s\n", targetFilePath) +} + +func parseValidatorPrivateKeysFromEmbeddedYaml(privateKeysYamlBytes []byte) (map[string]string, error) { + // Parse the YAML file and load into the config struct + var config struct { + ApiVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + MetaData map[string]string `yaml:"metadata"` + Type string `yaml:"type"` + StringData map[string]string `yaml:"stringData"` + } + if err := yaml.Unmarshal(privateKeysYamlBytes, &config); err != nil { + return nil, err + } + validatorKeysMap := make(map[string]string) + + for id, privHexString := range config.StringData { + validatorKeysMap[id] = privHexString + } + return validatorKeysMap, nil +} + +func cleanupStaleFiles(targetFolderPath string) { + fmt.Printf("🧹 Cleaning up stale backup files in in %s\n", targetFolderPath) + + if err := filepath.Walk(targetFolderPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + panic(err) + } + + if !info.IsDir() && (filepath.Ext(path) == ".bak" || filepath.Ext(path) == ".md5") { + if err := os.Remove(path); err != nil { + panic(err) + } + fmt.Println("🚮 Deleted file:", path) + } + + return nil + }); err != nil { + panic(err) + } +} + +func createHashFile(hashFilePath string) { + fmt.Printf("🔖 Creating the MD5 hash file used to track consistency: %s\n", hashFilePath) + + file, err := os.Create(hashFilePath) + if err != nil { + panic(err) + } + defer file.Close() + if _, err := file.WriteString("This file is used to check if the keybase dump is in sync with the YAML file. Its name is the MD5 hash of the private_keys.yaml"); err != nil { + panic(err) + } +} diff --git a/build/docs/CHANGELOG.md b/build/docs/CHANGELOG.md index 5f466205a..c97eb4174 100644 --- a/build/docs/CHANGELOG.md +++ b/build/docs/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.25] - 2023-03-24 + +- Introduced a new binary that's used to check if the debug_keybase.bak is up to date with the private-keys.yaml file and to update it if it's not. + ## [0.0.0.24] - 2023-03-17 - Added resource limits and PVC for debug client pod.