Integrate State Proof keys

Makefile

@@ -76,7 +76,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
 UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ ))
 ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... ))
 
-MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./compactcert ./txnsync ./data/pooldata
+MSGP_GENERATE	:= ./protocol ./protocol/test ./crypto ./crypto/merklearray ./crypto/merklekeystore ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./compactcert ./txnsync ./data/pooldata
 
 default: build
 

cmd/algokey/part.go

@@ -72,12 +72,14 @@ var partGenerateCmd = &cobra.Command{
 		partdb, err := db.MakeErasableAccessor(partKeyfile)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Cannot open partkey database %s: %v\n", partKeyfile, err)
+			os.Remove(partKeyfile)
 			os.Exit(1)
 		}
 
 		partkey, err := account.FillDBWithParticipationKeys(partdb, parent, basics.Round(partFirstRound), basics.Round(partLastRound), partKeyDilution)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Cannot generate partkey database %s: %v\n", partKeyfile, err)
+			os.Remove(partKeyfile)
 			os.Exit(1)
 		}
 
@@ -146,6 +148,9 @@ func printPartkey(partkey account.Participation) {
 	fmt.Printf("Parent address:    %s\n", partkey.Parent.String())
 	fmt.Printf("VRF public key:    %s\n", base64.StdEncoding.EncodeToString(partkey.VRF.PK[:]))
 	fmt.Printf("Voting public key: %s\n", base64.StdEncoding.EncodeToString(partkey.Voting.OneTimeSignatureVerifier[:]))
+	if partkey.StateProofSecrets != nil && !partkey.StateProofSecrets.GetVerifier().IsEmpty() {
+		fmt.Printf("State proof key:     %s\n", base64.StdEncoding.EncodeToString(partkey.StateProofSecrets.GetVerifier().Root[:]))
+	}
 	fmt.Printf("First round:       %d\n", partkey.FirstValid)
 	fmt.Printf("Last round:        %d\n", partkey.LastValid)
 	fmt.Printf("Key dilution:      %d\n", partkey.KeyDilution)

cmd/goal/account.go

@@ -26,10 +26,9 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/spf13/cobra"
-
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/crypto"
+	"github.com/algorand/go-algorand/crypto/merklekeystore"
 	"github.com/algorand/go-algorand/crypto/passphrase"
 	generatedV2 "github.com/algorand/go-algorand/daemon/algod/api/server/v2/generated"
 	algodAcct "github.com/algorand/go-algorand/data/account"
@@ -39,6 +38,8 @@ import (
 	"github.com/algorand/go-algorand/protocol"
 	"github.com/algorand/go-algorand/util"
 	"github.com/algorand/go-algorand/util/db"
+
+	"github.com/spf13/cobra"
 )
 
 var (
@@ -1283,6 +1284,7 @@ type partkeyInfo struct {
 	LastValid       basics.Round                    `codec:"last"`
 	VoteID          crypto.OneTimeSignatureVerifier `codec:"vote"`
 	SelectionID     crypto.VRFVerifier              `codec:"sel"`
+	StateProofID    merklekeystore.Verifier         `codec:"stprf"`
 	VoteKeyDilution uint64                          `codec:"voteKD"`
 }
 
@@ -1305,6 +1307,7 @@ var partkeyInfoCmd = &cobra.Command{
 
 			for filename, part := range parts {
 				fmt.Println("------------------------------------------------------------------")
+
 				info := partkeyInfo{
 					Address:         part.Address().String(),
 					FirstValid:      part.FirstValid,
@@ -1313,6 +1316,9 @@ var partkeyInfoCmd = &cobra.Command{
 					SelectionID:     part.VRFSecrets().PK,
 					VoteKeyDilution: part.KeyDilution,
 				}
+				if certSigner := part.StateProofSigner(); certSigner != nil {
+					info.StateProofID = *certSigner.GetVerifier()
+				}
 				infoString := protocol.EncodeJSON(&info)
 				fmt.Printf("File: %s\n%s\n", filename, string(infoString))
 			}

cmd/goal/network.go

@@ -101,7 +101,7 @@ var networkCreateCmd = &cobra.Command{
 			consensus, _ = config.PreloadConfigurableConsensusProtocols(dataDir)
 		}
 
-		network, err := netdeploy.CreateNetworkFromTemplate(networkName, networkRootDir, networkTemplateFile, binDir, !noImportKeys, nil, consensus)
+		network, err := netdeploy.CreateNetworkFromTemplate(networkName, networkRootDir, networkTemplateFile, binDir, !noImportKeys, nil, consensus, false)
 		if err != nil {
 			if noClean {
 				reportInfof(" ** failed ** - Preserving network rootdir '%s'", networkRootDir)

compactcert/db.go

@@ -20,7 +20,7 @@ import (
 	"database/sql"
 	"fmt"
 
-	"github.com/algorand/go-algorand/crypto"
+	"github.com/algorand/go-algorand/crypto/merklekeystore"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/protocol"
 )
@@ -47,7 +47,7 @@ var schema = []string{
 
 type pendingSig struct {
 	signer       basics.Address
-	sig          crypto.OneTimeSignature
+	sig          merklekeystore.Signature
 	fromThisNode bool
 }
 

compactcert/signer.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	"github.com/algorand/go-algorand/config"
-	"github.com/algorand/go-algorand/crypto"
+	"github.com/algorand/go-algorand/crypto/merklekeystore"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/protocol"
@@ -34,9 +34,9 @@ import (
 type sigFromAddr struct {
 	_struct struct{} `codec:",omitempty,omitemptyarray"`
 
-	Signer basics.Address          `codec:"signer"`
-	Round  basics.Round            `codec:"rnd"`
-	Sig    crypto.OneTimeSignature `codec:"sig"`
+	Signer basics.Address           `codec:"signer"`
+	Round  basics.Round             `codec:"rnd"`
+	Sig    merklekeystore.Signature `codec:"sig"`
 }
 
 func (ccw *Worker) signer(latest basics.Round) {
@@ -93,12 +93,7 @@ func (ccw *Worker) signBlock(hdr bookkeeping.BlockHeader) {
 		return
 	}
 
-	// Compact cert gets signed by the next round after the block,
-	// because by the time agreement is reached on the block,
-	// ephemeral keys for that round could be deleted.
-	sigKeyRound := hdr.Round + 1
-
-	keys := ccw.accts.Keys(sigKeyRound)
+	keys := ccw.accts.Keys(hdr.Round)
 	if len(keys) == 0 {
 		// No keys, nothing to do.
 		return
@@ -113,33 +108,30 @@ func (ccw *Worker) signBlock(hdr bookkeeping.BlockHeader) {
 		return
 	}
 
-	if votersHdr.CompactCert[protocol.CompactCertBasic].CompactCertVoters.IsZero() {
+	if votersHdr.CompactCert[protocol.CompactCertBasic].CompactCertVoters.IsEmpty() {
 		// No voter commitment, perhaps because compact certs were
 		// just enabled.
 		return
 	}
 
-	votersProto := config.Consensus[votersHdr.CurrentProtocol]
+	sigs := make([]sigFromAddr, 0, len(keys))
 
-	var sigs []sigFromAddr
-	var sigkeys []crypto.OneTimeSignatureVerifier
 	for _, key := range keys {
-		if key.FirstValid <= sigKeyRound && sigKeyRound <= key.LastValid {
-			keyDilution := key.KeyDilution
-			if keyDilution == 0 {
-				keyDilution = votersProto.DefaultKeyDilution
-			}
-
-			ephID := basics.OneTimeIDForRound(sigKeyRound, keyDilution)
-			sig := key.Voting.Sign(ephID, hdr)
+		if key.FirstValid > hdr.Round || hdr.Round > key.LastValid {
+			continue
+		}
 
-			sigs = append(sigs, sigFromAddr{
-				Signer: key.Parent,
-				Round:  hdr.Round,
-				Sig:    sig,
-			})
-			sigkeys = append(sigkeys, key.Voting.OneTimeSignatureVerifier)
+		sig, err := key.StateProofSecrets.Sign(hdr, uint64(hdr.Round))
+		if err != nil {
+			ccw.log.Warnf("ccw.signBlock(%d): StateProof.Sign: %v", hdr.Round, err)
+			continue
 		}
+
+		sigs = append(sigs, sigFromAddr{
+			Signer: key.Parent,
+			Round:  hdr.Round,
+			Sig:    sig,
+		})
 	}
 
 	for _, sfa := range sigs {

compactcert/worker_test.go

@@ -77,7 +77,11 @@ func (s *testWorkerStubs) addBlock(ccNextRound basics.Round) {
 	hdr.Round = s.latest
 	hdr.CurrentProtocol = protocol.ConsensusFuture
 
-	var ccBasic bookkeeping.CompactCertState
+	var ccBasic = bookkeeping.CompactCertState{
+		CompactCertVoters:      make([]byte, compactcert.HashSize),
+		CompactCertVotersTotal: basics.MicroAlgos{},
+		CompactCertNextRound:   0,
+	}
 	ccBasic.CompactCertVotersTotal.Raw = uint64(s.totalWeight)
 
 	if hdr.Round > 0 {
@@ -131,13 +135,13 @@ func (s *testWorkerStubs) CompactCertVoters(r basics.Round) (*ledgercore.VotersF
 	for i, k := range s.keysForVoters {
 		voters.AddrToPos[k.Parent] = uint64(i)
 		voters.Participants = append(voters.Participants, basics.Participant{
-			PK:          k.Voting.OneTimeSignatureVerifier,
-			Weight:      1,
-			KeyDilution: config.Consensus[protocol.ConsensusFuture].DefaultKeyDilution,
+			PK:         *k.StateProofSecrets.GetVerifier(),
+			Weight:     1,
+			FirstValid: uint64(k.FirstValid),
 		})
 	}
 
-	tree, err := merklearray.Build(voters.Participants)
+	tree, err := merklearray.Build(voters.Participants, crypto.HashFactory{HashType: compactcert.HashType})
 	if err != nil {
 		return nil, err
 	}
@@ -201,15 +205,17 @@ func newTestWorker(t testing.TB, s *testWorkerStubs) *Worker {
 	return newTestWorkerDB(t, s, dbs.Wdb)
 }
 
-func newPartKey(t testing.TB, parent basics.Address) account.Participation {
+// You must call defer part.Close() after calling this function,
+// since it creates a DB accessor but the caller must close it (required for PersistentKeystore)
+func newPartKey(t testing.TB, parent basics.Address) account.PersistedParticipation {
 	fn := fmt.Sprintf("%s.%d", strings.ReplaceAll(t.Name(), "/", "."), crypto.RandUint64())
 	partDB, err := db.MakeAccessor(fn, false, true)
 	require.NoError(t, err)
 
-	part, err := account.FillDBWithParticipationKeys(partDB, parent, 0, 1024*1024, config.Consensus[protocol.ConsensusFuture].DefaultKeyDilution)
+	part, err := account.FillDBWithParticipationKeys(partDB, parent, 0, 1024, config.Consensus[protocol.ConsensusFuture].DefaultKeyDilution)
 	require.NoError(t, err)
-	part.Close()
-	return part.Participation
+
+	return part
 }
 
 func TestWorkerAllSigs(t *testing.T) {
@@ -219,7 +225,9 @@ func TestWorkerAllSigs(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		var parent basics.Address
 		crypto.RandBytes(parent[:])
-		keys = append(keys, newPartKey(t, parent))
+		p := newPartKey(t, parent)
+		defer p.Close()
+		keys = append(keys, p.Participation)
 	}
 
 	s := newWorkerStubs(t, keys, len(keys))
@@ -257,10 +265,11 @@ func TestWorkerAllSigs(t *testing.T) {
 			require.False(t, overflowed)
 
 			ccparams := compactcert.Params{
-				Msg:          signedHdr,
-				ProvenWeight: provenWeight,
-				SigRound:     basics.Round(signedHdr.Round + 1),
-				SecKQ:        proto.CompactCertSecKQ,
+				Msg:               signedHdr,
+				ProvenWeight:      provenWeight,
+				SigRound:          basics.Round(signedHdr.Round),
+				SecKQ:             proto.CompactCertSecKQ,
+				CompactCertRounds: proto.CompactCertRounds,
 			}
 
 			voters, err := s.CompactCertVoters(tx.Txn.CertRound - basics.Round(proto.CompactCertRounds) - basics.Round(proto.CompactCertVotersLookback))
@@ -281,7 +290,9 @@ func TestWorkerPartialSigs(t *testing.T) {
 	for i := 0; i < 7; i++ {
 		var parent basics.Address
 		crypto.RandBytes(parent[:])
-		keys = append(keys, newPartKey(t, parent))
+		p := newPartKey(t, parent)
+		defer p.Close()
+		keys = append(keys, p.Participation)
 	}
 
 	s := newWorkerStubs(t, keys, 10)
@@ -318,10 +329,11 @@ func TestWorkerPartialSigs(t *testing.T) {
 	require.False(t, overflowed)
 
 	ccparams := compactcert.Params{
-		Msg:          signedHdr,
-		ProvenWeight: provenWeight,
-		SigRound:     basics.Round(signedHdr.Round + 1),
-		SecKQ:        proto.CompactCertSecKQ,
+		Msg:               signedHdr,
+		ProvenWeight:      provenWeight,
+		SigRound:          basics.Round(signedHdr.Round),
+		SecKQ:             proto.CompactCertSecKQ,
+		CompactCertRounds: proto.CompactCertRounds,
 	}
 
 	voters, err := s.CompactCertVoters(tx.Txn.CertRound - basics.Round(proto.CompactCertRounds) - basics.Round(proto.CompactCertVotersLookback))
@@ -339,7 +351,9 @@ func TestWorkerInsufficientSigs(t *testing.T) {
 	for i := 0; i < 2; i++ {
 		var parent basics.Address
 		crypto.RandBytes(parent[:])
-		keys = append(keys, newPartKey(t, parent))
+		p := newPartKey(t, parent)
+		defer p.Close()
+		keys = append(keys, p.Participation)
 	}
 
 	s := newWorkerStubs(t, keys, 10)
@@ -370,7 +384,9 @@ func TestLatestSigsFromThisNode(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		var parent basics.Address
 		crypto.RandBytes(parent[:])
-		keys = append(keys, newPartKey(t, parent))
+		p := newPartKey(t, parent)
+		defer p.Close()
+		keys = append(keys, p.Participation)
 	}
 
 	s := newWorkerStubs(t, keys, 10)
@@ -423,7 +439,9 @@ func TestWorkerRestart(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		var parent basics.Address
 		crypto.RandBytes(parent[:])
-		keys = append(keys, newPartKey(t, parent))
+		p := newPartKey(t, parent)
+		defer p.Close()
+		keys = append(keys, p.Participation)
 	}
 
 	s := newWorkerStubs(t, keys, 10)
@@ -464,7 +482,9 @@ func TestWorkerHandleSig(t *testing.T) {
 	for i := 0; i < 2; i++ {
 		var parent basics.Address
 		crypto.RandBytes(parent[:])
-		keys = append(keys, newPartKey(t, parent))
+		p := newPartKey(t, parent)
+		defer p.Close()
+		keys = append(keys, p.Participation)
 	}
 
 	s := newWorkerStubs(t, keys, 10)

config/consensus.go

@@ -395,6 +395,9 @@ type ConsensusParams struct {
 	// MaxProposedExpiredOnlineAccounts is the maximum number of online accounts, which need
 	// to be taken offline, that would be proposed to be taken offline.
 	MaxProposedExpiredOnlineAccounts int
+
+	// EnableStateProofKeyregCheck enables the check for stateProof key on key registration
+	EnableStateProofKeyregCheck bool
 }
 
 // PaysetCommitType enumerates possible ways for the block header to commit to
@@ -1056,6 +1059,9 @@ func initConsensusProtocols() {
 
 	vFuture.MaxProposedExpiredOnlineAccounts = 32
 
+	// stat proof key registration
+	vFuture.EnableStateProofKeyregCheck = true
+
 	Consensus[protocol.ConsensusFuture] = vFuture
 }