Rename --validator-source to --web3-signer-url and document it (#5389)

Also allows multiple instances to be configured

beacon_chain/conf.nim

@@ -162,14 +162,14 @@ type
       desc: "A directory containing validator keystores"
       name: "validators-dir" .}: Option[InputDir]
 
-    validatorsSource* {.
+    web3signers* {.
       desc: "Remote Web3Signer URL that will be used as a source of validators"
-      name: "validators-source"}: Option[string]
+      name: "web3-signer-url" .}: seq[Uri]
 
-    validatorsSourceInverval* {.
-      desc: "Number of minutes between validator list updates"
-      name: "validators-source-interval"
-      defaultValue: 60 .}: Natural
+    web3signerUpdateInterval* {.
+      desc: "Number of seconds between validator list updates"
+      name: "web3-signer-update-interval"
+      defaultValue: 3600 .}: Natural
 
     secretsDirFlag* {.
       desc: "A directory containing validator keystore passwords"
@@ -885,14 +885,14 @@ type
       desc: "A directory containing validator keystores"
       name: "validators-dir" .}: Option[InputDir]
 
-    validatorsSource* {.
+    web3signers* {.
       desc: "Remote Web3Signer URL that will be used as a source of validators"
-      name: "validators-source"}: Option[string]
+      name: "web3-signer-url" .}: seq[Uri]
 
-    validatorsSourceInverval* {.
-      desc: "Number of minutes between validator list updates"
-      name: "validators-source-interval"
-      defaultValue: 60 .}: Natural
+    web3signerUpdateInterval* {.
+      desc: "Number of seconds between validator list updates"
+      name: "web3-signer-update-interval"
+      defaultValue: 3600 .}: Natural
 
     secretsDirFlag* {.
       desc: "A directory containing validator keystore passwords"

beacon_chain/nimbus_beacon_node.nim

@@ -1617,7 +1617,17 @@ proc run(node: BeaconNode) {.raises: [CatchableError].} =
 
   waitFor node.updateGossipStatus(wallSlot)
 
-  asyncSpawn pollForDynamicValidators(node)
+  for web3signerUrl in node.config.web3signers:
+    # TODO
+    # The current strategy polls all remote signers independently
+    # from each other which may lead to some race conditions of
+    # validators are migrated from one signer to another
+    # (because the updates to our validator pool are not atomic).
+    # Consider using different strategies that would detect such
+    # race conditions.
+    asyncSpawn node.pollForDynamicValidators(
+      web3signerUrl, node.config.web3signerUpdateInterval)
+
   asyncSpawn runSlotLoop(node, wallTime, onSlotStart)
   asyncSpawn runOnSecondLoop(node)
   asyncSpawn runQueueProcessingLoop(node.blockProcessor)

beacon_chain/nimbus_validator_client.nim

@@ -85,16 +85,28 @@ proc initGenesis(vc: ValidatorClientRef): Future[RestGenesis] {.async.} =
             dec(counter)
       return melem
 
-proc initValidators(vc: ValidatorClientRef): Future[bool] {.async.} =
-  info "Loading validators", validatorsDir = vc.config.validatorsDir()
-  var duplicates: seq[ValidatorPubKey]
-  for keystore in listLoadableKeystores(vc.config, vc.keystoreCache):
-    vc.addValidator(keystore)
-  let res = await queryValidatorsSource(vc.config)
+proc addValidatorsFromWeb3Signer(vc: ValidatorClientRef, web3signerUrl: Uri) {.async.} =
+  let res = await queryValidatorsSource(web3signerUrl)
   if res.isOk():
     let dynamicKeystores = res.get()
     for keystore in dynamicKeystores:
       vc.addValidator(keystore)
+
+proc initValidators(vc: ValidatorClientRef): Future[bool] {.async.} =
+  info "Loading validators", validatorsDir = vc.config.validatorsDir()
+  for keystore in listLoadableKeystores(vc.config, vc.keystoreCache):
+    vc.addValidator(keystore)
+
+  let web3signerValidatorsFuts = mapIt(
+    vc.config.web3signers,
+    vc.addValidatorsFromWeb3Signer(it))
+
+  # We use `allFutures` because all failures are already reported as
+  # user-visible warnings in `queryValidatorsSource`.
+  # We don't consider them fatal because the Web3Signer may be experiencing
+  # a temporary hiccup that will be resolved later.
+  await allFutures(web3signerValidatorsFuts)
+
   true
 
 proc initClock(vc: ValidatorClientRef): Future[BeaconClock] {.async.} =

beacon_chain/validator_client/duties_service.nim

@@ -589,15 +589,17 @@ proc validatorIndexLoop(service: DutiesServiceRef) {.async.} =
     await service.pollForValidatorIndices()
     await service.waitForNextSlot()
 
-proc dynamicValidatorsLoop*(service: DutiesServiceRef) {.async.} =
+proc dynamicValidatorsLoop*(service: DutiesServiceRef,
+                            web3signerUrl: Uri,
+                            intervalInSeconds: int) {.async.} =
   let vc = service.client
-  doAssert(vc.config.validatorsSourceInverval > 0)
+  doAssert(intervalInSeconds > 0)
 
   proc addValidatorProc(data: KeystoreData) =
     vc.addValidator(data)
 
   var
-    timeout = minutes(vc.config.validatorsSourceInverval)
+    timeout = seconds(intervalInSeconds)
     exitLoop = false
 
   while not(exitLoop):
@@ -606,15 +608,16 @@ proc dynamicValidatorsLoop*(service: DutiesServiceRef) {.async.} =
         await sleepAsync(timeout)
         timeout =
           block:
-            let res = await vc.config.queryValidatorsSource()
+            let res = await queryValidatorsSource(web3signerUrl)
             if res.isOk():
               let keystores = res.get()
-              debug "Validators source has been polled for validators",
+              debug "Web3Signer has been polled for validators",
                     keystores_found = len(keystores),
-                    validators_source = vc.config.validatorsSource
-              vc.attachedValidators.updateDynamicValidators(keystores,
+                    web3signer_url = web3signerUrl
+              vc.attachedValidators.updateDynamicValidators(web3signerUrl,
+                                                            keystores,
                                                             addValidatorProc)
-              minutes(vc.config.validatorsSourceInverval)
+              seconds(intervalInSeconds)
             else:
               seconds(5)
         false
@@ -694,12 +697,13 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
         service.validatorRegisterLoop()
       else:
         nil
-    dynamicFut =
-      if vc.config.validatorsSourceInverval > 0:
-        service.dynamicValidatorsLoop()
+    dynamicFuts =
+      if vc.config.web3signerUpdateInterval > 0:
+        mapIt(vc.config.web3signers,
+              service.dynamicValidatorsLoop(it, vc.config.web3signerUpdateInterval))
       else:
         debug "Dynamic validators update loop disabled"
-        nil
+        @[]
 
   while true:
     # This loop could look much more nicer/better, when
@@ -713,8 +717,9 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
           FutureBase(indicesFut),
           FutureBase(syncFut),
           FutureBase(prepareFut),
-          FutureBase(dynamicFut)
         ]
+        for fut in dynamicFuts:
+          futures.add fut
         if not(isNil(registerFut)): futures.add(FutureBase(registerFut))
         discard await race(futures)
         checkAndRestart(AttesterLoop, attestFut, service.attesterDutiesLoop())
@@ -727,9 +732,11 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
         if not(isNil(registerFut)):
           checkAndRestart(ValidatorRegisterLoop, registerFut,
                           service.validatorRegisterLoop())
-        if not(isNil(dynamicFut)):
-          checkAndRestart(DynamicValidatorsLoop, dynamicFut,
-                          service.dynamicValidatorsLoop())
+        for i in 0 ..< dynamicFuts.len:
+          checkAndRestart(DynamicValidatorsLoop, dynamicFuts[i],
+                          service.dynamicValidatorsLoop(
+                            vc.config.web3signers[i],
+                            vc.config.web3signerUpdateInterval))
         false
       except CancelledError:
         debug "Service interrupted"
@@ -746,8 +753,9 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
           pending.add(prepareFut.cancelAndWait())
         if not(isNil(registerFut)) and not(registerFut.finished()):
           pending.add(registerFut.cancelAndWait())
-        if not(isNil(dynamicFut)) and not(dynamicFut.finished()):
-          pending.add(dynamicFut.cancelAndWait())
+        for dynamicFut in dynamicFuts:
+          if not dynamicFut.finished():
+            pending.add(dynamicFut.cancelAndWait())
         if not(isNil(service.pollingAttesterDutiesTask)) and
            not(service.pollingAttesterDutiesTask.finished()):
           pending.add(service.pollingAttesterDutiesTask.cancelAndWait())

beacon_chain/validators/beacon_validators.nim

@@ -114,31 +114,10 @@ proc getValidator*(validators: auto,
     Opt.some ValidatorAndIndex(index: ValidatorIndex(idx),
                                validator: validators[idx])
 
-proc addValidators*(node: BeaconNode) =
-  info "Loading validators", validatorsDir = node.config.validatorsDir(),
-                keystore_cache_available = not(isNil(node.keystoreCache))
-  let epoch = node.currentSlot().epoch
-
-  for keystore in listLoadableKeystores(node.config, node.keystoreCache):
-    let
-      data = withState(node.dag.headState):
-        getValidator(forkyState.data.validators.asSeq(), keystore.pubkey)
-      index =
-        if data.isSome():
-          Opt.some(data.get().index)
-        else:
-          Opt.none(ValidatorIndex)
-      feeRecipient = node.consensusManager[].getFeeRecipient(
-        keystore.pubkey, index, epoch)
-      gasLimit = node.consensusManager[].getGasLimit(keystore.pubkey)
-
-      v = node.attachedValidators[].addValidator(keystore, feeRecipient,
-                                                 gasLimit)
-    v.updateValidator(data)
-
+proc addValidatorsFromWeb3Signer(node: BeaconNode, web3signerUrl: Uri, epoch: Epoch) {.async.} =
   let dynamicStores =
     try:
-      let res = waitFor(queryValidatorsSource(node.config))
+      let res = await queryValidatorsSource(web3signerUrl)
       if res.isErr():
         # Error is already reported via log warning.
         default(seq[KeystoreData])
@@ -166,8 +145,47 @@ proc addValidators*(node: BeaconNode) =
                                                  gasLimit)
     v.updateValidator(data)
 
-proc pollForDynamicValidators*(node: BeaconNode) {.async.} =
-  if node.config.validatorsSourceInverval == 0:
+proc addValidators*(node: BeaconNode) =
+  info "Loading validators", validatorsDir = node.config.validatorsDir(),
+                keystore_cache_available = not(isNil(node.keystoreCache))
+  let epoch = node.currentSlot().epoch
+
+  for keystore in listLoadableKeystores(node.config, node.keystoreCache):
+    let
+      data = withState(node.dag.headState):
+        getValidator(forkyState.data.validators.asSeq(), keystore.pubkey)
+      index =
+        if data.isSome():
+          Opt.some(data.get().index)
+        else:
+          Opt.none(ValidatorIndex)
+      feeRecipient = node.consensusManager[].getFeeRecipient(
+        keystore.pubkey, index, epoch)
+      gasLimit = node.consensusManager[].getGasLimit(keystore.pubkey)
+
+      v = node.attachedValidators[].addValidator(keystore, feeRecipient,
+                                                 gasLimit)
+    v.updateValidator(data)
+
+  try:
+    # We use `allFutures` because all failures are already reported as
+    # user-visible warnings in `queryValidatorsSource`.
+    # We don't consider them fatal because the Web3Signer may be experiencing
+    # a temporary hiccup that will be resolved later.
+    waitFor allFutures(mapIt(node.config.web3signers,
+                             node.addValidatorsFromWeb3Signer(it, epoch)))
+  except CatchableError as err:
+    # This should never happen because all errors are handled within
+    # `addValidatorsFromWeb3Signer`. Furthermore, the code above is
+    # using `allFutures` which is guaranteed to not raise exceptions.
+    # Nevertheless, we need it to make the compiler's exception tracking happy.
+    debug "Unexpected error while fetching the list of validators from a remote signer",
+           err = err.msg
+
+proc pollForDynamicValidators*(node: BeaconNode,
+                               web3signerUrl: Uri,
+                               intervalInSeconds: int) {.async.} =
+  if intervalInSeconds == 0:
     return
 
   proc addValidatorProc(keystore: KeystoreData) =
@@ -182,7 +200,7 @@ proc pollForDynamicValidators*(node: BeaconNode) {.async.} =
                                                    gasLimit)
 
   var
-    timeout = minutes(node.config.validatorsSourceInverval)
+    timeout = seconds(intervalInSeconds)
     exitLoop = false
 
   while not(exitLoop):
@@ -191,15 +209,16 @@ proc pollForDynamicValidators*(node: BeaconNode) {.async.} =
         await sleepAsync(timeout)
         timeout =
           block:
-            let res = await node.config.queryValidatorsSource()
+            let res = await queryValidatorsSource(web3signerUrl)
             if res.isOk():
               let keystores = res.get()
               debug "Validators source has been polled for validators",
                     keystores_found = len(keystores),
-                    validators_source = node.config.validatorsSource
-              node.attachedValidators.updateDynamicValidators(keystores,
+                    web3signer_url = web3signerUrl
+              node.attachedValidators.updateDynamicValidators(web3signerUrl,
+                                                              keystores,
                                                               addValidatorProc)
-              minutes(node.config.validatorsSourceInverval)
+              seconds(intervalInSeconds)
             else:
               # In case of error we going to repeat our call with much smaller
               # interval.

beacon_chain/validators/keystore_management.nim

@@ -629,23 +629,19 @@ proc existsKeystore(keystoreDir: string,
       return true
   false
 
-proc queryValidatorsSource*(config: AnyConf): Future[QueryResult] {.async.} =
+proc queryValidatorsSource*(web3signerUrl: Uri): Future[QueryResult] {.async.} =
   var keystores: seq[KeystoreData]
-  if config.validatorsSource.isNone() or
-     len(config.validatorsSource.get()) == 0:
-    return QueryResult.ok(keystores)
 
-  let vsource = config.validatorsSource.get()
   logScope:
-    validators_source = vsource
+    web3signer_url = web3signerUrl
 
   let
     httpFlags: HttpClientFlags = {}
     prestoFlags = {RestClientFlag.CommaSeparatedArray}
     socketFlags = {SocketFlags.TcpNoDelay}
     client =
       block:
-        let res = RestClientRef.new(vsource, prestoFlags,
+        let res = RestClientRef.new($web3signerUrl, prestoFlags,
                                     httpFlags, socketFlags = socketFlags)
         if res.isErr():
           warn "Unable to resolve validator's source distributed signer " &
@@ -686,7 +682,7 @@ proc queryValidatorsSource*(config: AnyConf): Future[QueryResult] {.async.} =
       handle: FileLockHandle(opened: false),
       pubkey: pubkey,
       remotes: @[RemoteSignerInfo(
-        url: HttpHostUri(parseUri(vsource)),
+        url: HttpHostUri(web3signerUrl),
         pubkey: pubkey)],
       flags: {RemoteKeystoreFlag.DynamicKeystore},
       remoteType: RemoteSignerType.Web3Signer))

beacon_chain/validators/validator_pool.nim

@@ -380,6 +380,7 @@ func triggersDoppelganger*(
   v.isSome() and v[].triggersDoppelganger(epoch)
 
 proc updateDynamicValidators*(pool: ref ValidatorPool,
+                              web3signerUrl: Uri,
                               keystores: openArray[KeystoreData],
                               addProc: AddValidatorProc) =
   var
@@ -399,7 +400,12 @@ proc updateDynamicValidators*(pool: ref ValidatorPool,
         if keystore.isSome():
           # Just update validator's `data` field with new data from keystore.
           validator.data = keystore.get()
-        else:
+        elif validator.data.remotes[0].url == HttpHostUri(web3signerUrl):
+          # The "dynamic" keystores are guaratneed to not be distributed
+          # so they have a single remote. This code ensures that we are
+          # deleting all previous dynamically obtained keystores which
+          # were associated with a particular Web3Signer when the same
+          # signer no longer serves them.
           deleteValidators.add(validator.pubkey)
 
   for pubkey in deleteValidators:
@@ -517,7 +523,7 @@ proc getBlockSignature*(v: AttachedValidator, fork: Fork,
           fork, genesis_validators_root, slot, block_root,
           v.data.privateKey).toValidatorSig())
     of ValidatorKind.Remote:
-      let web3SignerRequest =
+      let web3signerRequest =
         when blck is ForkedBlindedBeaconBlock:
           case blck.kind
           of ConsensusFork.Phase0, ConsensusFork.Altair, ConsensusFork.Bellatrix:
@@ -617,7 +623,7 @@ proc getBlockSignature*(v: AttachedValidator, fork: Fork,
                 Web3SignerForkedBeaconBlock(kind: ConsensusFork.Deneb,
                   data: blck.denebData.toBeaconBlockHeader),
                 proofs)
-      await v.signData(web3SignerRequest)
+      await v.signData(web3signerRequest)
 
 # https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/deneb/validator.md#constructing-the-signedblobsidecars
 proc getBlobSignature*(v: AttachedValidator, fork: Fork,

docs/the_nimbus_book/src/options.md

@@ -33,8 +33,8 @@ The following options are available:
      --network                 The Eth2 network to join [=mainnet].
  -d, --data-dir                The directory where nimbus will store all blockchain data.
      --validators-dir          A directory containing validator keystores.
-     --validators-source       Remote Web3Signer URL that will be used as a source of validators.
-     --validators-source-interval  Number of minutes between validator list updates [=60].
+     --web3-signer-url         Remote Web3Signer URL that will be used as a source of validators.
+     --web3-signer-update-interval  Number of seconds between validator list updates [=3600].
      --secrets-dir             A directory containing validator keystore passwords.
      --wallets-dir             A directory containing wallet files.
      --web3-url                One or more execution layer Engine API URLs.