Skip to content

Commit

Permalink
clear BrokenClock status if Nimbus extensions no longer supported
Browse files Browse the repository at this point in the history
When BN clock is out of sync, VC sets BN status to `BrokenClock`. It is
only reset to `Offline` after restoring time sync. However, if VC fails
encounters an error while checking time, Nimbus extensions are assumed
to be unavailable and the BN is no longer checked for having a synced
clock. This means it is never reset back to `Offline` if errors start
occurring _after_ BN is already set to `BrokenClock`. This could be
because BN is changed from Nimbus to an alternative implementation,
or due to intermittent connection issues.

Ensure that BN status is reset back to `Offline` when Nimbus extensions
are disabled to ensure eventual connection recovery.
  • Loading branch information
etan-status committed Jan 25, 2024
1 parent d08e558 commit 91810e7
Showing 1 changed file with 14 additions and 11 deletions.
25 changes: 14 additions & 11 deletions beacon_chain/validator_client/fallback_service.nim
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,14 @@ proc checkOffsetStatus(node: BeaconNodeServerRef, offset: TimeOffset) =
"Beacon node has acceptable time offset")
node.updateStatus(RestBeaconNodeStatus.Offline, failure)

proc disableNimbusExtensions(node: BeaconNodeServerRef) =
node.features.incl(RestBeaconNodeFeature.NoNimbusExtensions)
if node.status == RestBeaconNodeStatus.BrokenClock:
let failure = ApiNodeFailure.init(ApiFailure.NoError,
"disableNimbusExtensions()", node, 200,
"Nimbus extensions no longer available")
node.updateStatus(RestBeaconNodeStatus.Offline, failure)

proc runTimeMonitor(service: FallbackServiceRef,
node: BeaconNodeServerRef) {.async.} =
const NimbusExtensionsLog = "Beacon node does not support Nimbus extensions"
Expand All @@ -398,10 +406,8 @@ proc runTimeMonitor(service: FallbackServiceRef,

let tres =
try:
let
delay = vc.processingDelay.valueOr: ZeroDuration
res = await node.client.getTimeOffset(delay)
Opt.some(res)
let delay = vc.processingDelay.valueOr: ZeroDuration
await node.client.getTimeOffset(delay)
except RestResponseError as exc:
case exc.status
of 400:
Expand All @@ -412,26 +418,23 @@ proc runTimeMonitor(service: FallbackServiceRef,
notice NimbusExtensionsLog, status = $exc.status,
reason = $exc.msg, error_message = $exc.message
# Exiting loop
node.features.incl(RestBeaconNodeFeature.NoNimbusExtensions)
node.disableNimbusExtensions()
return
except RestError as exc:
debug "Unable to obtain beacon node's time offset", reason = $exc.msg
notice NimbusExtensionsLog
node.features.incl(RestBeaconNodeFeature.NoNimbusExtensions)
node.disableNimbusExtensions()
return
except CancelledError as exc:
raise exc
except CatchableError as exc:
warn "An unexpected error occurred while asking for time offset",
reason = $exc.msg, error = $exc.name
notice NimbusExtensionsLog
node.features.incl(RestBeaconNodeFeature.NoNimbusExtensions)
node.disableNimbusExtensions()
return

if tres.isSome():
checkOffsetStatus(node, TimeOffset.init(tres.get()))
else:
debug "Beacon node's time offset was not updated"
checkOffsetStatus(node, TimeOffset.init(tres))

await service.waitForNextSlot()

Expand Down

0 comments on commit 91810e7

Please sign in to comment.