Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add metadata for the Holesky network #5337

Merged
merged 3 commits into from
Sep 8, 2023
Merged

Add metadata for the Holesky network #5337

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
75f126c
Add metadata for the Holesky network
zah Aug 22, 2023
87867fd
Add copyright banner to the new Nim module
zah Sep 7, 2023
89d2a98
Working version
zah Sep 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@
path = vendor/nim-chronos
url = https://github.com/status-im/nim-chronos.git
ignore = untracked
branch = master
branch = nimbus-v23.9.0
[submodule "vendor/nim-chronicles"]
path = vendor/nim-chronicles
url = https://github.com/status-im/nim-chronicles.git
Expand Down Expand Up @@ -220,3 +220,8 @@
url = https://github.com/arnetheduck/nim-results.git
ignore = untracked
branch = master
[submodule "vendor/holesky"]
path = vendor/holesky
url = https://github.com/eth-clients/holesky
ignore = untracked
branch = main
1 change: 1 addition & 0 deletions beacon_chain/el/el_manager.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1402,6 +1402,7 @@ proc exchangeConfigWithSingleEL(m: ELManager, connection: ELConnection) {.async.
of rinkeby: 4.Quantity
of goerli: 5.Quantity
of sepolia: 11155111.Quantity # https://chainid.network/
of holesky: 17000.Quantity
if expectedChain != providerChain:
warn "The specified EL client is connected to a different chain",
url = connection.engineUrl,
Expand Down
239 changes: 176 additions & 63 deletions beacon_chain/networking/network_metadata.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import
web3/[ethtypes, conversions],
chronicles,
eth/common/eth_types_json_serialization,
../spec/eth2_ssz_serialization
../spec/[eth2_ssz_serialization, forks]

# TODO(zah):
# We can compress the embedded states with snappy before embedding them here.
Expand Down Expand Up @@ -45,6 +45,29 @@ type
rinkeby
goerli
sepolia
holesky

GenesisMetadataKind* = enum
NoGenesis
UserSuppliedFile
BakedIn
BakedInUrl

DownloadInfo* = object
url: string
digest: Eth2Digest

GenesisMetadata* = object
case kind*: GenesisMetadataKind
of NoGenesis:
discard
of UserSuppliedFile:
path*: string
of BakedIn:
networkName*: string
of BakedInUrl:
url*: string
digest*: Eth2Digest

Eth2NetworkMetadata* = object
case incompatible*: bool
Expand All @@ -62,22 +85,13 @@ type
depositContractBlock*: uint64
depositContractBlockHash*: Eth2Digest

# `genesisData` will have `len == 0` for networks with a still
# unknown genesis state.
when incbinEnabled:
genesisData*: seq[byte]
else:
genesisData*: string

genesis*: GenesisMetadata
genesisDepositsSnapshot*: string
else:
incompatibilityDesc*: string

template genesisBytes*(metadata: Eth2NetworkMetadata): auto =
when incbinEnabled:
metadata.genesisData
else:
metadata.genesisData.toOpenArrayByte(0, metadata.genesisData.high)
func hasGenesis*(metadata: Eth2NetworkMetadata): bool =
metadata.genesis.kind != NoGenesis

proc readBootstrapNodes*(path: string): seq[string] {.raises: [IOError].} =
# Read a list of ENR values from a YAML file containing a flat list of entries
Expand All @@ -98,7 +112,11 @@ proc readBootEnr*(path: string): seq[string] {.raises: [IOError].} =
@[]

proc loadEth2NetworkMetadata*(
path: string, eth1Network = none(Eth1Network), loadGenesis = true):
path: string,
eth1Network = none(Eth1Network),
isCompileTime = false,
downloadGenesisFrom = none(DownloadInfo),
useBakedInGenesis = none(string)):
Eth2NetworkMetadata {.raises: [CatchableError].} =
# Load data in eth2-networks format
# https://github.com/eth-clients/eth2-networks
Expand Down Expand Up @@ -166,11 +184,6 @@ proc loadEth2NetworkMetadata*(
readBootstrapNodes(bootstrapNodesPath) &
readBootEnr(bootEnrPath))

genesisData = if loadGenesis and fileExists(genesisPath):
readFile(genesisPath)
else:
""

genesisDepositsSnapshot = if fileExists(genesisDepositsSnapshotPath):
readFile(genesisDepositsSnapshotPath)
else:
Expand All @@ -183,9 +196,18 @@ proc loadEth2NetworkMetadata*(
bootstrapNodes: bootstrapNodes,
depositContractBlock: depositContractBlock,
depositContractBlockHash: depositContractBlockHash,
genesisData:
when incbinEnabled: toBytes genesisData
else: genesisData,
genesis:
if downloadGenesisFrom.isSome:
GenesisMetadata(kind: BakedInUrl,
url: downloadGenesisFrom.get.url,
digest: downloadGenesisFrom.get.digest)
elif useBakedInGenesis.isSome:
GenesisMetadata(kind: BakedIn, networkName: useBakedInGenesis.get)
elif fileExists(genesisPath) and not isCompileTime:
GenesisMetadata(kind: UserSuppliedFile, path: genesisPath)
else:
GenesisMetadata(kind: NoGenesis)
,
genesisDepositsSnapshot: genesisDepositsSnapshot)

except PresetIncompatibleError as err:
Expand All @@ -195,10 +217,14 @@ proc loadEth2NetworkMetadata*(
proc loadCompileTimeNetworkMetadata(
path: string,
eth1Network = none(Eth1Network),
loadGenesis = true): Eth2NetworkMetadata {.raises: [].} =
useBakedInGenesis = none(string),
downloadGenesisFrom = none(DownloadInfo)): Eth2NetworkMetadata {.raises: [].} =
if fileExists(path & "/config.yaml"):
try:
result = loadEth2NetworkMetadata(path, eth1Network, loadGenesis)
result = loadEth2NetworkMetadata(path, eth1Network,
isCompileTime = true,
downloadGenesisFrom = downloadGenesisFrom,
useBakedInGenesis = useBakedInGenesis)
if result.incompatible:
macros.error "The current build is misconfigured. " &
"Attempt to load an incompatible network metadata: " &
Expand All @@ -209,27 +235,36 @@ proc loadCompileTimeNetworkMetadata(
macros.error "config.yaml not found for network '" & path

when const_preset == "gnosis":
import stew/assign2

when incbinEnabled:
let
gnosisGenesis {.importc: "gnosis_mainnet_genesis".}: ptr UncheckedArray[byte]
gnosisGenesisSize {.importc: "gnosis_mainnet_genesis_size".}: int
gnosisGenesis* {.importc: "gnosis_mainnet_genesis".}: ptr UncheckedArray[byte]
gnosisGenesisSize* {.importc: "gnosis_mainnet_genesis_size".}: int

chiadoGenesis {.importc: "gnosis_chiado_genesis".}: ptr UncheckedArray[byte]
chiadoGenesisSize {.importc: "gnosis_chiado_genesis_size".}: int
chiadoGenesis* {.importc: "gnosis_chiado_genesis".}: ptr UncheckedArray[byte]
chiadoGenesisSize* {.importc: "gnosis_chiado_genesis_size".}: int

# let `.incbin` in assembly file find the binary file through search path
{.passc: "-I" & vendorDir.}
{.compile: "network_metadata_gnosis.S".}

else:
const
gnosisGenesis* = slurp(
vendorDir & "/gnosis-chain-configs/mainnet/genesis.ssz")

chiadoGenesis* = slurp(
vendorDir & "/gnosis-chain-configs/chiado/genesis.ssz")

const
gnosisMetadata = loadCompileTimeNetworkMetadata(
vendorDir & "/gnosis-chain-configs/mainnet",
none(Eth1Network), not incbinEnabled)
none(Eth1Network),
useBakedInGenesis = some "gnosis")

chiadoMetadata = loadCompileTimeNetworkMetadata(
vendorDir & "/gnosis-chain-configs/chiado",
none(Eth1Network), not incbinEnabled)
none(Eth1Network),
useBakedInGenesis = some "chiado")

static:
for network in [gnosisMetadata, chiadoMetadata]:
Expand All @@ -242,38 +277,62 @@ when const_preset == "gnosis":
doAssert network.cfg.DENEB_FORK_EPOCH == FAR_FUTURE_EPOCH

elif const_preset == "mainnet":
import stew/assign2

when incbinEnabled:
# Nim is very inefficent at loading large constants from binary files so we
# use this trick instead which saves significant amounts of compile time
let
mainnetGenesis {.importc: "eth2_mainnet_genesis".}: ptr UncheckedArray[byte]
mainnetGenesisSize {.importc: "eth2_mainnet_genesis_size".}: int
mainnetGenesis* {.importc: "eth2_mainnet_genesis".}: ptr UncheckedArray[byte]
mainnetGenesisSize* {.importc: "eth2_mainnet_genesis_size".}: int

praterGenesis {.importc: "eth2_goerli_genesis".}: ptr UncheckedArray[byte]
praterGenesisSize {.importc: "eth2_goerli_genesis_size".}: int
praterGenesis* {.importc: "eth2_goerli_genesis".}: ptr UncheckedArray[byte]
praterGenesisSize* {.importc: "eth2_goerli_genesis_size".}: int

sepoliaGenesis {.importc: "eth2_sepolia_genesis".}: ptr UncheckedArray[byte]
sepoliaGenesisSize {.importc: "eth2_sepolia_genesis_size".}: int
sepoliaGenesis* {.importc: "eth2_sepolia_genesis".}: ptr UncheckedArray[byte]
sepoliaGenesisSize* {.importc: "eth2_sepolia_genesis_size".}: int

# let `.incbin` in assembly file find the binary file through search path
{.passc: "-I" & vendorDir.}
{.compile: "network_metadata_mainnet.S".}

else:
const
mainnetGenesis* = slurp(
vendorDir & "/eth2-networks/shared/mainnet/genesis.ssz")

praterGenesis* = slurp(
vendorDir & "/eth2-networks/shared/prater/genesis.ssz")

sepoliaGenesis* = slurp(
vendorDir & "/sepolia/bepolia/genesis.ssz")

const
mainnetMetadata = loadCompileTimeNetworkMetadata(
vendorDir & "/eth2-networks/shared/mainnet", some mainnet, not incbinEnabled)
vendorDir & "/eth2-networks/shared/mainnet",
some mainnet,
useBakedInGenesis = some "mainnet")

praterMetadata = loadCompileTimeNetworkMetadata(
vendorDir & "/eth2-networks/shared/prater", some goerli, not incbinEnabled)
vendorDir & "/eth2-networks/shared/prater",
some goerli,
useBakedInGenesis = some "prater")

holeskyMetadata = loadCompileTimeNetworkMetadata(
vendorDir & "/holesky/custom_config_data",
some holesky,
downloadGenesisFrom = some DownloadInfo(
url: "https://github.com/status-im/nimbus-eth2/releases/download/v23.8.0/holesky-genesis.ssz.snappy-framed",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the extension for framed snappy is generally .sz: https://github.com/google/snappy/blob/main/framing_format.txt

digest: Eth2Digest.fromHex "0x76631cd0b9ddc5b2c766b496e23f16759ce1181446a4efb40e5540cd15b78a07"))

sepoliaMetadata = loadCompileTimeNetworkMetadata(
vendorDir & "/sepolia/bepolia", some sepolia, not incbinEnabled)
vendorDir & "/sepolia/bepolia",
some sepolia,
useBakedInGenesis = some "sepolia")

static:
for network in [mainnetMetadata, praterMetadata, sepoliaMetadata]:
for network in [mainnetMetadata, praterMetadata, sepoliaMetadata, holeskyMetadata]:
checkForkConsistency(network.cfg)

for network in [mainnetMetadata, praterMetadata, sepoliaMetadata]:
for network in [mainnetMetadata, praterMetadata, sepoliaMetadata, holeskyMetadata]:
doAssert network.cfg.ALTAIR_FORK_EPOCH < FAR_FUTURE_EPOCH
doAssert network.cfg.BELLATRIX_FORK_EPOCH < FAR_FUTURE_EPOCH
doAssert network.cfg.CAPELLA_FORK_EPOCH < FAR_FUTURE_EPOCH
Expand All @@ -295,40 +354,30 @@ proc getMetadataForNetwork*(
if networkName == "ropsten":
warn "Ropsten is unsupported; https://blog.ethereum.org/2022/11/30/ropsten-shutdown-announcement suggests migrating to Goerli or Sepolia"

template withGenesis(metadata, genesis: untyped): untyped =
when incbinEnabled:
var tmp = metadata
case tmp.incompatible
of false:
assign(tmp.genesisData, genesis.toOpenArray(0, `genesis Size` - 1))
of true:
raiseAssert "Unreachable" # `loadCompileTimeNetworkMetadata`
tmp
else:
metadata

let metadata =
when const_preset == "gnosis":
case toLowerAscii(networkName)
of "gnosis":
withGenesis(gnosisMetadata, gnosisGenesis)
gnosisMetadata
of "gnosis-chain":
warn "`--network:gnosis-chain` is deprecated, " &
"use `--network:gnosis` instead"
withGenesis(gnosisMetadata, gnosisGenesis)
gnosisMetadata
of "chiado":
withGenesis(chiadoMetadata, chiadoGenesis)
chiadoMetadata
else:
loadRuntimeMetadata()

elif const_preset == "mainnet":
case toLowerAscii(networkName)
of "mainnet":
withGenesis(mainnetMetadata, mainnetGenesis)
mainnetMetadata
of "prater", "goerli":
withGenesis(praterMetadata, praterGenesis)
praterMetadata
of "holesky":
holeskyMetadata
of "sepolia":
withGenesis(sepoliaMetadata, sepoliaGenesis)
sepoliaMetadata
else:
loadRuntimeMetadata()

Expand Down Expand Up @@ -373,3 +422,67 @@ proc getRuntimeConfig*(
of true:
# `getMetadataForNetwork` / `loadCompileTimeNetworkMetadata`
raiseAssert "Unreachable"

template bakedInGenesisStateAsBytes(networkName: untyped): untyped =
when incbinEnabled:
`networkName Genesis`.toOpenArray(0, `networkName GenesisSize` - 1)
else:
`networkName Genesis`.toOpenArrayByte(0, `networkName Genesis`.high)

const
availableOnlyInMainnetBuild =
"Baked-in genesis states for the official Ethereum " &
"networks are available only in the mainnet build of Nimbus"

availableOnlyInGnosisBuild =
"Baked-in genesis states for the Gnosis network " &
"are available only in the gnosis build of Nimbus"

when const_preset in ["mainnet", "gnosis"]:
template bakedBytes*(metadata: GenesisMetadata): auto =
case metadata.networkName
of "mainnet":
when const_preset == "mainnet":
bakedInGenesisStateAsBytes mainnet
else:
raiseAssert availableOnlyInMainnetBuild
of "prater":
when const_preset == "mainnet":
bakedInGenesisStateAsBytes prater
else:
raiseAssert availableOnlyInMainnetBuild
of "sepolia":
when const_preset == "mainnet":
bakedInGenesisStateAsBytes sepolia
else:
raiseAssert availableOnlyInMainnetBuild
of "gnosis":
when const_preset == "gnosis":
bakedInGenesisStateAsBytes gnosis
else:
raiseAssert availableOnlyInGnosisBuild
of "chiado":
when const_preset == "gnosis":
bakedInGenesisStateAsBytes chiado
else:
raiseAssert availableOnlyInGnosisBuild
else:
raiseAssert "The baked network metadata should use one of the name above"

func bakedGenesisValidatorsRoot*(metadata: Eth2NetworkMetadata): Opt[Eth2Digest] =
if metadata.genesis.kind == BakedIn:
try:
let header = SSZ.decode(
toOpenArray(metadata.genesis.bakedBytes, 0, sizeof(BeaconStateHeader) - 1),
BeaconStateHeader)
Opt.some header.genesis_validators_root
except SerializationError as err:
raiseAssert "Invalid baken-in genesis state"
else:
Opt.none Eth2Digest
else:
func bakedBytes*(metadata: GenesisMetadata): seq[byte] =
raiseAssert "Baked genesis states are not available in the current build mode"

func bakedGenesisValidatorsRoot*(metadata: Eth2NetworkMetadata): Opt[Eth2Digest] =
Opt.none Eth2Digest
Loading