Skip to content

Commit 2d15a99

Browse files
okdasOlshansk
authored and
dylanlott
committed
[debug client] Limit key import concurrency to 4 (#591)
## Description <!-- REMOVE this comment block after following the instructions 1. Add a summary of the change including: motivation, reasons, context, dependencies, etc... 2. If applicable, specify the key files that should be looked at. --> `make localnet_client_debug` currently fails on machines with low~ish amount of RAM. Key import loads 999 validator keys quickly, but cryptographic calculations are so intense on memory it only imports 250 keys before dying OOM on my 16Gi virtual machine. ## Type of change Please mark the relevant option(s): - [ ] New feature, functionality or library - [x] Bug fix - [ ] Code health or cleanup - [ ] Major breaking change - [ ] Documentation - [ ] Other <!-- add details here if it a different type of change --> ## List of changes <!-- REMOVE this comment block after following the instructions List out all the changes made --> - Wrapped key import part into concurrency limiter (https://github.com/korovkin/limiter) ## Testing - [x] `make develop_test` - [x] `make localnet_client_debug` - [x] `make client_start && make client_connect` <!-- REMOVE this comment block after following the instructions If you added additional tests or infrastructure, describe it here. Bonus points for images and videos or gifs. --> ## Required Checklist - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have tested my changes using the available tooling - [x] I have updated the corresponding CHANGELOG ### If Applicable Checklist - [ ] I have updated the corresponding README(s); local and/or global - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] I have added, or updated, [mermaid.js](https://mermaid-js.github.io) diagrams in the corresponding README(s) - [ ] I have added, or updated, documentation and [mermaid.js](https://mermaid-js.github.io) diagrams in `shared/docs/*` if I updated `shared/*`README(s) --------- Co-authored-by: Daniel Olshansky <[email protected]>
1 parent 92845ea commit 2d15a99

File tree

7 files changed

+58
-22
lines changed

7 files changed

+58
-22
lines changed

‎app/client/doc/CHANGELOG.md

+4
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
## [0.0.0.22] - 2023-03-17
11+
12+
- Added a limit on concurrent key imports for debug client to avoid OOM.
13+
1014
## [0.0.0.21] - 2023-03-14
1115

1216
- Simplifies the debug CLI tooling by embedding private-keys.yaml manifest

‎app/client/keybase/debug/keystore.go

+10-10
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ package debug
55
import (
66
"fmt"
77
"os"
8-
"sync"
98

9+
"github.com/korovkin/limiter"
1010
"github.com/pokt-network/pocket/app/client/keybase"
1111
"github.com/pokt-network/pocket/build"
1212
"github.com/pokt-network/pocket/logger"
@@ -18,6 +18,11 @@ const (
1818
// NOTE: This is the number of validators in the private-keys.yaml manifest file
1919
numValidators = 999
2020
debugKeybaseSuffix = "/.pocket/keys"
21+
22+
// Increasing this number is linearly proportional to the amount of RAM required for the debug client to start and import
23+
// pre-generated keys into the keybase. Beware that might cause OOM and process can exit with 137 status code.
24+
// 4 threads takes 350-400MiB from a few tests which sounds acceptable.
25+
debugKeybaseImportConcurrencyLimit = 4
2126
)
2227

2328
var (
@@ -74,14 +79,10 @@ func initializeDebugKeybase() error {
7479
// Create a channel to receive errors from goroutines
7580
errCh := make(chan error, numValidators)
7681

77-
// Create a WaitGroup to wait for all goroutines to finish
78-
var wg sync.WaitGroup
79-
wg.Add(numValidators)
82+
limit := limiter.NewConcurrencyLimiter(debugKeybaseImportConcurrencyLimit)
8083

8184
for _, privHexString := range validatorKeysPairMap {
82-
go func(privHexString string) {
83-
defer wg.Done()
84-
85+
limit.Execute(func() {
8586
// Import the keys into the keybase with no passphrase or hint as these are for debug purposes
8687
keyPair, err := cryptoPocket.CreateNewKeyFromString(privHexString, "", "")
8788
if err != nil {
@@ -102,11 +103,10 @@ func initializeDebugKeybase() error {
102103
errCh <- err
103104
return
104105
}
105-
}(privHexString)
106+
})
106107
}
107108

108-
// Wait for all goroutines to finish
109-
wg.Wait()
109+
limit.WaitAndClose()
110110

111111
// Check if any goroutines returned an error
112112
select {

‎build/docs/CHANGELOG.md

+5
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
## [0.0.0.24] - 2023-03-17
11+
12+
- Added resource limits and PVC for debug client pod.
13+
- Added `procps` to the client pod image.
14+
1015
## [0.0.0.23] - 2023-03-14
1116

1217
- Simplifies the debug CLI tooling by embedding private-keys.yaml manifest

‎build/localnet/Tiltfile

+1
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ docker_build_with_restart(
133133
"client-image",
134134
root_dir,
135135
dockerfile_contents="""FROM debian:bullseye
136+
RUN apt-get update && apt-get install -y procps
136137
WORKDIR /
137138
COPY bin/client-linux /usr/local/bin/client
138139
""",

‎build/localnet/manifests/cli-client.yaml

+23
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,13 @@ spec:
3030
image: client-image
3131
imagePullPolicy: IfNotPresent
3232
command: ["sleep", "infinity"]
33+
resources:
34+
requests:
35+
memory: "64Mi"
36+
cpu: "250m"
37+
limits:
38+
memory: "512Mi"
39+
cpu: "4"
3340
env:
3441
- name: POCKET_P2P_IS_CLIENT_ONLY
3542
value: "true"
@@ -74,6 +81,8 @@ spec:
7481
name: config-volume
7582
- mountPath: /var/pocket/genesis
7683
name: genesis-volume
84+
- mountPath: /home/root/.pocket
85+
name: datadir
7786
volumes:
7887
- name: config-volume
7988
configMap:
@@ -83,3 +92,17 @@ spec:
8392
configMap:
8493
name: v1-localnet-genesis
8594
defaultMode: 420
95+
- name: datadir
96+
persistentVolumeClaim:
97+
claimName: pocket-v1-cli-client-datadir
98+
---
99+
apiVersion: v1
100+
kind: PersistentVolumeClaim
101+
metadata:
102+
name: pocket-v1-cli-client-datadir
103+
spec:
104+
accessModes:
105+
- ReadWriteOnce
106+
resources:
107+
requests:
108+
storage: 1Gi

‎go.mod

+5-4
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ require (
2727
github.com/getkin/kin-openapi v0.107.0
2828
github.com/jackc/pgconn v1.13.0
2929
github.com/jordanorelli/lexnum v0.0.0-20141216151731-460eeb125754
30+
github.com/korovkin/limiter v0.0.0-20230307205149-3d4b2b34c99d
3031
github.com/labstack/echo/v4 v4.9.1
3132
github.com/libp2p/go-libp2p v0.25.1
3233
github.com/libp2p/go-libp2p-pubsub v0.9.2
@@ -38,7 +39,7 @@ require (
3839
github.com/rs/zerolog v1.27.0
3940
github.com/spf13/cobra v1.6.0
4041
github.com/spf13/viper v1.13.0
41-
golang.org/x/term v0.3.0
42+
golang.org/x/term v0.5.0
4243
gopkg.in/yaml.v2 v2.4.0
4344
k8s.io/api v0.26.1
4445
k8s.io/apimachinery v0.26.1
@@ -70,7 +71,7 @@ require (
7071
github.com/prometheus/client_golang v1.14.0
7172
github.com/sirupsen/logrus v1.9.0 // indirect
7273
go.opencensus.io v0.23.0 // indirect
73-
golang.org/x/net v0.4.0 // indirect
74+
golang.org/x/net v0.7.0 // indirect
7475
gotest.tools v2.2.0+incompatible // indirect
7576
)
7677

@@ -213,8 +214,8 @@ require (
213214
github.com/valyala/bytebufferpool v1.0.0 // indirect
214215
github.com/valyala/fasttemplate v1.2.2 // indirect
215216
golang.org/x/mod v0.7.0 // indirect
216-
golang.org/x/sys v0.3.0 // indirect
217-
golang.org/x/text v0.5.0 // indirect
217+
golang.org/x/sys v0.5.0 // indirect
218+
golang.org/x/text v0.7.0 // indirect
218219
golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
219220
golang.org/x/tools v0.3.0 // indirect
220221
gopkg.in/ini.v1 v1.67.0 // indirect

‎go.sum

+10-8
Original file line numberDiff line numberDiff line change
@@ -462,6 +462,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv
462462
github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk=
463463
github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8=
464464
github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA=
465+
github.com/korovkin/limiter v0.0.0-20230307205149-3d4b2b34c99d h1:7CfsXfFpCG1wrUpuyOzG8+vpL1ZqH2goz23wZ9pboGE=
466+
github.com/korovkin/limiter v0.0.0-20230307205149-3d4b2b34c99d/go.mod h1:3NeYeWwAOTnDChps1fD7YGD/uWzp+tqmShgjhhMIHDM=
465467
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
466468
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
467469
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -941,8 +943,8 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx
941943
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
942944
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
943945
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
944-
golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
945-
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
946+
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
947+
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
946948
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
947949
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
948950
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1052,13 +1054,13 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
10521054
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
10531055
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
10541056
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1055-
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
1056-
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1057+
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
1058+
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
10571059
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
10581060
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
10591061
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
1060-
golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
1061-
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
1062+
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
1063+
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
10621064
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
10631065
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
10641066
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1067,8 +1069,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
10671069
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
10681070
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
10691071
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
1070-
golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
1071-
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
1072+
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
1073+
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
10721074
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
10731075
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
10741076
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

0 commit comments

Comments
 (0)