Skip to content

Commit b89a298

Browse files
StebalienStebalien
authored
feat(shed): add commands for importing/exporting datastore snapshots (#12685)
This makes it possible to import/export arbitrary datastore snapshots.
1 parent 4eb45ca commit b89a298

File tree

5 files changed

+440
-0
lines changed

5 files changed

+440
-0
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
- Implement new `lotus f3` CLI commands to list F3 participants, dump manifest, get/list finality certificates and check the F3 status. ([filecoin-project/lotus#12617](https://github.com/filecoin-project/lotus/pull/12617), [filecoin-project/lotus#12627](https://github.com/filecoin-project/lotus/pull/12627))
1414
- Return a `"data"` field on the `"error"` returned from RPC when `eth_call` and `eth_estimateGas` APIs encounter `execution reverted` errors. ([filecoin-project/lotus#12553](https://github.com/filecoin-project/lotus/pull/12553))
1515
- Implement `EthGetTransactionByBlockNumberAndIndex` (`eth_getTransactionByBlockNumberAndIndex`) and `EthGetTransactionByBlockHashAndIndex` (`eth_getTransactionByBlockHashAndIndex`) methods. ([filecoin-project/lotus#12618](https://github.com/filecoin-project/lotus/pull/12618))
16+
- Add a set of `lotus-shed datastore` commands for importing, exporting, and clearing parts of the datastore ([filecoin-project/lotus#12685](https://github.com/filecoin-project/lotus/pull/12685)):
1617

1718
## Bug Fixes
1819
- Fix a bug in the `lotus-shed indexes backfill-events` command that may result in either duplicate events being backfilled where there are existing events (such an operation *should* be idempotent) or events erroneously having duplicate `logIndex` values when queried via ETH APIs. ([filecoin-project/lotus#12567](https://github.com/filecoin-project/lotus/pull/12567))

‎cmd/lotus-shed/datastore.go

+277
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bufio"
55
"context"
66
"encoding/json"
7+
"errors"
78
"fmt"
89
"io"
910
"os"
@@ -17,10 +18,12 @@ import (
1718
"github.com/mitchellh/go-homedir"
1819
"github.com/polydawn/refmt/cbor"
1920
"github.com/urfave/cli/v2"
21+
cbg "github.com/whyrusleeping/cbor-gen"
2022
"go.uber.org/multierr"
2123
"golang.org/x/xerrors"
2224

2325
lcli "github.com/filecoin-project/lotus/cli"
26+
"github.com/filecoin-project/lotus/cmd/lotus-shed/shedgen"
2427
"github.com/filecoin-project/lotus/lib/backupds"
2528
"github.com/filecoin-project/lotus/node/repo"
2629
)
@@ -34,6 +37,9 @@ var datastoreCmd = &cli.Command{
3437
datastoreGetCmd,
3538
datastoreRewriteCmd,
3639
datastoreVlog2CarCmd,
40+
datastoreImportCmd,
41+
datastoreExportCmd,
42+
datastoreClearCmd,
3743
},
3844
}
3945

@@ -106,6 +112,98 @@ var datastoreListCmd = &cli.Command{
106112
},
107113
}
108114

115+
var datastoreClearCmd = &cli.Command{
116+
Name: "clear",
117+
Description: "Clear a part or all of the given datastore.",
118+
Flags: []cli.Flag{
119+
&cli.StringFlag{
120+
Name: "repo-type",
121+
Usage: "node type (FullNode, StorageMiner, Worker, Wallet)",
122+
Value: "FullNode",
123+
},
124+
&cli.StringFlag{
125+
Name: "prefix",
126+
Usage: "only delete key/values with the given prefix",
127+
Value: "",
128+
},
129+
&cli.BoolFlag{
130+
Name: "really-do-it",
131+
Usage: "must be specified for the action to take effect",
132+
},
133+
},
134+
ArgsUsage: "[namespace]",
135+
Action: func(cctx *cli.Context) (_err error) {
136+
if cctx.NArg() != 2 {
137+
return xerrors.Errorf("requires 2 arguments: the datastore prefix")
138+
}
139+
namespace := cctx.Args().Get(0)
140+
141+
r, err := repo.NewFS(cctx.String("repo"))
142+
if err != nil {
143+
return xerrors.Errorf("opening fs repo: %w", err)
144+
}
145+
146+
exists, err := r.Exists()
147+
if err != nil {
148+
return err
149+
}
150+
if !exists {
151+
return xerrors.Errorf("lotus repo doesn't exist")
152+
}
153+
154+
lr, err := r.Lock(repo.NewRepoTypeFromString(cctx.String("repo-type")))
155+
if err != nil {
156+
return err
157+
}
158+
defer lr.Close() //nolint:errcheck
159+
160+
ds, err := lr.Datastore(cctx.Context, namespace)
161+
if err != nil {
162+
return err
163+
}
164+
defer func() {
165+
_err = multierr.Append(_err, ds.Close())
166+
}()
167+
168+
dryRun := !cctx.Bool("really-do-it")
169+
170+
query, err := ds.Query(cctx.Context, dsq.Query{
171+
Prefix: cctx.String("prefix"),
172+
})
173+
if err != nil {
174+
return err
175+
}
176+
defer query.Close() //nolint:errcheck
177+
178+
batch, err := ds.Batch(cctx.Context)
179+
if err != nil {
180+
return xerrors.Errorf("failed to create a datastore batch: %w", err)
181+
}
182+
183+
for res, ok := query.NextSync(); ok; res, ok = query.NextSync() {
184+
if res.Error != nil {
185+
return xerrors.Errorf("failed to read from datastore: %w", res.Error)
186+
}
187+
_, _ = fmt.Fprintf(cctx.App.Writer, "deleting: %q\n", res.Key)
188+
if !dryRun {
189+
if err := batch.Delete(cctx.Context, datastore.NewKey(res.Key)); err != nil {
190+
return xerrors.Errorf("failed to delete %q: %w", res.Key, err)
191+
}
192+
}
193+
}
194+
195+
if !dryRun {
196+
if err := batch.Commit(cctx.Context); err != nil {
197+
return xerrors.Errorf("failed to flush the batch: %w", err)
198+
}
199+
} else {
200+
_, _ = fmt.Fprintln(cctx.App.Writer, "NOTE: dry run complete, re-run with --really-do-it to actually delete this state.")
201+
}
202+
203+
return nil
204+
},
205+
}
206+
109207
var datastoreGetCmd = &cli.Command{
110208
Name: "get",
111209
Description: "list datastore keys",
@@ -158,6 +256,185 @@ var datastoreGetCmd = &cli.Command{
158256
},
159257
}
160258

259+
var datastoreExportCmd = &cli.Command{
260+
Name: "export",
261+
Description: "Export part or all of the specified datastore, appending to the specified datastore snapshot.",
262+
Flags: []cli.Flag{
263+
&cli.StringFlag{
264+
Name: "repo-type",
265+
Usage: "node type (FullNode, StorageMiner, Worker, Wallet)",
266+
Value: "FullNode",
267+
},
268+
&cli.StringFlag{
269+
Name: "prefix",
270+
Usage: "export only keys with the given prefix",
271+
Value: "",
272+
},
273+
},
274+
ArgsUsage: "[namespace filename]",
275+
Action: func(cctx *cli.Context) (_err error) {
276+
if cctx.NArg() != 2 {
277+
return xerrors.Errorf("requires 2 arguments: the datastore prefix and the filename to which the snapshot will be written")
278+
}
279+
namespace := cctx.Args().Get(0)
280+
fname := cctx.Args().Get(1)
281+
282+
snapshot, err := os.OpenFile(fname, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.ModePerm)
283+
if err != nil {
284+
return xerrors.Errorf("failed to open snapshot: %w", err)
285+
}
286+
defer func() {
287+
_err = multierr.Append(_err, snapshot.Close())
288+
}()
289+
290+
r, err := repo.NewFS(cctx.String("repo"))
291+
if err != nil {
292+
return xerrors.Errorf("opening fs repo: %w", err)
293+
}
294+
295+
exists, err := r.Exists()
296+
if err != nil {
297+
return err
298+
}
299+
if !exists {
300+
return xerrors.Errorf("lotus repo doesn't exist")
301+
}
302+
303+
lr, err := r.Lock(repo.NewRepoTypeFromString(cctx.String("repo-type")))
304+
if err != nil {
305+
return err
306+
}
307+
defer lr.Close() //nolint:errcheck
308+
309+
ds, err := lr.Datastore(cctx.Context, namespace)
310+
if err != nil {
311+
return err
312+
}
313+
defer func() {
314+
_err = multierr.Append(_err, ds.Close())
315+
}()
316+
317+
query, err := ds.Query(cctx.Context, dsq.Query{
318+
Prefix: cctx.String("prefix"),
319+
})
320+
if err != nil {
321+
return err
322+
}
323+
324+
bufWriter := bufio.NewWriter(snapshot)
325+
snapshotWriter := cbg.NewCborWriter(bufWriter)
326+
for res, ok := query.NextSync(); ok; res, ok = query.NextSync() {
327+
if res.Error != nil {
328+
return xerrors.Errorf("failed to read from datastore: %w", res.Error)
329+
}
330+
331+
entry := shedgen.DatastoreEntry{
332+
Key: []byte(res.Key),
333+
Value: res.Value,
334+
}
335+
336+
_, _ = fmt.Fprintf(cctx.App.Writer, "exporting: %q\n", res.Key)
337+
if err := entry.MarshalCBOR(snapshotWriter); err != nil {
338+
return xerrors.Errorf("failed to write %q to snapshot: %w", res.Key, err)
339+
}
340+
}
341+
if err := bufWriter.Flush(); err != nil {
342+
return xerrors.Errorf("failed to flush snapshot: %w", err)
343+
}
344+
345+
return nil
346+
},
347+
}
348+
349+
var datastoreImportCmd = &cli.Command{
350+
Name: "import",
351+
Flags: []cli.Flag{
352+
&cli.StringFlag{
353+
Name: "repo-type",
354+
Usage: "node type (FullNode, StorageMiner, Worker, Wallet)",
355+
Value: "FullNode",
356+
},
357+
},
358+
Description: "Import the specified datastore snapshot.",
359+
ArgsUsage: "[namespace filename]",
360+
Action: func(cctx *cli.Context) (_err error) {
361+
if cctx.NArg() != 2 {
362+
return xerrors.Errorf("requires 2 arguments: the datastore prefix and the filename of the snapshot to import")
363+
}
364+
namespace := cctx.Args().Get(0)
365+
fname := cctx.Args().Get(1)
366+
367+
snapshot, err := os.Open(fname)
368+
if err != nil {
369+
return xerrors.Errorf("failed to open snapshot: %w", err)
370+
}
371+
defer snapshot.Close() //nolint:errcheck
372+
373+
r, err := repo.NewFS(cctx.String("repo"))
374+
if err != nil {
375+
return xerrors.Errorf("opening fs repo: %w", err)
376+
}
377+
378+
exists, err := r.Exists()
379+
if err != nil {
380+
return err
381+
}
382+
if !exists {
383+
return xerrors.Errorf("lotus repo doesn't exist")
384+
}
385+
386+
lr, err := r.Lock(repo.NewRepoTypeFromString(cctx.String("repo-type")))
387+
if err != nil {
388+
return err
389+
}
390+
defer lr.Close() //nolint:errcheck
391+
392+
ds, err := lr.Datastore(cctx.Context, namespace)
393+
if err != nil {
394+
return err
395+
}
396+
defer func() {
397+
_err = multierr.Append(_err, ds.Close())
398+
}()
399+
400+
batch, err := ds.Batch(cctx.Context)
401+
if err != nil {
402+
return err
403+
}
404+
405+
dryRun := !cctx.Bool("really-do-it")
406+
407+
snapshotReader := cbg.NewCborReader(bufio.NewReader(snapshot))
408+
for {
409+
var entry shedgen.DatastoreEntry
410+
if err := entry.UnmarshalCBOR(snapshotReader); err != nil {
411+
if errors.Is(err, io.EOF) {
412+
break
413+
}
414+
return xerrors.Errorf("failed to read entry from snapshot: %w", err)
415+
}
416+
417+
_, _ = fmt.Fprintf(cctx.App.Writer, "importing: %q\n", string(entry.Key))
418+
419+
if !dryRun {
420+
key := datastore.NewKey(string(entry.Key))
421+
if err := batch.Put(cctx.Context, key, entry.Value); err != nil {
422+
return xerrors.Errorf("failed to put %q: %w", key, err)
423+
}
424+
}
425+
}
426+
427+
if !dryRun {
428+
if err := batch.Commit(cctx.Context); err != nil {
429+
return xerrors.Errorf("failed to commit batch: %w", err)
430+
}
431+
} else {
432+
_, _ = fmt.Fprintln(cctx.App.Writer, "NOTE: dry run complete, re-run with --really-do-it to actually import the datastore snapshot, overwriting any conflicting state.")
433+
}
434+
return nil
435+
},
436+
}
437+
161438
var datastoreBackupCmd = &cli.Command{
162439
Name: "backup",
163440
Description: "manage datastore backups",

0 commit comments

Comments
 (0)