Skip to content

Commit d34aa3e

Browse files
FUDCoFUDCo
committed
feat: implement swingStore data export/import in support of state sync
Closes #6773
1 parent 0de845d commit d34aa3e

19 files changed

+1234
-337
lines changed

‎packages/SwingSet/misc-tools/extract-transcript-from-kerneldb.js

+4-5
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ if (!dirPath) {
1818
if (!isSwingStore(dirPath)) {
1919
throw Error(`${dirPath} does not appear to be a swingstore (no ./data.mdb)`);
2020
}
21-
const { kvStore, streamStore } = openSwingStore(dirPath).kernelStorage;
21+
const { kvStore, transcriptStore } = openSwingStore(dirPath).kernelStorage;
2222
function get(key) {
2323
return kvStore.get(key);
2424
}
@@ -98,7 +98,7 @@ if (!vatName) {
9898
fs.writeSync(fd, JSON.stringify(first));
9999
fs.writeSync(fd, '\n');
100100

101-
// The streamStore holds concatenated transcripts from all upgraded
101+
// The transcriptStore holds concatenated transcripts from all upgraded
102102
// versions. For each old version, it holds every delivery from
103103
// `startVat` through `stopVat`. For the current version, it holds
104104
// every delivery from `startVat` up through the last delivery
@@ -123,9 +123,8 @@ if (!vatName) {
123123
console.log(`${transcriptLength} transcript entries`);
124124

125125
let deliveryNum = 0;
126-
const transcriptStream = `transcript-${vatID}`;
127-
const stream = streamStore.readStream(transcriptStream, startPos, endPos);
128-
for (const entry of stream) {
126+
const transcript = transcriptStore.readTranscript(vatID, startPos, endPos);
127+
for (const entry of transcript) {
129128
// entry is JSON.stringify({ d, syscalls }), syscall is { d, response }
130129
const t = { transcriptNum, ...JSON.parse(entry) };
131130
// console.log(`t.${deliveryNum} : ${t}`);

‎packages/SwingSet/src/kernel/state/kernelKeeper.js

+5-9
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,7 @@ const enableKernelGC = true;
3838
* @typedef { import('../../types-external.js').KernelSlog } KernelSlog
3939
* @typedef { import('../../types-external.js').ManagerType } ManagerType
4040
* @typedef { import('../../types-external.js').SnapStore } SnapStore
41-
* @typedef { import('../../types-external.js').StreamPosition } StreamPosition
42-
* @typedef { import('../../types-external.js').StreamStore } StreamStore
41+
* @typedef { import('../../types-external.js').TranscriptStore } TranscriptStore
4342
* @typedef { import('../../types-external.js').VatKeeper } VatKeeper
4443
* @typedef { import('../../types-external.js').VatManager } VatManager
4544
*/
@@ -86,8 +85,6 @@ const enableKernelGC = true;
8685
// $vatSlot is one of: o+$NN/o-$NN/p+$NN/p-$NN/d+$NN/d-$NN
8786
// v$NN.c.$vatSlot = $kernelSlot = ko$NN/kp$NN/kd$NN
8887
// v$NN.nextDeliveryNum = $NN
89-
// v$NN.t.startPosition = $NN // inclusive
90-
// v$NN.t.endPosition = $NN // exclusive
9188
// v$NN.vs.$key = string
9289
// v$NN.meter = m$NN // XXX does this exist?
9390
// v$NN.reapInterval = $NN or 'never'
@@ -174,7 +171,7 @@ const FIRST_METER_ID = 1n;
174171
* @param {KernelSlog|null} kernelSlog
175172
*/
176173
export default function makeKernelKeeper(kernelStorage, kernelSlog) {
177-
const { kvStore, streamStore, snapStore } = kernelStorage;
174+
const { kvStore, transcriptStore, snapStore } = kernelStorage;
178175

179176
insistStorageAPI(kvStore);
180177

@@ -1297,11 +1294,11 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {
12971294
return found;
12981295
}
12991296
if (!kvStore.has(`${vatID}.o.nextID`)) {
1300-
initializeVatState(kvStore, streamStore, vatID);
1297+
initializeVatState(kvStore, transcriptStore, vatID);
13011298
}
13021299
const vk = makeVatKeeper(
13031300
kvStore,
1304-
streamStore,
1301+
transcriptStore,
13051302
kernelSlog,
13061303
vatID,
13071304
addKernelObject,
@@ -1334,8 +1331,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {
13341331
*/
13351332
function closeVatTranscript(vatID) {
13361333
insistVatID(vatID);
1337-
const transcriptStream = `transcript-${vatID}`;
1338-
streamStore.closeStream(transcriptStream);
1334+
transcriptStore.closeTranscript(vatID);
13391335
}
13401336

13411337
/**

‎packages/SwingSet/src/kernel/state/vatKeeper.js

+17-37
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ import { enumeratePrefixedKeys } from './storageHelper.js';
1818
* @typedef { import('../../types-external.js').ManagerOptions } ManagerOptions
1919
* @typedef { import('../../types-external.js').SnapStore } SnapStore
2020
* @typedef { import('../../types-external.js').SourceOfBundle } SourceOfBundle
21-
* @typedef { import('../../types-external.js').StreamPosition } StreamPosition
22-
* @typedef { import('../../types-external.js').StreamStore } StreamStore
21+
* @typedef { import('../../types-external.js').TranscriptStore } TranscriptStore
2322
* @typedef { import('../../types-external.js').VatManager } VatManager
2423
* @typedef { import('../../types-internal.js').RecordedVatOptions } RecordedVatOptions
2524
* @typedef { import('../../types-external.js').TranscriptEntry } TranscriptEntry
@@ -36,25 +35,24 @@ const FIRST_DEVICE_ID = 70n;
3635
* Establish a vat's state.
3736
*
3837
* @param {*} kvStore The key-value store in which the persistent state will be kept
39-
* @param {*} streamStore Accompanying stream store
38+
* @param {*} transcriptStore Accompanying transcript store
4039
* @param {string} vatID The vat ID string of the vat in question
4140
* TODO: consider making this part of makeVatKeeper
4241
*/
43-
export function initializeVatState(kvStore, streamStore, vatID) {
42+
export function initializeVatState(kvStore, transcriptStore, vatID) {
4443
kvStore.set(`${vatID}.o.nextID`, `${FIRST_OBJECT_ID}`);
4544
kvStore.set(`${vatID}.p.nextID`, `${FIRST_PROMISE_ID}`);
4645
kvStore.set(`${vatID}.d.nextID`, `${FIRST_DEVICE_ID}`);
4746
kvStore.set(`${vatID}.nextDeliveryNum`, `0`);
4847
kvStore.set(`${vatID}.incarnationNumber`, `1`);
49-
kvStore.set(`${vatID}.t.startPosition`, `${streamStore.STREAM_START}`);
50-
kvStore.set(`${vatID}.t.endPosition`, `${streamStore.STREAM_START}`);
48+
transcriptStore.initTranscript(vatID);
5149
}
5250

5351
/**
5452
* Produce a vat keeper for a vat.
5553
*
5654
* @param {KVStore} kvStore The keyValue store in which the persistent state will be kept
57-
* @param {StreamStore} streamStore Accompanying stream store, for the transcripts
55+
* @param {TranscriptStore} transcriptStore Accompanying transcript store, for the transcripts
5856
* @param {*} kernelSlog
5957
* @param {string} vatID The vat ID string of the vat in question
6058
* @param {*} addKernelObject Kernel function to add a new object to the kernel's
@@ -76,7 +74,7 @@ export function initializeVatState(kvStore, streamStore, vatID) {
7674
*/
7775
export function makeVatKeeper(
7876
kvStore,
79-
streamStore,
77+
transcriptStore,
8078
kernelSlog,
8179
vatID,
8280
addKernelObject,
@@ -94,7 +92,6 @@ export function makeVatKeeper(
9492
snapStore = undefined,
9593
) {
9694
insistVatID(vatID);
97-
const transcriptStream = `transcript-${vatID}`;
9895

9996
function getRequired(key) {
10097
const value = kvStore.get(key);
@@ -475,20 +472,12 @@ export function makeVatKeeper(
475472
/**
476473
* Generator function to return the vat's transcript, one entry at a time.
477474
*
478-
* @param {StreamPosition} [startPos] Optional position to begin reading from
475+
* @param {number} [startPos] Optional position to begin reading from
479476
*
480477
* @yields { TranscriptEntry } a stream of transcript entries
481478
*/
482479
function* getTranscript(startPos) {
483-
if (startPos === undefined) {
484-
startPos = Number(getRequired(`${vatID}.t.startPosition`));
485-
}
486-
const endPos = Number(getRequired(`${vatID}.t.endPosition`));
487-
for (const entry of streamStore.readStream(
488-
transcriptStream,
489-
/** @type { StreamPosition } */ (startPos),
490-
endPos,
491-
)) {
480+
for (const entry of transcriptStore.readTranscript(vatID, startPos)) {
492481
yield /** @type { TranscriptEntry } */ (JSON.parse(entry));
493482
}
494483
}
@@ -499,21 +488,13 @@ export function makeVatKeeper(
499488
* @param {object} entry The transcript entry to append.
500489
*/
501490
function addToTranscript(entry) {
502-
const oldPos = Number(getRequired(`${vatID}.t.endPosition`));
503-
const newPos = streamStore.writeStreamItem(
504-
transcriptStream,
505-
JSON.stringify(entry),
506-
oldPos,
507-
);
508-
kvStore.set(`${vatID}.t.endPosition`, `${newPos}`);
491+
transcriptStore.writeTranscriptItem(vatID, JSON.stringify(entry));
509492
}
510493

511-
/** @returns {StreamPosition} */
494+
/** @returns {number} */
512495
function getTranscriptEndPosition() {
513-
const endPosition =
514-
kvStore.get(`${vatID}.t.endPosition`) ||
515-
assert.fail('missing endPosition');
516-
return Number(endPosition);
496+
const { endPos } = transcriptStore.getCurrentTranscriptBounds(vatID);
497+
return endPos;
517498
}
518499

519500
function getSnapshotInfo() {
@@ -540,6 +521,7 @@ export function makeVatKeeper(
540521

541522
const endPosition = getTranscriptEndPosition();
542523
const info = await manager.makeSnapshot(endPosition, snapStore);
524+
transcriptStore.rolloverSpan(vatID);
543525
const {
544526
hash,
545527
uncompressedSize,
@@ -570,9 +552,7 @@ export function makeVatKeeper(
570552
if (snapStore) {
571553
snapStore.deleteVatSnapshots(vatID);
572554
}
573-
574-
const endPos = getRequired(`${vatID}.t.endPosition`);
575-
kvStore.set(`${vatID}.t.startPosition`, endPos);
555+
transcriptStore.rolloverSpan(vatID);
576556
}
577557

578558
function vatStats() {
@@ -584,9 +564,9 @@ export function makeVatKeeper(
584564
const objectCount = getCount(`${vatID}.o.nextID`, FIRST_OBJECT_ID);
585565
const promiseCount = getCount(`${vatID}.p.nextID`, FIRST_PROMISE_ID);
586566
const deviceCount = getCount(`${vatID}.d.nextID`, FIRST_DEVICE_ID);
587-
const startCount = Number(getRequired(`${vatID}.t.startPosition`));
588-
const endCount = Number(getRequired(`${vatID}.t.endPosition`));
589-
const transcriptCount = endCount - startCount;
567+
const { startPos, endPos } =
568+
transcriptStore.getCurrentTranscriptBounds(vatID);
569+
const transcriptCount = endPos - startPos;
590570

591571
// TODO: Fix the downstream JSON.stringify to allow the counts to be BigInts
592572
return harden({

‎packages/SwingSet/src/types-ambient.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -135,8 +135,7 @@
135135
*/
136136
/**
137137
* @typedef { import('@agoric/swing-store').KVStore } KVStore
138-
* @typedef { import('@agoric/swing-store').StreamStore } StreamStore
139-
* @typedef { import('@agoric/swing-store').StreamPosition } StreamPosition
138+
* @typedef { import('@agoric/swing-store').TranscriptStore } TranscriptStore
140139
* @typedef { import('@agoric/swing-store').SwingStore } SwingStore
141140
* @typedef { import('@agoric/swing-store').SwingStoreKernelStorage } SwingStoreKernelStorage
142141
* @typedef { import('@agoric/swing-store').SwingStoreHostStorage } SwingStoreHostStorage

‎packages/SwingSet/src/types-external.js

+2-3
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ export {};
205205
* vatSyscallHandler: unknown) => Promise<VatManager>,
206206
* } } VatManagerFactory
207207
* @typedef { { deliver: (delivery: VatDeliveryObject) => Promise<VatDeliveryResult>,
208-
* replayTranscript: (startPos: StreamPosition | undefined) => Promise<number?>,
208+
* replayTranscript: (startPos: number | undefined) => Promise<number?>,
209209
* makeSnapshot?: (endPos: number, ss: SnapStore) => Promise<SnapshotResult>,
210210
* shutdown: () => Promise<void>,
211211
* } } VatManager
@@ -277,8 +277,7 @@ export {};
277277
* @typedef { import('@agoric/swing-store').KVStore } KVStore
278278
* @typedef { import('@agoric/swing-store').SnapStore } SnapStore
279279
* @typedef { import('@agoric/swing-store').SnapshotResult } SnapshotResult
280-
* @typedef { import('@agoric/swing-store').StreamStore } StreamStore
281-
* @typedef { import('@agoric/swing-store').StreamPosition } StreamPosition
280+
* @typedef { import('@agoric/swing-store').TranscriptStore } TranscriptStore
282281
* @typedef { import('@agoric/swing-store').SwingStore } SwingStore
283282
* @typedef { import('@agoric/swing-store').SwingStoreKernelStorage } SwingStoreKernelStorage
284283
* @typedef { import('@agoric/swing-store').SwingStoreHostStorage } SwingStoreHostStorage

‎packages/SwingSet/test/snapshots/test-xsnap-store.js.md

+6-6
Original file line numberDiff line numberDiff line change
@@ -10,25 +10,25 @@ Generated by [AVA](https://avajs.dev).
1010
1111
{
1212
compressSeconds: 0,
13-
hash: 'c77e111a40bba743b29cc5bb3abfbd57c1a7143772117cb01da874f5cfe59415',
13+
hash: '8a0e3873976c50462d1b1dac59c912152b0e5cad5eeb9deca0ca64a087b4a873',
1414
rawSaveSeconds: 0,
15-
uncompressedSize: 168123,
15+
uncompressedSize: 167887,
1616
}
1717

1818
> after SES boot - sensitive to SES-shim, XS, and supervisor
1919
2020
{
2121
compressSeconds: 0,
22-
hash: '3a62221548f1d4e9da4961e4fe1cff13f7170c9abfade9171eecfa58d6028c34',
22+
hash: '253ffe0fb0b9e555119f046990d58fb85693e1170e681adb52211f49d94e37d0',
2323
rawSaveSeconds: 0,
24-
uncompressedSize: 777171,
24+
uncompressedSize: 775831,
2525
}
2626

2727
> after use of harden() - sensitive to SES-shim, XS, and supervisor
2828
2929
{
3030
compressSeconds: 0,
31-
hash: '2a595ba90b014defb02d02595bc89c1058a5186eab3b8c0ff0dda9d214104543',
31+
hash: '2bca522840c90b8a0519fe846642746f01f62205154877c2332ad8829b69aa3e',
3232
rawSaveSeconds: 0,
33-
uncompressedSize: 779331,
33+
uncompressedSize: 777983,
3434
}

‎packages/SwingSet/test/snapshots/test-xsnap-store.js.snap

0 Bytes
Binary file not shown.

‎packages/SwingSet/test/upgrade/test-upgrade.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ test('failed upgrade - explode', async t => {
481481
},
482482
};
483483

484-
const kernelStorage = initSwingStore().kernelStorage;
484+
const { kernelStorage } = initSwingStore();
485485
await initializeSwingset(config, [], kernelStorage);
486486
const c = await makeSwingsetController(kernelStorage);
487487
c.pinVatRoot('bootstrap');

‎packages/SwingSet/test/vat-warehouse/test-reload-snapshot.js

+3-2
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,9 @@ test('vat reload from snapshot', async t => {
3838
const snapshotInfo = snapStore.getSnapshotInfo(vatID);
3939

4040
const start = snapshotInfo ? snapshotInfo.endPos : 0;
41-
const endPosition = kernelStorage.kvStore.get(`${vatID}.t.endPosition`);
42-
const end = Number(endPosition);
41+
const bounds =
42+
kernelStorage.transcriptStore.getCurrentTranscriptBounds(vatID);
43+
const end = bounds.endPos;
4344
return [start, end];
4445
}
4546

‎packages/swing-store/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"@agoric/assert": "^0.5.1",
2222
"@agoric/internal": "^0.2.1",
2323
"better-sqlite3": "^7.5.0",
24+
"readline-transform": "^1.0.0",
2425
"tmp": "^0.2.1"
2526
},
2627
"devDependencies": {

‎packages/swing-store/src/hasher.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ import { assert } from '@agoric/assert';
33
import { createHash } from 'crypto';
44

55
/**
6-
* @typedef { (initial?: string) => {
7-
* add: (more: string) => void,
6+
* @typedef { (initial?: string | Buffer) => {
7+
* add: (more: string | Buffer) => void,
88
* finish: () => string,
99
* }
1010
* } CreateSHA256

0 commit comments

Comments
 (0)