Skip to content

Commit a8a962b

Browse files
FUDCoFUDCo
committed
feat: implement swingStore data export/import in support of state sync
Closes #6773
1 parent 3080ea7 commit a8a962b

18 files changed

+1427
-380
lines changed

‎packages/SwingSet/misc-tools/extract-transcript-from-kerneldb.js

+4-5
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ if (!dirPath) {
1818
if (!isSwingStore(dirPath)) {
1919
throw Error(`${dirPath} does not appear to be a swingstore (no ./data.mdb)`);
2020
}
21-
const { kvStore, streamStore } = openSwingStore(dirPath).kernelStorage;
21+
const { kvStore, transcriptStore } = openSwingStore(dirPath).kernelStorage;
2222
function get(key) {
2323
return kvStore.get(key);
2424
}
@@ -98,7 +98,7 @@ if (!vatName) {
9898
fs.writeSync(fd, JSON.stringify(first));
9999
fs.writeSync(fd, '\n');
100100

101-
// The streamStore holds concatenated transcripts from all upgraded
101+
// The transcriptStore holds concatenated transcripts from all upgraded
102102
// versions. For each old version, it holds every delivery from
103103
// `startVat` through `stopVat`. For the current version, it holds
104104
// every delivery from `startVat` up through the last delivery
@@ -123,9 +123,8 @@ if (!vatName) {
123123
console.log(`${transcriptLength} transcript entries`);
124124

125125
let deliveryNum = 0;
126-
const transcriptStream = `transcript-${vatID}`;
127-
const stream = streamStore.readStream(transcriptStream, startPos, endPos);
128-
for (const entry of stream) {
126+
const transcript = transcriptStore.readSpan(vatID, startPos, endPos);
127+
for (const entry of transcript) {
129128
// entry is JSON.stringify({ d, syscalls }), syscall is { d, response }
130129
const t = { transcriptNum, ...JSON.parse(entry) };
131130
// console.log(`t.${deliveryNum} : ${t}`);

‎packages/SwingSet/src/kernel/state/kernelKeeper.js

+4-23
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,7 @@ const enableKernelGC = true;
3838
* @typedef { import('../../types-external.js').KernelSlog } KernelSlog
3939
* @typedef { import('../../types-external.js').ManagerType } ManagerType
4040
* @typedef { import('../../types-external.js').SnapStore } SnapStore
41-
* @typedef { import('../../types-external.js').StreamPosition } StreamPosition
42-
* @typedef { import('../../types-external.js').StreamStore } StreamStore
41+
* @typedef { import('../../types-external.js').TranscriptStore } TranscriptStore
4342
* @typedef { import('../../types-external.js').VatKeeper } VatKeeper
4443
* @typedef { import('../../types-external.js').VatManager } VatManager
4544
*/
@@ -86,8 +85,6 @@ const enableKernelGC = true;
8685
// $vatSlot is one of: o+$NN/o-$NN/p+$NN/p-$NN/d+$NN/d-$NN
8786
// v$NN.c.$vatSlot = $kernelSlot = ko$NN/kp$NN/kd$NN
8887
// v$NN.nextDeliveryNum = $NN
89-
// v$NN.t.startPosition = $NN // inclusive
90-
// v$NN.t.endPosition = $NN // exclusive
9188
// v$NN.vs.$key = string
9289
// v$NN.meter = m$NN // XXX does this exist?
9390
// v$NN.reapInterval = $NN or 'never'
@@ -174,7 +171,7 @@ const FIRST_METER_ID = 1n;
174171
* @param {KernelSlog|null} kernelSlog
175172
*/
176173
export default function makeKernelKeeper(kernelStorage, kernelSlog) {
177-
const { kvStore, streamStore, snapStore } = kernelStorage;
174+
const { kvStore, transcriptStore, snapStore } = kernelStorage;
178175

179176
insistStorageAPI(kvStore);
180177

@@ -1297,11 +1294,11 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {
12971294
return found;
12981295
}
12991296
if (!kvStore.has(`${vatID}.o.nextID`)) {
1300-
initializeVatState(kvStore, streamStore, vatID);
1297+
initializeVatState(kvStore, transcriptStore, vatID);
13011298
}
13021299
const vk = makeVatKeeper(
13031300
kvStore,
1304-
streamStore,
1301+
transcriptStore,
13051302
kernelSlog,
13061303
vatID,
13071304
addKernelObject,
@@ -1328,20 +1325,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {
13281325
}
13291326

13301327
/**
1331-
* Cease writing to the vat's transcript.
1332-
*
1333-
* @param {string} vatID
1334-
*/
1335-
function closeVatTranscript(vatID) {
1336-
insistVatID(vatID);
1337-
const transcriptStream = `transcript-${vatID}`;
1338-
streamStore.closeStream(transcriptStream);
1339-
}
1340-
1341-
/**
1342-
* NOTE: caller is responsible to closeVatTranscript()
1343-
* before evicting a VatKeeper.
1344-
*
13451328
* @param {string} vatID
13461329
*/
13471330
function evictVatKeeper(vatID) {
@@ -1448,7 +1431,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {
14481431
if (vk) {
14491432
// TODO: find some way to expose the liveSlots internal tables, the
14501433
// kernel doesn't see them
1451-
closeVatTranscript(vatID);
14521434
const vatTable = {
14531435
vatID,
14541436
state: { transcript: Array.from(vk.getTranscript()) },
@@ -1615,7 +1597,6 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {
16151597
provideVatKeeper,
16161598
vatIsAlive,
16171599
evictVatKeeper,
1618-
closeVatTranscript,
16191600
cleanupAfterTerminatedVat,
16201601
addDynamicVatID,
16211602
getDynamicVats,

‎packages/SwingSet/src/kernel/state/vatKeeper.js

+16-37
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ import { enumeratePrefixedKeys } from './storageHelper.js';
1818
* @typedef { import('../../types-external.js').ManagerOptions } ManagerOptions
1919
* @typedef { import('../../types-external.js').SnapStore } SnapStore
2020
* @typedef { import('../../types-external.js').SourceOfBundle } SourceOfBundle
21-
* @typedef { import('../../types-external.js').StreamPosition } StreamPosition
22-
* @typedef { import('../../types-external.js').StreamStore } StreamStore
21+
* @typedef { import('../../types-external.js').TranscriptStore } TranscriptStore
2322
* @typedef { import('../../types-external.js').VatManager } VatManager
2423
* @typedef { import('../../types-internal.js').RecordedVatOptions } RecordedVatOptions
2524
* @typedef { import('../../types-external.js').TranscriptEntry } TranscriptEntry
@@ -36,25 +35,24 @@ const FIRST_DEVICE_ID = 70n;
3635
* Establish a vat's state.
3736
*
3837
* @param {*} kvStore The key-value store in which the persistent state will be kept
39-
* @param {*} streamStore Accompanying stream store
38+
* @param {*} transcriptStore Accompanying transcript store
4039
* @param {string} vatID The vat ID string of the vat in question
4140
* TODO: consider making this part of makeVatKeeper
4241
*/
43-
export function initializeVatState(kvStore, streamStore, vatID) {
42+
export function initializeVatState(kvStore, transcriptStore, vatID) {
4443
kvStore.set(`${vatID}.o.nextID`, `${FIRST_OBJECT_ID}`);
4544
kvStore.set(`${vatID}.p.nextID`, `${FIRST_PROMISE_ID}`);
4645
kvStore.set(`${vatID}.d.nextID`, `${FIRST_DEVICE_ID}`);
4746
kvStore.set(`${vatID}.nextDeliveryNum`, `0`);
4847
kvStore.set(`${vatID}.incarnationNumber`, `1`);
49-
kvStore.set(`${vatID}.t.startPosition`, `${streamStore.STREAM_START}`);
50-
kvStore.set(`${vatID}.t.endPosition`, `${streamStore.STREAM_START}`);
48+
transcriptStore.initTranscript(vatID);
5149
}
5250

5351
/**
5452
* Produce a vat keeper for a vat.
5553
*
5654
* @param {KVStore} kvStore The keyValue store in which the persistent state will be kept
57-
* @param {StreamStore} streamStore Accompanying stream store, for the transcripts
55+
* @param {TranscriptStore} transcriptStore Accompanying transcript store, for the transcripts
5856
* @param {*} kernelSlog
5957
* @param {string} vatID The vat ID string of the vat in question
6058
* @param {*} addKernelObject Kernel function to add a new object to the kernel's
@@ -76,7 +74,7 @@ export function initializeVatState(kvStore, streamStore, vatID) {
7674
*/
7775
export function makeVatKeeper(
7876
kvStore,
79-
streamStore,
77+
transcriptStore,
8078
kernelSlog,
8179
vatID,
8280
addKernelObject,
@@ -94,7 +92,6 @@ export function makeVatKeeper(
9492
snapStore = undefined,
9593
) {
9694
insistVatID(vatID);
97-
const transcriptStream = `transcript-${vatID}`;
9895

9996
function getRequired(key) {
10097
const value = kvStore.get(key);
@@ -475,20 +472,12 @@ export function makeVatKeeper(
475472
/**
476473
* Generator function to return the vat's transcript, one entry at a time.
477474
*
478-
* @param {StreamPosition} [startPos] Optional position to begin reading from
475+
* @param {number} [startPos] Optional position to begin reading from
479476
*
480477
* @yields { TranscriptEntry } a stream of transcript entries
481478
*/
482479
function* getTranscript(startPos) {
483-
if (startPos === undefined) {
484-
startPos = Number(getRequired(`${vatID}.t.startPosition`));
485-
}
486-
const endPos = Number(getRequired(`${vatID}.t.endPosition`));
487-
for (const entry of streamStore.readStream(
488-
transcriptStream,
489-
/** @type { StreamPosition } */ (startPos),
490-
endPos,
491-
)) {
480+
for (const entry of transcriptStore.readSpan(vatID, startPos)) {
492481
yield /** @type { TranscriptEntry } */ (JSON.parse(entry));
493482
}
494483
}
@@ -499,21 +488,13 @@ export function makeVatKeeper(
499488
* @param {object} entry The transcript entry to append.
500489
*/
501490
function addToTranscript(entry) {
502-
const oldPos = Number(getRequired(`${vatID}.t.endPosition`));
503-
const newPos = streamStore.writeStreamItem(
504-
transcriptStream,
505-
JSON.stringify(entry),
506-
oldPos,
507-
);
508-
kvStore.set(`${vatID}.t.endPosition`, `${newPos}`);
491+
transcriptStore.addItem(vatID, JSON.stringify(entry));
509492
}
510493

511-
/** @returns {StreamPosition} */
494+
/** @returns {number} */
512495
function getTranscriptEndPosition() {
513-
const endPosition =
514-
kvStore.get(`${vatID}.t.endPosition`) ||
515-
assert.fail('missing endPosition');
516-
return Number(endPosition);
496+
const { endPos } = transcriptStore.getCurrentSpanBounds(vatID);
497+
return endPos;
517498
}
518499

519500
function getSnapshotInfo() {
@@ -540,6 +521,7 @@ export function makeVatKeeper(
540521

541522
const endPosition = getTranscriptEndPosition();
542523
const info = await manager.makeSnapshot(endPosition, snapStore);
524+
transcriptStore.rolloverSpan(vatID);
543525
const {
544526
hash,
545527
uncompressedSize,
@@ -570,9 +552,7 @@ export function makeVatKeeper(
570552
if (snapStore) {
571553
snapStore.deleteVatSnapshots(vatID);
572554
}
573-
574-
const endPos = getRequired(`${vatID}.t.endPosition`);
575-
kvStore.set(`${vatID}.t.startPosition`, endPos);
555+
transcriptStore.rolloverSpan(vatID);
576556
}
577557

578558
function vatStats() {
@@ -584,9 +564,8 @@ export function makeVatKeeper(
584564
const objectCount = getCount(`${vatID}.o.nextID`, FIRST_OBJECT_ID);
585565
const promiseCount = getCount(`${vatID}.p.nextID`, FIRST_PROMISE_ID);
586566
const deviceCount = getCount(`${vatID}.d.nextID`, FIRST_DEVICE_ID);
587-
const startCount = Number(getRequired(`${vatID}.t.startPosition`));
588-
const endCount = Number(getRequired(`${vatID}.t.endPosition`));
589-
const transcriptCount = endCount - startCount;
567+
const { startPos, endPos } = transcriptStore.getCurrentSpanBounds(vatID);
568+
const transcriptCount = endPos - startPos;
590569

591570
// TODO: Fix the downstream JSON.stringify to allow the counts to be BigInts
592571
return harden({

‎packages/SwingSet/src/kernel/vat-warehouse.js

-1
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,6 @@ export function makeVatWarehouse(kernelKeeper, vatLoader, policyOptions) {
241241
}
242242
ephemeral.vats.delete(vatID);
243243
xlate.delete(vatID);
244-
kernelKeeper.closeVatTranscript(vatID);
245244
kernelKeeper.evictVatKeeper(vatID);
246245

247246
// console.log('evict: shutting down', vatID);

‎packages/SwingSet/src/types-ambient.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -135,8 +135,7 @@
135135
*/
136136
/**
137137
* @typedef { import('@agoric/swing-store').KVStore } KVStore
138-
* @typedef { import('@agoric/swing-store').StreamStore } StreamStore
139-
* @typedef { import('@agoric/swing-store').StreamPosition } StreamPosition
138+
* @typedef { import('@agoric/swing-store').TranscriptStore } TranscriptStore
140139
* @typedef { import('@agoric/swing-store').SwingStore } SwingStore
141140
* @typedef { import('@agoric/swing-store').SwingStoreKernelStorage } SwingStoreKernelStorage
142141
* @typedef { import('@agoric/swing-store').SwingStoreHostStorage } SwingStoreHostStorage

‎packages/SwingSet/src/types-external.js

+2-3
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ export {};
205205
* vatSyscallHandler: unknown) => Promise<VatManager>,
206206
* } } VatManagerFactory
207207
* @typedef { { deliver: (delivery: VatDeliveryObject) => Promise<VatDeliveryResult>,
208-
* replayTranscript: (startPos: StreamPosition | undefined) => Promise<number?>,
208+
* replayTranscript: (startPos: number | undefined) => Promise<number?>,
209209
* makeSnapshot?: (endPos: number, ss: SnapStore) => Promise<SnapshotResult>,
210210
* shutdown: () => Promise<void>,
211211
* } } VatManager
@@ -284,8 +284,7 @@ export {};
284284
* @typedef { import('@agoric/swing-store').KVStore } KVStore
285285
* @typedef { import('@agoric/swing-store').SnapStore } SnapStore
286286
* @typedef { import('@agoric/swing-store').SnapshotResult } SnapshotResult
287-
* @typedef { import('@agoric/swing-store').StreamStore } StreamStore
288-
* @typedef { import('@agoric/swing-store').StreamPosition } StreamPosition
287+
* @typedef { import('@agoric/swing-store').TranscriptStore } TranscriptStore
289288
* @typedef { import('@agoric/swing-store').SwingStore } SwingStore
290289
* @typedef { import('@agoric/swing-store').SwingStoreKernelStorage } SwingStoreKernelStorage
291290
* @typedef { import('@agoric/swing-store').SwingStoreHostStorage } SwingStoreHostStorage

‎packages/SwingSet/test/upgrade/test-upgrade.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ test('failed upgrade - explode', async t => {
481481
},
482482
};
483483

484-
const kernelStorage = initSwingStore().kernelStorage;
484+
const { kernelStorage } = initSwingStore();
485485
await initializeSwingset(config, [], kernelStorage);
486486
const c = await makeSwingsetController(kernelStorage);
487487
c.pinVatRoot('bootstrap');

‎packages/SwingSet/test/vat-warehouse/test-reload-snapshot.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ test('vat reload from snapshot', async t => {
3838
const snapshotInfo = snapStore.getSnapshotInfo(vatID);
3939

4040
const start = snapshotInfo ? snapshotInfo.endPos : 0;
41-
const endPosition = kernelStorage.kvStore.get(`${vatID}.t.endPosition`);
42-
const end = Number(endPosition);
41+
const bounds = kernelStorage.transcriptStore.getCurrentSpanBounds(vatID);
42+
const end = bounds.endPos;
4343
return [start, end];
4444
}
4545

‎packages/swing-store/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"@agoric/assert": "^0.5.1",
2222
"@agoric/internal": "^0.2.1",
2323
"better-sqlite3": "^7.5.0",
24+
"readline-transform": "^1.0.0",
2425
"tmp": "^0.2.1"
2526
},
2627
"devDependencies": {

‎packages/swing-store/src/hasher.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ import { assert } from '@agoric/assert';
33
import { createHash } from 'crypto';
44

55
/**
6-
* @typedef { (initial?: string) => {
7-
* add: (more: string) => void,
6+
* @typedef { (initial?: string | Buffer) => {
7+
* add: (more: string | Buffer) => void,
88
* finish: () => string,
99
* }
1010
* } CreateSHA256

0 commit comments

Comments
 (0)