feat: move snapstore into SQLite database with the rest of the swingstore

This is phase 1 of #6742.  These changes cease storing snapshots in
files but instead keep them in a new table in the swingstore SQLite
database.  However, in this commit, snapshot tracking metadata is
still managed the old way using entries in the kvstore, rather than
being integrated directly into the snapshots table.

Author: FUDCo

packages/SwingSet/misc-tools/replay-transcript.js

@@ -7,7 +7,6 @@ import zlib from 'zlib';
 import readline from 'readline';
 import process from 'process';
 import { spawn } from 'child_process';
-import path from 'path';
 import { promisify } from 'util';
 import { createHash } from 'crypto';
 import { pipeline } from 'stream';
@@ -63,11 +62,8 @@ function makeSnapStoreIO() {
   return {
     createReadStream: fs.createReadStream,
     createWriteStream: fs.createWriteStream,
-    fsync: fs.fsync,
     measureSeconds: makeMeasureSeconds(performance.now),
     open: fs.promises.open,
-    rename: fs.promises.rename,
-    resolve: path.resolve,
     stat: fs.promises.stat,
     tmpFile,
     tmpName,

packages/SwingSet/package.json

@@ -22,6 +22,8 @@
   },
   "devDependencies": {
     "@types/microtime": "^2.1.0",
+    "better-sqlite3": "^7.5.0",
+    "@types/better-sqlite3": "^7.5.0",
     "@types/tmp": "^0.2.0",
     "tmp": "^0.2.1"
   },

packages/SwingSet/src/kernel/state/vatKeeper.js

@@ -602,7 +602,6 @@ export function makeVatKeeper(
     const info = await manager.makeSnapshot(snapStore);
     const {
       hash: snapshotID,
-      newFile,
       rawByteCount,
       rawSaveSeconds,
       compressedByteCount,
@@ -611,7 +610,7 @@ export function makeVatKeeper(
     const old = getLastSnapshot();
     if (old && old.snapshotID !== snapshotID) {
       if (removeFromSnapshot(old.snapshotID) === 0) {
-        snapStore.prepareToDelete(old.snapshotID);
+        snapStore.deleteSnapshot(old.snapshotID);
       }
     }
     const endPosition = getTranscriptEndPosition();
@@ -624,7 +623,6 @@ export function makeVatKeeper(
       type: 'heap-snapshot-save',
       vatID,
       snapshotID,
-      newFile,
       rawByteCount,
       rawSaveSeconds,
       compressedByteCount,
@@ -641,14 +639,11 @@ export function makeVatKeeper(
       if (notation) {
         const { snapshotID } = JSON.parse(notation);
         if (removeFromSnapshot(snapshotID) === 0) {
-          // TODO: if we roll back (because the upgrade failed), we must
-          // not really delete the snapshot
-          snapStore.prepareToDelete(snapshotID);
+          snapStore.deleteSnapshot(snapshotID);
         }
         kvStore.delete(skey);
       }
     }
-    // TODO: same rollback concern
 
     const endPos = getRequired(`${vatID}.t.endPosition`);
     kvStore.set(`${vatID}.t.startPosition`, endPos);

packages/SwingSet/test/snapshots/test-xsnap-store.js.md

@@ -11,7 +11,6 @@ Generated by [AVA](https://avajs.dev).
     {
       compressSeconds: 0,
       hash: '8a0e3873976c50462d1b1dac59c912152b0e5cad5eeb9deca0ca64a087b4a873',
-      newFile: true,
       rawByteCount: 167887,
       rawSaveSeconds: 0,
     }
@@ -21,7 +20,6 @@ Generated by [AVA](https://avajs.dev).
     {
       compressSeconds: 0,
       hash: '253ffe0fb0b9e555119f046990d58fb85693e1170e681adb52211f49d94e37d0',
-      newFile: true,
       rawByteCount: 775831,
       rawSaveSeconds: 0,
     }
@@ -31,7 +29,6 @@ Generated by [AVA](https://avajs.dev).
     {
       compressSeconds: 0,
       hash: '2bca522840c90b8a0519fe846642746f01f62205154877c2332ad8829b69aa3e',
-      newFile: true,
       rawByteCount: 777983,
       rawSaveSeconds: 0,
     }

packages/SwingSet/test/snapshots/test-xsnap-store.js.snap

@@ -4,6 +4,7 @@ import { test } from '../tools/prepare-test-env-ava.js';
 import { spawn } from 'child_process';
 import fs from 'fs';
 import tmp from 'tmp';
+import sqlite3 from 'better-sqlite3';
 import { makePromiseKit } from '@endo/promise-kit';
 import { makeSnapStore, makeSnapStoreIO } from '@agoric/swing-store';
 
@@ -48,7 +49,8 @@ async function doTest(t, metered) {
   const pool = tmp.dirSync({ unsafeCleanup: true });
   t.teardown(() => pool.removeCallback());
   await fs.promises.mkdir(pool.name, { recursive: true });
-  const store = makeSnapStore(pool.name, makeSnapStoreIO());
+  const db = sqlite3(':memory:');
+  const store = makeSnapStore(db, pool.name, makeSnapStoreIO());
 
   const { p: p1, startXSnap: start1 } = make(store);
   let snapshotHash;

packages/SwingSet/test/test-xsnap-metering.js

@@ -4,6 +4,7 @@ import fs from 'fs';
 import { spawn } from 'child_process';
 import { type as osType } from 'os';
 import tmp from 'tmp';
+import sqlite3 from 'better-sqlite3';
 import test from 'ava';
 import { makeMeasureSeconds } from '@agoric/internal';
 import { xsnap } from '@agoric/xsnap';
@@ -32,9 +33,9 @@ const ld = (() => {
   });
 })();
 
-/** @type {(fn: string, fullSize: number) => number} */
-const relativeSize = (fn, fullSize) =>
-  Math.round((fs.statSync(fn).size / 1024 / fullSize) * 10) / 10;
+/** @type {(compressedSize: number, fullSize: number) => number} */
+const relativeSize = (compressedSize, fullSize) =>
+  Math.round((compressedSize / 1024 / fullSize) * 10) / 10;
 
 const snapSize = {
   raw: 417,
@@ -80,14 +81,15 @@ test(`create XS Machine, snapshot (${snapSize.raw} Kb), compress to smaller`, as
   t.teardown(() => pool.removeCallback());
   await fs.promises.mkdir(pool.name, { recursive: true });
 
-  const store = makeSnapStore(pool.name, makeMockSnapStoreIO());
+  const db = sqlite3(':memory:');
+  const store = makeSnapStore(db, pool.name, makeMockSnapStoreIO());
 
-  const { filePath: zfile } = await store.save(async snapFile => {
+  const { compressedByteCount } = await store.save(async snapFile => {
     await vat.snapshot(snapFile);
   });
 
   t.true(
-    relativeSize(zfile, snapSize.raw) < 0.5,
+    relativeSize(compressedByteCount, snapSize.raw) < 0.5,
     'compressed snapshots are smaller',
   );
 });
@@ -99,16 +101,17 @@ test('SES bootstrap, save, compress', async t => {
   const pool = tmp.dirSync({ unsafeCleanup: true });
   t.teardown(() => pool.removeCallback());
 
-  const store = makeSnapStore(pool.name, makeMockSnapStoreIO());
+  const db = sqlite3(':memory:');
+  const store = makeSnapStore(db, pool.name, makeMockSnapStoreIO());
 
   await vat.evaluate('globalThis.x = harden({a: 1})');
 
-  const { filePath: zfile } = await store.save(async snapFile => {
+  const { compressedByteCount } = await store.save(async snapFile => {
     await vat.snapshot(snapFile);
   });
 
   t.true(
-    relativeSize(zfile, snapSize.SESboot) < 0.5,
+    relativeSize(compressedByteCount, snapSize.SESboot) < 0.5,
     'compressed snapshots are smaller',
   );
 });
@@ -117,7 +120,8 @@ test('create SES worker, save, restore, resume', async t => {
   const pool = tmp.dirSync({ unsafeCleanup: true });
   t.teardown(() => pool.removeCallback());
 
-  const store = makeSnapStore(pool.name, makeMockSnapStoreIO());
+  const db = sqlite3(':memory:');
+  const store = makeSnapStore(db, pool.name, makeMockSnapStoreIO());
 
   const vat0 = await bootSESWorker('ses-boot2', async m => m);
   t.teardown(() => vat0.close());
@@ -146,7 +150,8 @@ test('XS + SES snapshots are long-term deterministic', async t => {
   t.teardown(() => pool.removeCallback());
   t.log({ pool: pool.name });
   await fs.promises.mkdir(pool.name, { recursive: true });
-  const store = makeSnapStore(pool.name, makeMockSnapStoreIO());
+  const db = sqlite3(':memory:');
+  const store = makeSnapStore(db, pool.name, makeMockSnapStoreIO());
 
   const vat = await bootWorker('xs1', async m => m, '1 + 1');
   t.teardown(() => vat.close());
@@ -197,7 +202,8 @@ async function makeTestSnapshot(t) {
   t.teardown(() => pool.removeCallback());
   // t.log({ pool: pool.name });
   await fs.promises.mkdir(pool.name, { recursive: true });
-  const store = makeSnapStore(pool.name, makeMockSnapStoreIO());
+  const db = sqlite3(':memory:');
+  const store = makeSnapStore(db, pool.name, makeMockSnapStoreIO());
   const vat = await bootWorker('xs1', async m => m, '1 + 1');
   const bootScript = await ld.asset(
     '@agoric/xsnap/dist/bundle-ses-boot.umd.js',

packages/SwingSet/test/test-xsnap-store.js

@@ -3,6 +3,7 @@ import { test } from '../../tools/prepare-test-env-ava.js';
 
 // import * as proc from 'child_process';
 import tmp from 'tmp';
+import sqlite3 from 'better-sqlite3';
 import {
   initSwingStore,
   makeSnapStore,
@@ -34,7 +35,8 @@ test('only preload maxVatsOnline vats', async t => {
   const argv = [];
 
   const snapstorePath = tmp.dirSync({ unsafeCleanup: true }).name;
-  const snapStore = makeSnapStore(snapstorePath, makeSnapStoreIO());
+  const db = sqlite3(':memory:');
+  const snapStore = makeSnapStore(db, snapstorePath, makeSnapStoreIO());
   const kernelStorage = { ...initSwingStore().kernelStorage, snapStore };
 
   await initializeSwingset(config, argv, kernelStorage, initOpts);

packages/SwingSet/test/vat-warehouse/test-preload.js

@@ -2,6 +2,7 @@
 import { test } from '../../tools/prepare-test-env-ava.js';
 
 import tmp from 'tmp';
+import sqlite3 from 'better-sqlite3';
 import {
   initSwingStore,
   makeSnapStore,
@@ -25,7 +26,8 @@ test('vat reload from snapshot', async t => {
 
   const snapstorePath = tmp.dirSync({ unsafeCleanup: true }).name;
 
-  const snapStore = makeSnapStore(snapstorePath, makeSnapStoreIO());
+  const db = sqlite3(':memory:');
+  const snapStore = makeSnapStore(db, snapstorePath, makeSnapStoreIO());
   const kernelStorage = { ...initSwingStore().kernelStorage, snapStore };
 
   const argv = [];

packages/SwingSet/test/vat-warehouse/test-reload-snapshot.js

@@ -68,7 +68,7 @@ test.skip('snapshots', async t => {
 
   // the delivery of startVat and bootstrap() results in snapshot A
   const sidA = getLatestSnapshot();
-  t.true(await snapStore.has(sidA));
+  t.true(snapStore.has(sidA));
   t.deepEqual(getSnapshotUsers(sidA), [vatID]);
 
   // increment() results in snapshot B
@@ -78,8 +78,8 @@ test.skip('snapshots', async t => {
   t.not(sidA, sidB);
   // the DB remembers 'A' as unused, so commit() will delete it, but
   // until then we should have both
-  t.true(await snapStore.has(sidA));
-  t.true(await snapStore.has(sidB));
+  t.true(snapStore.has(sidA));
+  t.true(snapStore.has(sidB));
   t.deepEqual(getSnapshotUsers(sidA), []);
   t.deepEqual(getSnapshotUsers(sidB), [vatID]);
 
@@ -89,8 +89,8 @@ test.skip('snapshots', async t => {
   // used-by entries around forever
   await commit();
 
-  t.false(await snapStore.has(sidA));
-  t.true(await snapStore.has(sidB));
+  t.false(snapStore.has(sidA));
+  t.true(snapStore.has(sidB));
   // t.deepEqual(getSnapshotUsers(sidA), undefined);
   t.deepEqual(getSnapshotUsers(sidA), []); // not deleted
   t.deepEqual(getSnapshotUsers(sidB), [vatID]);
@@ -101,31 +101,31 @@ test.skip('snapshots', async t => {
   await run('read');
 
   t.is(getLatestSnapshot(), sidB);
-  t.true(await snapStore.has(sidB));
+  t.true(snapStore.has(sidB));
   t.deepEqual(getSnapshotUsers(sidA), []); // not deleted
   t.deepEqual(getSnapshotUsers(sidB), [vatID]);
 
   // in the buggy version, this commit() deleted B
   await commit();
   t.is(getLatestSnapshot(), sidB);
-  t.true(await snapStore.has(sidB)); // .. so this failed
+  t.true(snapStore.has(sidB)); // .. so this failed
   t.deepEqual(getSnapshotUsers(sidA), []); // not deleted
   t.deepEqual(getSnapshotUsers(sidB), [vatID]);
 
   await run('increment'); // results in snapshot C
   const sidC = getLatestSnapshot();
   t.not(sidC, sidB);
-  t.true(await snapStore.has(sidB));
-  t.true(await snapStore.has(sidC));
+  t.true(snapStore.has(sidB));
+  t.true(snapStore.has(sidC));
   t.deepEqual(getSnapshotUsers(sidA), []); // not deleted
   t.deepEqual(getSnapshotUsers(sidB), []);
   t.deepEqual(getSnapshotUsers(sidC), [vatID]);
 
   // the commit() will delete B now that it is unused
   await commit();
   // in the buggy version, commit() failed because B was already deleted
-  t.false(await snapStore.has(sidB));
-  t.true(await snapStore.has(sidC));
+  t.false(snapStore.has(sidB));
+  t.true(snapStore.has(sidC));
   t.deepEqual(getSnapshotUsers(sidA), []); // not deleted
   t.deepEqual(getSnapshotUsers(sidB), []); // not deleted
   t.deepEqual(getSnapshotUsers(sidC), [vatID]);