Skip to content

Commit 6cd8d0a

Browse files
authored
✨ persist cache (#10)
* ✨ persit cache * 🔖 release note * 💡 docs options
1 parent c4ee502 commit 6cd8d0a

17 files changed

+547
-89
lines changed

.changeset/famous-pumpkins-develop.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"vsit": patch
3+
---
4+
5+
support persist cache in node side

package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@
2020
},
2121
"scripts": {
2222
"test": "pnpm --filter=./packages/** run test",
23-
"dev:cli": "pnpm --filter=./packages/vsit dev",
24-
"dev": "pnpm --filter=./packages/client dev",
23+
"dev": "pnpm --filter=./packages/vsit dev",
24+
"play": "pnpm --filter=./packages/client dev",
2525
"build:cli": "pnpm --filter=./packages/vsit build",
2626
"build:client": "pnpm --filter=./packages/client build",
2727
"build:copy": "esno ./scripts/client.ts",
@@ -48,6 +48,6 @@
4848
"husky": "^8.0.1",
4949
"lint-staged": "^13.0.3",
5050
"npm-run-all": "^4.1.5",
51-
"typescript": "4.4.4"
51+
"typescript": "5.1.6"
5252
}
5353
}

packages/vsit/package.json

+7-1
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
"@rollup/plugin-node-resolve": "15.0.1",
7676
"@rollup/plugin-replace": "^5.0.2",
7777
"@types/debug": "^4.1.8",
78+
"@types/fs-extra": "^11.0.1",
7879
"@types/inquirer": "^8.1.3",
7980
"@types/node": "20.3.1",
8081
"@types/rimraf": "^3.0.2",
@@ -86,24 +87,29 @@
8687
"debug": "^4.3.4",
8788
"esbuild": "^0.18.6",
8889
"execa": "^6.0.0",
90+
"fs-extra": "^11.1.1",
8991
"husky": "^8.0.3",
9092
"inquirer": "8.2.0",
9193
"npm-run-all": "^4.1.5",
9294
"ofetch": "^1.0.1",
9395
"ora": "6.0.1",
9496
"picocolors": "1.0.0",
9597
"publish-police": "^0.1.0",
98+
"read-yaml-file": "2.1.0",
9699
"rimraf": "^3.0.2",
97100
"rollup": "3.19.1",
98101
"rollup-plugin-condition-exports": "2.0.0-next.3",
99102
"rollup-plugin-esbuild": "^5.0.0",
100103
"rollup-plugin-node-externals": "5.1.2",
101104
"rollup-plugin-size": "^0.3.1",
102105
"source-map-support": "^0.5.21",
106+
"tempy": "^3.1.0",
103107
"ttypescript": "^1.5.15",
108+
"type-fest": "^3.12.0",
104109
"typescript": "^4.6.4",
105110
"typescript-transform-paths": "^3.4.6",
106111
"ufo": "^1.1.2",
107-
"vitest": "^0.22.1"
112+
"vitest": "^0.22.1",
113+
"write-yaml-file": "5.0.0"
108114
}
109115
}

packages/vsit/src/common/resolver/constants.ts

+6
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,9 @@ export const VIRTUAL_RE = /^(?:virtual:)|\0/
99

1010
export const VIRUTAL_NODE_ID = 'fake-node-file.ts'
1111
export const VIRUTAL_WEB_ID = 'fake-web-file.ts'
12+
/**
13+
* Borrowed from vite
14+
*/
15+
export const VALID_ID_PREFIX = '/@id/'
16+
export const NULL_BYTE_PLACEHOLDER = '__x00__'
17+
export const NULL_BYTE = '\x00'
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import os from 'node:os'
2+
import path from 'node:path'
3+
4+
const STORE_DIR = '.vsit-store'
5+
export const STORE_PATH = path.join(os.homedir(), STORE_DIR)
6+
export const LOCK_FILE = 'vist-lock.yaml'
7+
export const STORE_PACKAGES_DIR = 'packages'

packages/vsit/src/common/store/index.ts

+6-5
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,20 @@ import { createHash } from 'node:crypto'
22

33
import { fetch } from 'ofetch'
44

5+
import { PersistCache } from './persist-cache'
56
import { debug } from '@/common/log'
67

7-
export const createStore = () => {
8+
export const createStore = async () => {
89
const pool = new Map<string, Promise<string>>()
9-
const globalCache = new Map<string, string>()
10+
const cacheManager = await PersistCache.create()
1011
const createInstance = (id: string, url: string, options?: RequestInit) => {
1112
const promise = (async () => {
1213
try {
1314
debug.store('start fetch %s', url)
1415
return fetch(url, options)
1516
.then(async (res) => {
1617
const content = await res.text()
17-
globalCache.set(id, content)
18+
cacheManager.saveCache(url, content)
1819
pool.delete(id)
1920
return content
2021
})
@@ -27,14 +28,14 @@ export const createStore = () => {
2728
return promise
2829
}
2930
return {
31+
cache: cacheManager,
3032
async clear(url: string) {
3133
const hash = createHash('sha256').update(url).digest('hex')
32-
globalCache.delete(hash)
3334
pool.delete(hash)
3435
},
3536
async fetch(url: string, options?: RequestInit) {
3637
const hash = createHash('sha256').update(url).digest('hex')
37-
const cache = globalCache.get(hash)
38+
const cache = await cacheManager.getCache(url)
3839
if (cache) {
3940
debug.store('load cache %s', url)
4041
return Promise.resolve(cache)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
import { resolve } from 'node:path'
2+
3+
import {
4+
existsSync,
5+
outputFile,
6+
readFile,
7+
} from 'fs-extra'
8+
import readYaml from 'read-yaml-file'
9+
import writeYaml from 'write-yaml-file'
10+
11+
import { version } from '../../../package.json'
12+
import {
13+
LOCK_FILE,
14+
STORE_PACKAGES_DIR,
15+
STORE_PATH,
16+
} from './constants'
17+
import { computeCacheKey } from './utils'
18+
import { debug } from '@/common/log'
19+
20+
interface Options {
21+
storePath?: string
22+
}
23+
24+
interface ResolvedLockFileOptions {
25+
/**
26+
* @description Virtual store path
27+
* @default <homedir>/<.vsit-store>
28+
*/
29+
storePath: string
30+
/**
31+
* @description Virtual store lock file path
32+
* @default <homedir>/<.vsit-store>/vsit-lock.yaml
33+
*/
34+
lockFilePath: string
35+
}
36+
37+
export interface Package {
38+
/**
39+
* @description Encoded url
40+
*/
41+
id: string
42+
/**
43+
* @description Remote package url
44+
*/
45+
url: string
46+
/**
47+
* @description Dependent packages
48+
* @todo not used currently, in the future, we will outdated the persist cache, and concurrent download the packages based on deps
49+
*/
50+
deps?: string[]
51+
}
52+
53+
interface LockFileYaml {
54+
version: string
55+
packages?: Record<string, Package>
56+
}
57+
58+
export class LockFile {
59+
options: ResolvedLockFileOptions
60+
lockFile: LockFileYaml = { version }
61+
constructor(options: Options) {
62+
this.options = this.resolveOptions(options)
63+
debug.store('resolved lock-file options %o', this.options)
64+
}
65+
66+
resolveOptions(options: Options): ResolvedLockFileOptions {
67+
const resolvedStorePath = options.storePath ?? STORE_PATH
68+
return {
69+
storePath: resolvedStorePath,
70+
lockFilePath: resolve(resolvedStorePath, LOCK_FILE),
71+
}
72+
}
73+
74+
async init() {
75+
if (!existsSync(this.options.lockFilePath)) {
76+
await writeYaml(this.options.lockFilePath, { version })
77+
}
78+
await this.read()
79+
}
80+
81+
async read(): Promise<LockFileYaml> {
82+
this.lockFile = await readYaml(this.options.lockFilePath)
83+
return this.lockFile as LockFileYaml
84+
}
85+
86+
async write(data: LockFileYaml): Promise<void> {
87+
await writeYaml(this.options.lockFilePath, data)
88+
}
89+
90+
async save() {
91+
await writeYaml(this.options.lockFilePath, this.lockFile)
92+
}
93+
94+
async writePackage(url: string, deps: string[] = []) {
95+
const id = computeCacheKey(url)
96+
this.lockFile.packages = {
97+
...this.lockFile.packages,
98+
[id]: {
99+
id,
100+
url,
101+
deps,
102+
},
103+
}
104+
await this.save()
105+
}
106+
107+
async writePackages(packages: Record<string, Package> = {}) {
108+
this.lockFile.packages = {
109+
...this.lockFile.packages,
110+
...packages,
111+
}
112+
await this.save()
113+
}
114+
115+
static async create(options: Options) {
116+
const instance = new LockFile(options)
117+
await instance.init()
118+
return instance
119+
}
120+
}
121+
122+
interface ResolvedCacheOptions {
123+
storePath: string
124+
packagesPath: string
125+
}
126+
127+
export class PersistCache {
128+
options: ResolvedCacheOptions
129+
lockFile?: LockFile
130+
constructor(options: Options = { storePath: STORE_PATH }) {
131+
this.options = this.resolveOptions(options)
132+
this.lockFile = undefined
133+
}
134+
135+
resolveOptions(options: Options): ResolvedCacheOptions {
136+
const resolvedStorePath = options.storePath ?? STORE_PATH
137+
return {
138+
storePath: resolvedStorePath,
139+
packagesPath: resolve(resolvedStorePath, STORE_PACKAGES_DIR),
140+
}
141+
}
142+
143+
static async create(options: Options = { storePath: STORE_PATH }) {
144+
const instance = new PersistCache(options)
145+
instance.lockFile = await LockFile.create(options)
146+
return instance
147+
}
148+
149+
async writePackages(packages: Record<string, Package> = {}) {
150+
this.lockFile?.writePackages(packages)
151+
}
152+
153+
async getCache(url: string) {
154+
const id = computeCacheKey(url)
155+
const path = resolve(this.options.packagesPath, id)
156+
if (existsSync(path)) {
157+
const content = (await readFile(path)).toString('utf-8')
158+
return content
159+
}
160+
return ''
161+
}
162+
163+
async saveCache(url: string, content: string) {
164+
const id = computeCacheKey(url)
165+
const path = resolve(this.options.packagesPath, id)
166+
await this.lockFile?.writePackage(url)
167+
await outputFile(path, content)
168+
}
169+
}
+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import { createHash } from 'node:crypto'
2+
3+
export const computeCacheKey = (url: string) => {
4+
const hash = createHash('sha256').update(url).digest('hex')
5+
return hash
6+
}

packages/vsit/src/common/resolver/normalize.ts packages/vsit/src/common/utils.ts

+57-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,23 @@ import { join } from 'node:path'
22

33
import { withoutLeadingSlash, withoutTrailingSlash } from 'ufo'
44

5-
import { pkgRoot } from '../path'
5+
import { pkgRoot } from './path'
66
import {
77
ESM_HOST,
88
ESMSH_HTTP_RE,
99
ESMSH_HTTP_SUB_RE,
1010
ESMSH_PROTOCOL,
1111
ESMSH_PROTOCOL_RE,
12+
NULL_BYTE,
13+
NULL_BYTE_PLACEHOLDER,
14+
VALID_ID_PREFIX,
1215
VIRTUAL_RE,
13-
} from './constants'
16+
} from './resolver/constants'
17+
import { isEsmSh } from './resolver/is'
18+
import { computeCacheKey } from './store/utils'
19+
20+
import type { ModuleNode } from 'vite'
21+
import type { Package } from './store/persist-cache'
1422

1523
// '\0' tell vite to not resolve this id via internal node resolver algorithm
1624
export const wrapId = (id: string) => {
@@ -25,12 +33,15 @@ export const wrapId = (id: string) => {
2533
return id
2634
}
2735

28-
export const unWrapId = (id: string) => {
36+
export const unwrapId = (id: string) => {
2937
let stripId = id.replace(VIRTUAL_RE, '')
3038
// unwrap
3139
// https:/esm.sh -> https://esm.sh
3240
// esm.sh: -> https://esm.sh
3341
stripId = stripId
42+
.replace(VALID_ID_PREFIX, '')
43+
.replace(NULL_BYTE_PLACEHOLDER, '')
44+
.replace(NULL_BYTE, '')
3445
.replace(ESMSH_HTTP_RE, withoutTrailingSlash(ESM_HOST))
3546
.replace(ESMSH_PROTOCOL_RE, ESM_HOST)
3647
return stripId
@@ -56,3 +67,46 @@ globalThis.__hook(consolehook, (log) => {
5667
${content}
5768
`
5869
}
70+
71+
export const parseDeps = (deps: string[] = []) => {
72+
return deps
73+
.map((dep) => {
74+
return unwrapId(dep)
75+
})
76+
.filter((dep) => {
77+
return isEsmSh(dep)
78+
})
79+
}
80+
81+
const createPackage = (url: string, deps: string[]): Record<string, Package> => {
82+
if (!url) {
83+
return {}
84+
}
85+
const id = computeCacheKey(url)
86+
return {
87+
[id]: {
88+
id,
89+
url,
90+
deps,
91+
},
92+
}
93+
}
94+
95+
export const parseModulesDeps = (m?: ModuleNode): Record<string, Package> => {
96+
if (!m || !m.id) {
97+
return {}
98+
}
99+
const id = unwrapId(m.id)
100+
const deps = parseDeps(m.ssrTransformResult?.deps)
101+
let records = id && isEsmSh(id) && deps.length
102+
? createPackage(id, deps)
103+
: {}
104+
m.importedModules.forEach((importedModule) => {
105+
const result = parseModulesDeps(importedModule)
106+
records = {
107+
...records,
108+
...result,
109+
}
110+
})
111+
return records
112+
}

0 commit comments

Comments
 (0)