diff --git a/packages/gatsby-core-utils/src/fetch-remote-file.ts b/packages/gatsby-core-utils/src/fetch-remote-file.ts index 780660023c980..2e936167dbc01 100644 --- a/packages/gatsby-core-utils/src/fetch-remote-file.ts +++ b/packages/gatsby-core-utils/src/fetch-remote-file.ts @@ -159,6 +159,28 @@ const requestRemoteNode = ( }) }) +const fetchCache = new Map() + +export async function fetchRemoteFileWithCache( + args: IFetchRemoteFileOptions +): Promise { + // If we are already fetching the file, return the unresolved promise + const inFlight = fetchCache.get(args.url) + if (inFlight) { + return inFlight + } + + // Create file fetch promise and store it into cache + const fetchPromise = fetchRemoteFile(args) + fetchCache.set(args.url, fetchPromise) + const result = await fetchPromise + + // When the file is fetched, replace the promise with the result + fetchCache.set(args.url, result) + + return result +} + export async function fetchRemoteFile({ url, cache, diff --git a/packages/gatsby-core-utils/src/index.ts b/packages/gatsby-core-utils/src/index.ts index a4cb4c4cb375f..9effbb7c07de2 100644 --- a/packages/gatsby-core-utils/src/index.ts +++ b/packages/gatsby-core-utils/src/index.ts @@ -7,7 +7,11 @@ export { createRequireFromPath } from "./create-require-from-path" export { getConfigStore } from "./get-config-store" export { getGatsbyVersion } from "./get-gatsby-version" export { getTermProgram } from "./get-term-program" -export { fetchRemoteFile, IFetchRemoteFileOptions } from "./fetch-remote-file" +export { + fetchRemoteFile, + fetchRemoteFileWithCache, + IFetchRemoteFileOptions, +} from "./fetch-remote-file" export { isTruthy } from "./is-truthy" export { getMatchPath } from "./match-path" export * from "./service-lock" diff --git a/packages/gatsby-source-contentful/src/cache-image.js b/packages/gatsby-source-contentful/src/cache-image.js deleted file mode 100644 index 3386eedb65be6..0000000000000 --- a/packages/gatsby-source-contentful/src/cache-image.js +++ /dev/null @@ -1,85 +0,0 @@ -const crypto = require(`crypto`) -const { resolve, parse } = require(`path`) - -const { pathExists, createWriteStream } = require(`fs-extra`) - -const downloadWithRetry = require(`./download-with-retry`).default - -const inFlightImageCache = new Map() - -module.exports = async function cacheImage(store, image, options, reporter) { - const program = store.getState().program - const CACHE_DIR = resolve(`${program.directory}/.cache/contentful/assets/`) - const { - file: { url, fileName, details }, - } = image - const { - width, - height, - maxWidth, - maxHeight, - resizingBehavior, - cropFocus, - background, - } = options - const userWidth = maxWidth || width - const userHeight = maxHeight || height - - const aspectRatio = details.image.height / details.image.width - const resultingWidth = Math.round(userWidth || 800) - const resultingHeight = Math.round(userHeight || resultingWidth * aspectRatio) - - const params = [`w=${resultingWidth}`, `h=${resultingHeight}`] - if (resizingBehavior) { - params.push(`fit=${resizingBehavior}`) - } - if (cropFocus) { - params.push(`f=${cropFocus}`) - } - if (background) { - params.push(`bg=${background}`) - } - - const optionsHash = crypto - .createHash(`md5`) - .update(JSON.stringify([url, ...params])) - .digest(`hex`) - - const { name, ext } = parse(fileName) - const absolutePath = resolve(CACHE_DIR, `${name}-${optionsHash}${ext}`) - - // Query the filesystem for file existence - const alreadyExists = await pathExists(absolutePath) - // Whether the file exists or not, if we are downloading it then await - const inFlight = inFlightImageCache.get(absolutePath) - if (inFlight) { - await inFlight - } else if (!alreadyExists) { - // File doesn't exist and is not being download yet - const downloadPromise = new Promise((resolve, reject) => { - const previewUrl = `http:${url}?${params.join(`&`)}` - - downloadWithRetry( - { - url: previewUrl, - responseType: `stream`, - }, - reporter - ) - .then(response => { - const file = createWriteStream(absolutePath) - response.data.pipe(file) - file.on(`finish`, resolve) - file.on(`error`, reject) - }) - .catch(reject) - }) - inFlightImageCache.set(absolutePath, downloadPromise) - await downloadPromise - // When the file is downloaded, remove the promise from the cache - inFlightImageCache.delete(absolutePath) - } - - // Now the file should be completely downloaded - return absolutePath -} diff --git a/packages/gatsby-source-contentful/src/create-schema-customization.js b/packages/gatsby-source-contentful/src/create-schema-customization.js new file mode 100644 index 0000000000000..885c1ad7ea8a8 --- /dev/null +++ b/packages/gatsby-source-contentful/src/create-schema-customization.js @@ -0,0 +1,68 @@ +const _ = require(`lodash`) + +const { createPluginConfig } = require(`./plugin-options`) + +export async function createSchemaCustomization( + { schema, actions, cache }, + pluginOptions +) { + const { createTypes } = actions + + const pluginConfig = createPluginConfig(pluginOptions) + const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( + `environment` + )}` + + const { contentTypeItems } = await cache.get( + `contentful-sync-result-${sourceId}` + ) + + createTypes(` + interface ContentfulEntry implements Node { + contentful_id: String! + id: ID! + node_locale: String! + } + `) + + createTypes(` + interface ContentfulReference { + contentful_id: String! + id: ID! + } + `) + + createTypes( + schema.buildObjectType({ + name: `ContentfulAsset`, + fields: { + contentful_id: { type: `String!` }, + id: { type: `ID!` }, + }, + interfaces: [`ContentfulReference`, `Node`], + }) + ) + + // Create types for each content type + const gqlTypes = contentTypeItems.map(contentTypeItem => + schema.buildObjectType({ + name: _.upperFirst( + _.camelCase( + `Contentful ${ + pluginConfig.get(`useNameForId`) + ? contentTypeItem.name + : contentTypeItem.sys.id + }` + ) + ), + fields: { + contentful_id: { type: `String!` }, + id: { type: `ID!` }, + node_locale: { type: `String!` }, + }, + interfaces: [`ContentfulReference`, `ContentfulEntry`, `Node`], + }) + ) + + createTypes(gqlTypes) +} diff --git a/packages/gatsby-source-contentful/src/extend-node-type.js b/packages/gatsby-source-contentful/src/extend-node-type.js index a6dc22ef5a475..3d8c629c19cab 100644 --- a/packages/gatsby-source-contentful/src/extend-node-type.js +++ b/packages/gatsby-source-contentful/src/extend-node-type.js @@ -14,10 +14,11 @@ const { GraphQLJSON, GraphQLList, } = require(`gatsby/graphql`) -const qs = require(`qs`) +const { fetchRemoteFileWithCache } = require(`gatsby-core-utils`) + const { stripIndent } = require(`common-tags`) +const qs = require(`qs`) -const cacheImage = require(`./cache-image`) const downloadWithRetry = require(`./download-with-retry`).default const { ImageFormatType, @@ -32,9 +33,6 @@ const { // cache is more likely to go stale than the images (which never go stale) // Note that the same image might be requested multiple times in the same run -// Supported Image Formats from https://www.contentful.com/developers/docs/references/images-api/#/reference/changing-formats/image-format -const validImageFormats = new Set([`jpg`, `png`, `webp`, `gif`]) - if (process.env.GATSBY_REMOTE_CACHE) { console.warn( `Note: \`GATSBY_REMOTE_CACHE\` will be removed soon because it has been renamed to \`GATSBY_CONTENTFUL_EXPERIMENTAL_REMOTE_CACHE\`` @@ -60,10 +58,20 @@ const resolvedBase64Cache = new Map() // @see https://www.contentful.com/developers/docs/references/images-api/#/reference/resizing-&-cropping/specify-width-&-height const CONTENTFUL_IMAGE_MAX_SIZE = 4000 -const isImage = image => - [`image/jpeg`, `image/jpg`, `image/png`, `image/webp`, `image/gif`].includes( - image?.file?.contentType - ) +// Supported Image Formats from https://www.contentful.com/developers/docs/references/images-api/#/reference/changing-formats/image-format +const validImageFormats = new Set([`jpg`, `png`, `webp`, `gif`]) + +const mimeTypeExtensions = new Map([ + [`image/jpeg`, `.jpg`], + [`image/jpg`, `.jpg`], + [`image/gif`, `.gif`], + [`image/png`, `.png`], + [`image/webp`, `.webp`], +]) + +exports.mimeTypeExtensions = mimeTypeExtensions + +const isImage = image => mimeTypeExtensions.has(image?.file?.contentType) // Note: this may return a Promise, body (sync), or null const getBase64Image = (imageProps, reporter) => { @@ -666,7 +674,7 @@ const fluidNodeType = ({ name, getTracedSVG, reporter }) => { } } -exports.extendNodeType = ({ type, store, reporter }) => { +exports.extendNodeType = ({ type, cache, reporter }) => { if (type.name !== `ContentfulAsset`) { return {} } @@ -676,15 +684,23 @@ exports.extendNodeType = ({ type, store, reporter }) => { const { image, options } = args const { - file: { contentType }, + file: { contentType, url: imgUrl, fileName }, } = image if (contentType.indexOf(`image/`) !== 0) { return null } - const absolutePath = await cacheImage(store, image, options, reporter) - const extension = path.extname(absolutePath) + const extension = mimeTypeExtensions.get(contentType) + const url = `https:` + createUrl(imgUrl, options) + const name = path.basename(fileName, extension) + + const absolutePath = await fetchRemoteFileWithCache({ + url, + name, + cache, + ext: extension, + }) return traceSVG({ file: { @@ -712,7 +728,29 @@ exports.extendNodeType = ({ type, store, reporter }) => { } try { - const absolutePath = await cacheImage(store, image, options, reporter) + const { + file: { contentType, url: imgUrl, fileName }, + } = image + + if (contentType.indexOf(`image/`) !== 0) { + return null + } + + // 256px should be enough to properly detect the dominant color + if (!options.width) { + options.width = 256 + } + + const extension = mimeTypeExtensions.get(contentType) + const url = `https:` + createUrl(imgUrl, options) + const name = path.basename(fileName, extension) + + const absolutePath = await fetchRemoteFileWithCache({ + url, + name, + cache, + ext: extension, + }) if (!(`getDominantColor` in pluginSharp)) { console.error( diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index 6ef21cc9652b3..fc7f9cee2587f 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -1,30 +1,19 @@ -const path = require(`path`) -const isOnline = require(`is-online`) +// @todo import syntax! const _ = require(`lodash`) -const fs = require(`fs-extra`) -const { createClient } = require(`contentful`) -const v8 = require(`v8`) const fetch = require(`@vercel/fetch-retry`)(require(`node-fetch`)) -const { CODES } = require(`./report`) -const normalize = require(`./normalize`) -const fetchData = require(`./fetch`) -const { createPluginConfig, maskText } = require(`./plugin-options`) -const { downloadContentfulAssets } = require(`./download-contentful-assets`) +const { createSchemaCustomization } = require(`./create-schema-customization`) +const { sourceNodes } = require(`./source-nodes`) +const { onPreBootstrap } = require(`./on-pre-bootstrap`) +const { maskText } = require(`./plugin-options`) -const conflictFieldPrefix = `contentful` +exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`).extendNodeType + +exports.createSchemaCustomization = createSchemaCustomization -// restrictedNodeFields from here https://www.gatsbyjs.org/docs/node-interface/ -const restrictedNodeFields = [ - `children`, - `contentful_id`, - `fields`, - `id`, - `internal`, - `parent`, -] +exports.sourceNodes = sourceNodes -const restrictedContentTypes = [`entity`, `reference`] +exports.onPreBootstrap = onPreBootstrap exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`).extendNodeType @@ -150,555 +139,3 @@ List of locales and their codes can be found in Contentful app -> Settings -> Lo .external(validateContentfulAccess) exports.pluginOptionsSchema = pluginOptionsSchema - -/*** - * Localization algorithm - * - * 1. Make list of all resolvable IDs worrying just about the default ids not - * localized ids - * 2. Make mapping between ids, again not worrying about localization. - * 3. When creating entries and assets, make the most localized version - * possible for each localized node i.e. get the localized field if it exists - * or the fallback field or the default field. - */ - -exports.sourceNodes = async ( - { - actions, - getNode, - getNodes, - getNodesByType, - createNodeId, - store, - cache, - getCache, - reporter, - schema, - parentSpan, - }, - pluginOptions -) => { - const { createNode, deleteNode, touchNode, createTypes } = actions - - let currentSyncData - let contentTypeItems - let defaultLocale - let locales - let space - if (process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE: Storing/loading remote data through \`` + - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE + - `\` so Remote changes CAN NOT be detected!` - ) - } - const forceCache = await fs.exists( - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE - ) - - const pluginConfig = createPluginConfig(pluginOptions) - const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( - `environment` - )}` - - const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` - const CACHE_SYNC_DATA = `contentful-sync-data-${sourceId}` - - /* - * Subsequent calls of Contentfuls sync API return only changed data. - * - * In some cases, especially when using rich-text fields, there can be data - * missing from referenced entries. This breaks the reference matching. - * - * To workround this, we cache the initial sync data and merge it - * with all data from subsequent syncs. Afterwards the references get - * resolved via the Contentful JS SDK. - */ - const syncToken = await cache.get(CACHE_SYNC_TOKEN) - let previousSyncData = { - assets: [], - entries: [], - } - const cachedData = await cache.get(CACHE_SYNC_DATA) - - if (cachedData) { - previousSyncData = cachedData - } - - const fetchActivity = reporter.activityTimer( - `Contentful: Fetch data (${sourceId})`, - { - parentSpan, - } - ) - - if (forceCache) { - // If the cache has data, use it. Otherwise do a remote fetch anyways and prime the cache now. - // If present, do NOT contact contentful, skip the round trips entirely - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Skipping remote fetch, using data stored in \`${process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE}\`` - ) - ;({ - currentSyncData, - contentTypeItems, - defaultLocale, - locales, - space, - } = v8.deserialize( - fs.readFileSync(process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) - )) - } else { - const online = await isOnline() - - // If the user knows they are offline, serve them cached result - // For prod builds though always fail if we can't get the latest data - if ( - !online && - process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && - process.env.NODE_ENV !== `production` - ) { - getNodes().forEach(node => { - if (node.internal.owner !== `gatsby-source-contentful`) { - return - } - touchNode(node) - if (node.localFile___NODE) { - // Prevent GraphQL type inference from crashing on this property - touchNode(getNode(node.localFile___NODE)) - } - }) - - reporter.info(`Using Contentful Offline cache ⚠️`) - reporter.info( - `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` - ) - - return - } - if (process.env.GATSBY_CONTENTFUL_OFFLINE) { - reporter.info( - `Note: \`GATSBY_CONTENTFUL_OFFLINE\` was set but it either was not \`true\`, we _are_ online, or we are in production mode, so the flag is ignored.` - ) - } - - fetchActivity.start() - ;({ - currentSyncData, - contentTypeItems, - defaultLocale, - locales, - space, - } = await fetchData({ - syncToken, - reporter, - pluginConfig, - parentSpan, - })) - - if (process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Writing v8 serialized glob of remote data to: ` + - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE - ) - fs.writeFileSync( - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE, - v8.serialize({ - currentSyncData, - contentTypeItems, - defaultLocale, - locales, - space, - }) - ) - } - } - - // Check for restricted content type names - const useNameForId = pluginConfig.get(`useNameForId`) - - contentTypeItems.forEach(contentTypeItem => { - // Establish identifier for content type - // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, - // but sometimes a base62 uuid generated by Contentful, hence the option) - let contentTypeItemId - if (useNameForId) { - contentTypeItemId = contentTypeItem.name.toLowerCase() - } else { - contentTypeItemId = contentTypeItem.sys.id.toLowerCase() - } - - if (restrictedContentTypes.includes(contentTypeItemId)) { - reporter.panic({ - id: CODES.FetchContentTypes, - context: { - sourceMessage: `Restricted ContentType name found. The name "${contentTypeItemId}" is not allowed.`, - }, - }) - } - }) - - const allLocales = locales - locales = locales.filter(pluginConfig.get(`localeFilter`)) - reporter.verbose( - `Default locale: ${defaultLocale}. All locales: ${allLocales - .map(({ code }) => code) - .join(`, `)}` - ) - if (allLocales.length !== locales.length) { - reporter.verbose( - `After plugin.options.localeFilter: ${locales - .map(({ code }) => code) - .join(`, `)}` - ) - } - if (locales.length === 0) { - reporter.panic({ - id: CODES.LocalesMissing, - context: { - sourceMessage: `Please check if your localeFilter is configured properly. Locales '${allLocales - .map(item => item.code) - .join(`,`)}' were found but were filtered down to none.`, - }, - }) - } - - createTypes(` - interface ContentfulEntry implements Node { - contentful_id: String! - id: ID! - node_locale: String! - } -`) - - createTypes(` - interface ContentfulReference { - contentful_id: String! - id: ID! - } -`) - - createTypes( - schema.buildObjectType({ - name: `ContentfulAsset`, - fields: { - contentful_id: { type: `String!` }, - id: { type: `ID!` }, - }, - interfaces: [`ContentfulReference`, `Node`], - }) - ) - - const gqlTypes = contentTypeItems.map(contentTypeItem => - schema.buildObjectType({ - name: _.upperFirst( - _.camelCase( - `Contentful ${ - pluginConfig.get(`useNameForId`) - ? contentTypeItem.name - : contentTypeItem.sys.id - }` - ) - ), - fields: { - contentful_id: { type: `String!` }, - id: { type: `ID!` }, - node_locale: { type: `String!` }, - }, - interfaces: [`ContentfulReference`, `ContentfulEntry`, `Node`], - }) - ) - - createTypes(gqlTypes) - - fetchActivity.end() - - const processingActivity = reporter.activityTimer( - `Contentful: Process data (${sourceId})`, - { - parentSpan, - } - ) - processingActivity.start() - - // Create a map of up to date entries and assets - function mergeSyncData(previous, current, deleted) { - const entryMap = new Map() - previous.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) - current.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) - return [...entryMap.values()] - } - - const deletedSet = new Set(currentSyncData.deletedEntries.map(e => e.sys.id)) - - const mergedSyncData = { - entries: mergeSyncData( - previousSyncData.entries, - currentSyncData.entries, - deletedSet - ), - assets: mergeSyncData( - previousSyncData.assets, - currentSyncData.assets, - deletedSet - ), - } - - // Store a raw and unresolved copy of the data for caching - const mergedSyncDataRaw = _.cloneDeep(mergedSyncData) - - // Use the JS-SDK to resolve the entries and assets - const res = createClient({ - space: `none`, - accessToken: `fake-access-token`, - }).parseEntries({ - items: mergedSyncData.entries, - includes: { - assets: mergedSyncData.assets, - entries: mergedSyncData.entries, - }, - }) - - mergedSyncData.entries = res.items - - // Inject raw API output to rich text fields - const richTextFieldMap = new Map() - contentTypeItems.forEach(contentType => { - richTextFieldMap.set( - contentType.sys.id, - contentType.fields - .filter(field => field.type === `RichText`) - .map(field => field.id) - ) - }) - - const rawEntries = new Map() - mergedSyncDataRaw.entries.forEach(rawEntry => - rawEntries.set(rawEntry.sys.id, rawEntry) - ) - - mergedSyncData.entries.forEach(entry => { - const contentTypeId = entry.sys.contentType.sys.id - const richTextFieldIds = richTextFieldMap.get(contentTypeId) - if (richTextFieldIds) { - richTextFieldIds.forEach(richTextFieldId => { - if (!entry.fields[richTextFieldId]) { - return - } - entry.fields[richTextFieldId] = rawEntries.get(entry.sys.id).fields[ - richTextFieldId - ] - }) - } - }) - - const entryList = normalize.buildEntryList({ - mergedSyncData, - contentTypeItems, - }) - - // Remove deleted entries & assets. - // TODO figure out if entries referencing now deleted entries/assets - // are "updated" so will get the now deleted reference removed. - - function deleteContentfulNode(node) { - const normalizedType = node.sys.type.startsWith(`Deleted`) - ? node.sys.type.substring(`Deleted`.length) - : node.sys.type - - const localizedNodes = locales - .map(locale => { - const nodeId = createNodeId( - normalize.makeId({ - spaceId: space.sys.id, - id: node.sys.id, - type: normalizedType, - currentLocale: locale.code, - defaultLocale, - }) - ) - return getNode(nodeId) - }) - .filter(node => node) - - localizedNodes.forEach(node => { - // touchNode first, to populate typeOwners & avoid erroring - touchNode(node) - deleteNode(node) - }) - } - - currentSyncData.deletedEntries.forEach(deleteContentfulNode) - currentSyncData.deletedAssets.forEach(deleteContentfulNode) - - const existingNodes = getNodes().filter( - n => n.internal.owner === `gatsby-source-contentful` - ) - existingNodes.forEach(n => touchNode(n)) - - const assets = mergedSyncData.assets - - reporter.info(`Updated entries ${currentSyncData.entries.length}`) - reporter.info(`Deleted entries ${currentSyncData.deletedEntries.length}`) - reporter.info(`Updated assets ${currentSyncData.assets.length}`) - reporter.info(`Deleted assets ${currentSyncData.deletedAssets.length}`) - - // Update syncToken - const nextSyncToken = currentSyncData.nextSyncToken - - await Promise.all([ - cache.set(CACHE_SYNC_DATA, mergedSyncDataRaw), - cache.set(CACHE_SYNC_TOKEN, nextSyncToken), - ]) - - reporter.verbose(`Building Contentful reference map`) - - // Create map of resolvable ids so we can check links against them while creating - // links. - const resolvable = normalize.buildResolvableSet({ - existingNodes, - entryList, - assets, - defaultLocale, - locales, - space, - }) - - // Build foreign reference map before starting to insert any nodes - const foreignReferenceMap = normalize.buildForeignReferenceMap({ - contentTypeItems, - entryList, - resolvable, - defaultLocale, - locales, - space, - useNameForId: pluginConfig.get(`useNameForId`), - }) - - reporter.verbose(`Resolving Contentful references`) - - const newOrUpdatedEntries = new Set() - entryList.forEach(entries => { - entries.forEach(entry => { - newOrUpdatedEntries.add(`${entry.sys.id}___${entry.sys.type}`) - }) - }) - - // Update existing entry nodes that weren't updated but that need reverse - // links added. - existingNodes - .filter(n => newOrUpdatedEntries.has(`${n.id}___${n.sys.type}`)) - .forEach(n => { - if (foreignReferenceMap[`${n.id}___${n.sys.type}`]) { - foreignReferenceMap[`${n.id}___${n.sys.type}`].forEach( - foreignReference => { - // Add reverse links - if (n[foreignReference.name]) { - n[foreignReference.name].push(foreignReference.id) - // It might already be there so we'll uniquify after pushing. - n[foreignReference.name] = _.uniq(n[foreignReference.name]) - } else { - // If is one foreign reference, there can always be many. - // Best to be safe and put it in an array to start with. - n[foreignReference.name] = [foreignReference.id] - } - } - ) - } - }) - - processingActivity.end() - - const creationActivity = reporter.activityTimer( - `Contentful: Create nodes (${sourceId})`, - { - parentSpan, - } - ) - creationActivity.start() - - for (let i = 0; i < contentTypeItems.length; i++) { - const contentTypeItem = contentTypeItems[i] - - if (entryList[i].length) { - reporter.info( - `Creating ${entryList[i].length} Contentful ${ - pluginConfig.get(`useNameForId`) - ? contentTypeItem.name - : contentTypeItem.sys.id - } nodes` - ) - } - - // A contentType can hold lots of entries which create nodes - // We wait until all nodes are created and processed until we handle the next one - // TODO add batching in gatsby-core - await Promise.all( - normalize.createNodesForContentType({ - contentTypeItem, - contentTypeItems, - restrictedNodeFields, - conflictFieldPrefix, - entries: entryList[i], - createNode, - createNodeId, - getNode, - resolvable, - foreignReferenceMap, - defaultLocale, - locales, - space, - useNameForId: pluginConfig.get(`useNameForId`), - }) - ) - } - - if (assets.length) { - reporter.info(`Creating ${assets.length} Contentful asset nodes`) - } - - for (let i = 0; i < assets.length; i++) { - // We wait for each asset to be process until handling the next one. - await Promise.all( - normalize.createAssetNodes({ - assetItem: assets[i], - createNode, - createNodeId, - defaultLocale, - locales, - space, - }) - ) - } - - creationActivity.end() - - if (pluginConfig.get(`downloadLocal`)) { - reporter.info(`Download Contentful asset files`) - - await downloadContentfulAssets({ - actions, - createNodeId, - store, - cache, - getCache, - getNode, - getNodesByType, - reporter, - assetDownloadWorkers: pluginConfig.get(`assetDownloadWorkers`), - }) - } - - return -} - -// Check if there are any ContentfulAsset nodes and if gatsby-image is installed. If so, -// add fragments for ContentfulAsset and gatsby-image. The fragment will cause an error -// if there's not ContentfulAsset nodes and without gatsby-image, the fragment is useless. -exports.onPreExtractQueries = async ({ store }) => { - const program = store.getState().program - - const CACHE_DIR = path.resolve( - `${program.directory}/.cache/contentful/assets/` - ) - await fs.ensureDir(CACHE_DIR) -} diff --git a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js new file mode 100644 index 0000000000000..fa49b6f97e4b6 --- /dev/null +++ b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js @@ -0,0 +1,352 @@ +// @todo import syntax! +import normalize from "./normalize" +const isOnline = require(`is-online`) +const _ = require(`lodash`) +const fs = require(`fs-extra`) +const { createClient } = require(`contentful`) +const v8 = require(`v8`) +const { CODES } = require(`./report`) +const fetchData = require(`./fetch`) +const { createPluginConfig } = require(`./plugin-options`) + +const restrictedContentTypes = [`entity`, `reference`] + +export async function onPreBootstrap( + { reporter, cache, actions, parentSpan, getNode, getNodes, createNodeId }, + pluginOptions +) { + // Fetch data for sourceNodes & createSchemaCustomization + const { deleteNode, touchNode } = actions + + let currentSyncData + let contentTypeItems + let defaultLocale + let locales + let space + if (process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { + reporter.info( + `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE: Storing/loading remote data through \`` + + process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE + + `\` so Remote changes CAN NOT be detected!` + ) + } + const forceCache = await fs.exists( + process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE + ) + + const pluginConfig = createPluginConfig(pluginOptions) + const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( + `environment` + )}` + + const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` + const CACHE_SYNC_DATA = `contentful-sync-data-${sourceId}` + + /* + * Subsequent calls of Contentfuls sync API return only changed data. + * + * In some cases, especially when using rich-text fields, there can be data + * missing from referenced entries. This breaks the reference matching. + * + * To workround this, we cache the initial sync data and merge it + * with all data from subsequent syncs. Afterwards the references get + * resolved via the Contentful JS SDK. + */ + const syncToken = await cache.get(CACHE_SYNC_TOKEN) + let previousSyncData = { + assets: [], + entries: [], + } + const cachedData = await cache.get(CACHE_SYNC_DATA) + + if (cachedData) { + previousSyncData = cachedData + } + + const fetchActivity = reporter.activityTimer( + `Contentful: Fetch data (${sourceId})`, + { + parentSpan, + } + ) + + // If the cache has data, use it. Otherwise do a remote fetch anyways and prime the cache now. + // If present, do NOT contact contentful, skip the round trips entirely + if (forceCache) { + reporter.info( + `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Skipping remote fetch, using data stored in \`${process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE}\`` + ) + ;({ + currentSyncData, + contentTypeItems, + defaultLocale, + locales, + space, + } = v8.deserialize( + fs.readFileSync(process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) + )) + } else { + const online = await isOnline() + + // If the user knows they are offline, serve them cached result + // For prod builds though always fail if we can't get the latest data + if ( + !online && + process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && + process.env.NODE_ENV !== `production` + ) { + getNodes().forEach(node => { + if (node.internal.owner !== `gatsby-source-contentful`) { + return + } + touchNode(node) + if (node.localFile___NODE) { + // Prevent GraphQL type inference from crashing on this property + touchNode(getNode(node.localFile___NODE)) + } + }) + + reporter.info(`Using Contentful Offline cache ⚠️`) + reporter.info( + `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` + ) + + return + } + if (process.env.GATSBY_CONTENTFUL_OFFLINE) { + reporter.info( + `Note: \`GATSBY_CONTENTFUL_OFFLINE\` was set but it either was not \`true\`, we _are_ online, or we are in production mode, so the flag is ignored.` + ) + } + + fetchActivity.start() + ;({ + currentSyncData, + contentTypeItems, + defaultLocale, + locales, + space, + } = await fetchData({ + syncToken, + reporter, + pluginConfig, + parentSpan, + })) + + if (process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { + reporter.info( + `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Writing v8 serialized glob of remote data to: ` + + process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE + ) + fs.writeFileSync( + process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE, + v8.serialize({ + currentSyncData, + contentTypeItems, + defaultLocale, + locales, + space, + }) + ) + } + } + + // Check for restricted content type names + const useNameForId = pluginConfig.get(`useNameForId`) + + contentTypeItems.forEach(contentTypeItem => { + // Establish identifier for content type + // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, + // but sometimes a base62 uuid generated by Contentful, hence the option) + let contentTypeItemId + if (useNameForId) { + contentTypeItemId = contentTypeItem.name.toLowerCase() + } else { + contentTypeItemId = contentTypeItem.sys.id.toLowerCase() + } + + if (restrictedContentTypes.includes(contentTypeItemId)) { + reporter.panic({ + id: CODES.FetchContentTypes, + context: { + sourceMessage: `Restricted ContentType name found. The name "${contentTypeItemId}" is not allowed.`, + }, + }) + } + }) + + const allLocales = locales + locales = locales.filter(pluginConfig.get(`localeFilter`)) + reporter.verbose( + `Default locale: ${defaultLocale}. All locales: ${allLocales + .map(({ code }) => code) + .join(`, `)}` + ) + if (allLocales.length !== locales.length) { + reporter.verbose( + `After plugin.options.localeFilter: ${locales + .map(({ code }) => code) + .join(`, `)}` + ) + } + if (locales.length === 0) { + reporter.panic({ + id: CODES.LocalesMissing, + context: { + sourceMessage: `Please check if your localeFilter is configured properly. Locales '${allLocales + .map(item => item.code) + .join(`,`)}' were found but were filtered down to none.`, + }, + }) + } + + fetchActivity.end() + + // Process data fetch results and turn them into GraphQL entities + const processingActivity = reporter.activityTimer( + `Contentful: Process data (${sourceId})`, + { + parentSpan, + } + ) + processingActivity.start() + + // Create a map of up to date entries and assets + function mergeSyncData(previous, current, deleted) { + const entryMap = new Map() + previous.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) + current.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) + return [...entryMap.values()] + } + + const deletedSet = new Set(currentSyncData.deletedEntries.map(e => e.sys.id)) + + const mergedSyncData = { + entries: mergeSyncData( + previousSyncData.entries, + currentSyncData.entries, + deletedSet + ), + assets: mergeSyncData( + previousSyncData.assets, + currentSyncData.assets, + deletedSet + ), + } + + // Store a raw and unresolved copy of the data for caching + const mergedSyncDataRaw = _.cloneDeep(mergedSyncData) + + // Use the JS-SDK to resolve the entries and assets + const res = createClient({ + space: `none`, + accessToken: `fake-access-token`, + }).parseEntries({ + items: mergedSyncData.entries, + includes: { + assets: mergedSyncData.assets, + entries: mergedSyncData.entries, + }, + }) + + mergedSyncData.entries = res.items + + // Inject raw API output to rich text fields + const richTextFieldMap = new Map() + contentTypeItems.forEach(contentType => { + richTextFieldMap.set( + contentType.sys.id, + contentType.fields + .filter(field => field.type === `RichText`) + .map(field => field.id) + ) + }) + + const rawEntries = new Map() + mergedSyncDataRaw.entries.forEach(rawEntry => + rawEntries.set(rawEntry.sys.id, rawEntry) + ) + + mergedSyncData.entries.forEach(entry => { + const contentTypeId = entry.sys.contentType.sys.id + const richTextFieldIds = richTextFieldMap.get(contentTypeId) + if (richTextFieldIds) { + richTextFieldIds.forEach(richTextFieldId => { + if (!entry.fields[richTextFieldId]) { + return + } + entry.fields[richTextFieldId] = rawEntries.get(entry.sys.id).fields[ + richTextFieldId + ] + }) + } + }) + + // @todo based on the sys metadata we should be able to differentiate new and updated entities + reporter.info( + `Contentful: ${currentSyncData.entries.length} new/updated entries` + ) + reporter.info( + `Contentful: ${currentSyncData.deletedEntries.length} deleted entries` + ) + reporter.info(`Contentful: ${previousSyncData.entries.length} cached entries`) + reporter.info( + `Contentful: ${currentSyncData.assets.length} new/updated assets` + ) + reporter.info(`Contentful: ${previousSyncData.assets.length} cached assets`) + reporter.info( + `Contentful: ${currentSyncData.deletedAssets.length} deleted assets` + ) + + // Remove deleted entries & assets + reporter.verbose(`Removing deleted Contentful entries & assets`) + + // @todo this should happen when sourcing? + function deleteContentfulNode(node) { + const normalizedType = node.sys.type.startsWith(`Deleted`) + ? node.sys.type.substring(`Deleted`.length) + : node.sys.type + + const localizedNodes = locales + .map(locale => { + const nodeId = createNodeId( + normalize.makeId({ + spaceId: space.sys.id, + id: node.sys.id, + type: normalizedType, + currentLocale: locale.code, + defaultLocale, + }) + ) + return getNode(nodeId) + }) + .filter(node => node) + + localizedNodes.forEach(node => { + // touchNode first, to populate typeOwners & avoid erroring + touchNode(node) + deleteNode(node) + }) + } + + currentSyncData.deletedEntries.forEach(deleteContentfulNode) + currentSyncData.deletedAssets.forEach(deleteContentfulNode) + + // Update syncToken + const nextSyncToken = currentSyncData.nextSyncToken + + await Promise.all([ + cache.set(CACHE_SYNC_DATA, mergedSyncDataRaw), + cache.set(CACHE_SYNC_TOKEN, nextSyncToken), + ]) + + await cache.set(`contentful-sync-result-${sourceId}`, { + mergedSyncData, + contentTypeItems, + locales, + space, + defaultLocale, + }) + + processingActivity.end() +} diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js new file mode 100644 index 0000000000000..9e8546458c207 --- /dev/null +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -0,0 +1,221 @@ +// @todo import syntax! +import _ from "lodash" +const path = require(`path`) +const fs = require(`fs-extra`) +import normalize from "./normalize" + +const { createPluginConfig } = require(`./plugin-options`) +import { downloadContentfulAssets } from "./download-contentful-assets" + +const conflictFieldPrefix = `contentful` + +// restrictedNodeFields from here https://www.gatsbyjs.org/docs/node-interface/ +const restrictedNodeFields = [ + `children`, + `contentful_id`, + `fields`, + `id`, + `internal`, + `parent`, +] + +/*** + * Localization algorithm + * + * 1. Make list of all resolvable IDs worrying just about the default ids not + * localized ids + * 2. Make mapping between ids, again not worrying about localization. + * 3. When creating entries and assets, make the most localized version + * possible for each localized node i.e. get the localized field if it exists + * or the fallback field or the default field. + */ + +export async function sourceNodes( + { + actions, + getNode, + getNodes, + getNodesByType, + createNodeId, + store, + cache, + getCache, + reporter, + parentSpan, + }, + pluginOptions +) { + const { createNode, touchNode } = actions + + const pluginConfig = createPluginConfig(pluginOptions) + const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( + `environment` + )}` + + const { + mergedSyncData, + contentTypeItems, + locales, + space, + defaultLocale, + } = await cache.get(`contentful-sync-result-${sourceId}`) + + const { assets } = mergedSyncData + + const entryList = normalize.buildEntryList({ + mergedSyncData, + contentTypeItems, + }) + + const existingNodes = getNodes().filter( + n => n.internal.owner === `gatsby-source-contentful` + ) + existingNodes.forEach(n => touchNode(n)) + + reporter.verbose(`Building Contentful reference map`) + + // Create map of resolvable ids so we can check links against them while creating + // links. + const resolvable = normalize.buildResolvableSet({ + existingNodes, + entryList, + assets, + defaultLocale, + locales, + space, + }) + + // Build foreign reference map before starting to insert any nodes + const foreignReferenceMap = normalize.buildForeignReferenceMap({ + contentTypeItems, + entryList, + resolvable, + defaultLocale, + locales, + space, + useNameForId: pluginConfig.get(`useNameForId`), + }) + + reporter.verbose(`Resolving Contentful references`) + + const newOrUpdatedEntries = new Set() + entryList.forEach(entries => { + entries.forEach(entry => { + newOrUpdatedEntries.add(`${entry.sys.id}___${entry.sys.type}`) + }) + }) + + // Update existing entry nodes that weren't updated but that need reverse + // links added. + existingNodes + .filter(n => newOrUpdatedEntries.has(`${n.id}___${n.sys.type}`)) + .forEach(n => { + if (foreignReferenceMap[`${n.id}___${n.sys.type}`]) { + foreignReferenceMap[`${n.id}___${n.sys.type}`].forEach( + foreignReference => { + // Add reverse links + if (n[foreignReference.name]) { + n[foreignReference.name].push(foreignReference.id) + // It might already be there so we'll uniquify after pushing. + n[foreignReference.name] = _.uniq(n[foreignReference.name]) + } else { + // If is one foreign reference, there can always be many. + // Best to be safe and put it in an array to start with. + n[foreignReference.name] = [foreignReference.id] + } + } + ) + } + }) + + const creationActivity = reporter.activityTimer( + `Contentful: Create nodes (${sourceId})`, + { + parentSpan, + } + ) + creationActivity.start() + + for (let i = 0; i < contentTypeItems.length; i++) { + const contentTypeItem = contentTypeItems[i] + + if (entryList[i].length) { + reporter.info( + `Creating ${entryList[i].length} Contentful ${ + pluginConfig.get(`useNameForId`) + ? contentTypeItem.name + : contentTypeItem.sys.id + } nodes` + ) + } + + // A contentType can hold lots of entries which create nodes + // We wait until all nodes are created and processed until we handle the next one + // TODO add batching in gatsby-core + await Promise.all( + normalize.createNodesForContentType({ + contentTypeItem, + contentTypeItems, + restrictedNodeFields, + conflictFieldPrefix, + entries: entryList[i], + createNode, + createNodeId, + getNode, + resolvable, + foreignReferenceMap, + defaultLocale, + locales, + space, + useNameForId: pluginConfig.get(`useNameForId`), + }) + ) + } + + if (assets.length) { + reporter.info(`Creating ${assets.length} Contentful asset nodes`) + } + + for (let i = 0; i < assets.length; i++) { + // We wait for each asset to be process until handling the next one. + await Promise.all( + normalize.createAssetNodes({ + assetItem: assets[i], + createNode, + createNodeId, + defaultLocale, + locales, + space, + }) + ) + } + + creationActivity.end() + + // @todo add own activity! + + if (pluginConfig.get(`downloadLocal`)) { + reporter.info(`Download Contentful asset files`) + + // Ensure cache dir exists for downloadLocal + const program = store.getState().program + + const CACHE_DIR = path.resolve( + `${program.directory}/.cache/contentful/assets/` + ) + + await fs.ensureDir(CACHE_DIR) + + await downloadContentfulAssets({ + actions, + createNodeId, + store, + cache, + getCache, + getNode, + getNodesByType, + reporter, + assetDownloadWorkers: pluginConfig.get(`assetDownloadWorkers`), + }) + } +} diff --git a/packages/gatsby-transformer-sqip/src/extend-node-type.js b/packages/gatsby-transformer-sqip/src/extend-node-type.js index 217f408d27f98..58a52536bc48a 100644 --- a/packages/gatsby-transformer-sqip/src/extend-node-type.js +++ b/packages/gatsby-transformer-sqip/src/extend-node-type.js @@ -1,22 +1,22 @@ -const { resolve } = require(`path`) -const md5File = require(`md5-file`) - -const { - DuotoneGradientType, - ImageCropFocusType, -} = require(`gatsby-transformer-sharp/types`) -const { queueImageResizing } = require(`gatsby-plugin-sharp`) +const path = require(`path`) const Debug = require(`debug`) const fs = require(`fs-extra`) +const sharp = require(`sharp`) +const md5File = require(`md5-file`) + const { GraphQLObjectType, GraphQLString, GraphQLInt, GraphQLBoolean, } = require(`gatsby/graphql`) -const sharp = require(`sharp`) -const { ensureDir } = require(`fs-extra`) +const { queueImageResizing } = require(`gatsby-plugin-sharp`) +const { fetchRemoteFileWithCache } = require(`gatsby-core-utils`) +const { + DuotoneGradientType, + ImageCropFocusType, +} = require(`gatsby-transformer-sharp/types`) const generateSqip = require(`./generate-sqip`) @@ -44,11 +44,11 @@ module.exports = async args => { async function sqipSharp({ type, cache, getNodeAndSavePathDependency, store }) { const program = store.getState().program - const cacheDir = resolve( + const cacheDir = path.resolve( `${program.directory}/node_modules/.cache/gatsby-transformer-sqip/` ) - await ensureDir(cacheDir) + await fs.ensureDir(cacheDir) return { sqip: { @@ -140,14 +140,12 @@ async function sqipContentful({ type, cache, store }) { schemes: { ImageResizingBehavior, ImageCropFocusType }, } = require(`gatsby-source-contentful`) - const cacheImage = require(`gatsby-source-contentful/cache-image`) - const program = store.getState().program - const cacheDir = resolve( + const cacheDir = path.resolve( `${program.directory}/node_modules/.cache/gatsby-transformer-sqip/` ) - await ensureDir(cacheDir) + await fs.ensureDir(cacheDir) return { sqip: { @@ -192,7 +190,12 @@ async function sqipContentful({ type, cache, store }) { }, async resolve(asset, fieldArgs, context) { const { - file: { contentType }, + createUrl, + mimeTypeExtensions, + } = require(`gatsby-source-contentful/extend-node-type`) + + const { + file: { contentType, url: imgUrl, fileName }, } = asset if (!contentType.includes(`image/`)) { @@ -223,7 +226,17 @@ async function sqipContentful({ type, cache, store }) { background, } - const absolutePath = await cacheImage(store, asset, options) + const extension = mimeTypeExtensions.get(contentType) + const url = `https:` + createUrl(imgUrl, options) + const name = path.basename(fileName, extension) + + console.log({ extension, url, name }) + const absolutePath = await fetchRemoteFileWithCache({ + url, + name, + cache, + ext: extension, + }) const contentDigest = await md5File(absolutePath)