From a16e21fc2564b9ee2068e30d276f58e8b3cf1657 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Thu, 16 Sep 2021 11:48:31 +0200 Subject: [PATCH 01/27] refactor(contentful): move code to createSchemaCustomization --- .../__tests__/download-contentful-assets.js | 2 +- .../src/__tests__/gatsby-node.js | 320 +++------ .../src/create-schema-customization.js | 83 +++ .../src/gatsby-node.js | 626 +----------------- .../src/on-pre-bootstrap.js | 356 ++++++++++ .../src/source-nodes.js | 234 +++++++ 6 files changed, 778 insertions(+), 843 deletions(-) create mode 100644 packages/gatsby-source-contentful/src/create-schema-customization.js create mode 100644 packages/gatsby-source-contentful/src/on-pre-bootstrap.js create mode 100644 packages/gatsby-source-contentful/src/source-nodes.js diff --git a/packages/gatsby-source-contentful/src/__tests__/download-contentful-assets.js b/packages/gatsby-source-contentful/src/__tests__/download-contentful-assets.js index de3a56eaeb667..4ba2962a312b3 100644 --- a/packages/gatsby-source-contentful/src/__tests__/download-contentful-assets.js +++ b/packages/gatsby-source-contentful/src/__tests__/download-contentful-assets.js @@ -54,7 +54,7 @@ const fixtures = [ }, ] -describe.only(`downloadContentfulAssets`, () => { +describe(`downloadContentfulAssets`, () => { it(`derives unique cache key from node locale and id`, async () => { const cache = { get: jest.fn(() => Promise.resolve(null)), diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index 1b6a3c2f488b5..249560fa2864a 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -15,13 +15,15 @@ const startersBlogFixture = require(`../__fixtures__/starter-blog-data`) const richTextFixture = require(`../__fixtures__/rich-text-data`) const restrictedContentTypeFixture = require(`../__fixtures__/restricted-content-type`) -const pluginOptions = { spaceId: `testSpaceId` } +const defaultPluginOptions = { spaceId: `testSpaceId` } const createMockCache = () => { + const actualCacheMap = new Map() return { - get: jest.fn(), - set: jest.fn(), + get: jest.fn(key => actualCacheMap.get(key)), + set: jest.fn((key, value) => actualCacheMap.set(key, value)), directory: __dirname, + actualMap: actualCacheMap, } } @@ -30,14 +32,16 @@ describe(`gatsby-node`, () => { const schema = { buildObjectType: jest.fn() } const store = {} const cache = createMockCache() - const getCache = jest.fn() + const getCache = jest.fn(() => cache) const reporter = { info: jest.fn(), verbose: jest.fn(), + panic: jest.fn(), activityTimer: () => { return { start: jest.fn(), end: jest.fn() } }, } + const parentSpan = {} const createNodeId = jest.fn(value => value) let currentNodeMap const getNodes = () => Array.from(currentNodeMap.values()) @@ -46,6 +50,43 @@ describe(`gatsby-node`, () => { const getFieldValue = (value, locale, defaultLocale) => value[locale] ?? value[defaultLocale] + const simulateGatsbyBuild = async function ( + pluginOptions = defaultPluginOptions + ) { + await gatsbyNode.onPreBootstrap( + { + reporter, + cache, + actions, + parentSpan, + getNode, + getNodes, + createNodeId, + }, + pluginOptions + ) + + await gatsbyNode.createSchemaCustomization( + { schema, actions, cache }, + pluginOptions + ) + + await gatsbyNode.sourceNodes( + { + actions, + store, + getNodes, + getNode, + reporter, + createNodeId, + cache, + getCache, + schema, + }, + pluginOptions + ) + } + const testIfContentTypesExists = contentTypeItems => { contentTypeItems.forEach(contentType => { const contentTypeId = createNodeId(contentType.name) @@ -267,6 +308,7 @@ describe(`gatsby-node`, () => { status: {}, } }) + cache.actualMap.clear() }) it(`should create nodes from initial payload`, async () => { @@ -275,20 +317,7 @@ describe(`gatsby-node`, () => { fetch.mockImplementationOnce(startersBlogFixture.initialSync) const locales = [`en-US`, `nl`] - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() testIfContentTypesExists(startersBlogFixture.initialSync().contentTypeItems) testIfEntriesExists( @@ -308,7 +337,23 @@ describe(`gatsby-node`, () => { expect(cache.get).toHaveBeenCalledWith( `contentful-sync-data-testSpaceId-master` ) - expect(cache.get.mock.calls.length).toBe(2) + + expect(cache.get.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + "contentful-sync-token-testSpaceId-master", + ], + Array [ + "contentful-sync-data-testSpaceId-master", + ], + Array [ + "contentful-sync-result-testSpaceId-master", + ], + Array [ + "contentful-sync-result-testSpaceId-master", + ], + ] + `) // Stores sync token and raw/unparsed data to the cache expect(cache.set).toHaveBeenCalledWith( @@ -320,9 +365,16 @@ describe(`gatsby-node`, () => { { entries: startersBlogFixture.initialSync().currentSyncData.entries, assets: startersBlogFixture.initialSync().currentSyncData.assets, + tagItems: [], } ) - expect(cache.set.mock.calls.length).toBe(2) + expect(cache.set.mock.calls.map(v => v[0])).toMatchInlineSnapshot(` + Array [ + "contentful-sync-data-testSpaceId-master", + "contentful-sync-token-testSpaceId-master", + "contentful-sync-result-testSpaceId-master", + ] + `) }) it(`should add a new blogpost and update linkedNodes`, async () => { @@ -345,20 +397,7 @@ describe(`gatsby-node`, () => { ) // initial sync - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() // check if blog posts do not exists createdBlogEntryIds.forEach(entryId => { @@ -366,20 +405,8 @@ describe(`gatsby-node`, () => { }) // add new blog post - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() + testIfContentTypesExists( startersBlogFixture.createBlogPost().contentTypeItems ) @@ -420,56 +447,17 @@ describe(`gatsby-node`, () => { ) // initial sync - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() // create blog post - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() updatedBlogEntryIds.forEach(blogEntryId => { expect(getNode(blogEntryId).title).toBe(`Integration tests`) }) // updated blog post - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() testIfContentTypesExists( startersBlogFixture.updateBlogPost().contentTypeItems @@ -515,36 +503,10 @@ describe(`gatsby-node`, () => { ) // initial sync - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() // create blog post - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() const authorIds = [] // check if blog post exists @@ -555,20 +517,7 @@ describe(`gatsby-node`, () => { }) // remove blog post - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() testIfContentTypesExists( startersBlogFixture.removeBlogPost().contentTypeItems @@ -605,33 +554,10 @@ describe(`gatsby-node`, () => { ) // initial sync - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() // create blog post - await gatsbyNode.sourceNodes({ - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }) + await simulateGatsbyBuild() // check if blog post exists removedAssetEntryIds.forEach(assetId => { @@ -645,20 +571,7 @@ describe(`gatsby-node`, () => { ) // remove asset - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() testIfContentTypesExists(startersBlogFixture.removeAsset().contentTypeItems) testIfEntriesExists( @@ -676,20 +589,7 @@ describe(`gatsby-node`, () => { fetch.mockImplementationOnce(richTextFixture.initialSync) // initial sync - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() const initNodes = getNodes() @@ -710,30 +610,12 @@ describe(`gatsby-node`, () => { fetch.mockImplementationOnce(startersBlogFixture.initialSync) const locales = [`en-US`, `nl`] - const mockPanicReporter = { - ...reporter, - panic: jest.fn(), - } - - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter: mockPanicReporter, - createNodeId, - cache, - getCache, - schema, - }, - { - ...pluginOptions, - localeFilter: () => false, - } - ) + await simulateGatsbyBuild({ + ...defaultPluginOptions, + localeFilter: () => false, + }) - expect(mockPanicReporter.panic).toBeCalledWith( + expect(reporter.panic).toBeCalledWith( expect.objectContaining({ context: { sourceMessage: `Please check if your localeFilter is configured properly. Locales '${locales.join( @@ -749,27 +631,9 @@ describe(`gatsby-node`, () => { cache.set.mockClear() fetch.mockImplementationOnce(restrictedContentTypeFixture.initialSync) - const mockPanicReporter = { - ...reporter, - panic: jest.fn(), - } - - await gatsbyNode.sourceNodes( - { - actions, - store, - getNodes, - getNode, - reporter: mockPanicReporter, - createNodeId, - cache, - getCache, - schema, - }, - pluginOptions - ) + await simulateGatsbyBuild() - expect(mockPanicReporter.panic).toBeCalledWith( + expect(reporter.panic).toBeCalledWith( expect.objectContaining({ context: { sourceMessage: `Restricted ContentType name found. The name "reference" is not allowed.`, diff --git a/packages/gatsby-source-contentful/src/create-schema-customization.js b/packages/gatsby-source-contentful/src/create-schema-customization.js new file mode 100644 index 0000000000000..585561d086d88 --- /dev/null +++ b/packages/gatsby-source-contentful/src/create-schema-customization.js @@ -0,0 +1,83 @@ +const _ = require(`lodash`) + +const { createPluginConfig } = require(`./plugin-options`) + +export async function createSchemaCustomization( + { schema, actions, cache }, + pluginOptions +) { + const { createTypes } = actions + + const pluginConfig = createPluginConfig(pluginOptions) + const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( + `environment` + )}` + + const { contentTypeItems } = await cache.get( + `contentful-sync-result-${sourceId}` + ) + + createTypes(` + interface ContentfulEntry implements Node { + contentful_id: String! + id: ID! + node_locale: String! + } + `) + + createTypes(` + interface ContentfulReference { + contentful_id: String! + id: ID! + } + `) + + createTypes( + schema.buildObjectType({ + name: `ContentfulAsset`, + fields: { + contentful_id: { type: `String!` }, + id: { type: `ID!` }, + }, + interfaces: [`ContentfulReference`, `Node`], + }) + ) + + // Create types for each content type + const gqlTypes = contentTypeItems.map(contentTypeItem => + schema.buildObjectType({ + name: _.upperFirst( + _.camelCase( + `Contentful ${ + pluginConfig.get(`useNameForId`) + ? contentTypeItem.name + : contentTypeItem.sys.id + }` + ) + ), + fields: { + contentful_id: { type: `String!` }, + id: { type: `ID!` }, + node_locale: { type: `String!` }, + }, + interfaces: [`ContentfulReference`, `ContentfulEntry`, `Node`], + }) + ) + + createTypes(gqlTypes) + + if (pluginConfig.get(`enableTags`)) { + createTypes( + schema.buildObjectType({ + name: `ContentfulTag`, + fields: { + name: { type: `String!` }, + contentful_id: { type: `String!` }, + id: { type: `ID!` }, + }, + interfaces: [`Node`], + extensions: { dontInfer: {} }, + }) + ) + } +} diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index 5f83bc7996a96..249287e7b1857 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -1,28 +1,20 @@ -const path = require(`path`) -const isOnline = require(`is-online`) +// @todo import syntax! const _ = require(`lodash`) -const fs = require(`fs-extra`) -const { createClient } = require(`contentful`) -const v8 = require(`v8`) const fetch = require(`@vercel/fetch-retry`)(require(`node-fetch`)) -const { CODES } = require(`./report`) -const normalize = require(`./normalize`) -const fetchData = require(`./fetch`) -const { createPluginConfig, maskText } = require(`./plugin-options`) -const { downloadContentfulAssets } = require(`./download-contentful-assets`) +const { createSchemaCustomization } = require(`./create-schema-customization`) +const { sourceNodes } = require(`./source-nodes`) +const { onPreBootstrap } = require(`./on-pre-bootstrap`) +const { maskText } = require(`./plugin-options`) -const conflictFieldPrefix = `contentful` +exports.setFieldsOnGraphQLNodeType = + require(`./extend-node-type`).extendNodeType + +exports.createSchemaCustomization = createSchemaCustomization + +exports.sourceNodes = sourceNodes -// restrictedNodeFields from here https://www.gatsbyjs.org/docs/node-interface/ -const restrictedNodeFields = [ - `children`, - `contentful_id`, - `fields`, - `id`, - `internal`, - `parent`, -] +exports.onPreBootstrap = onPreBootstrap exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`).extendNodeType @@ -149,597 +141,3 @@ List of locales and their codes can be found in Contentful app -> Settings -> Lo .external(validateContentfulAccess) exports.pluginOptionsSchema = pluginOptionsSchema - -/*** - * Localization algorithm - * - * 1. Make list of all resolvable IDs worrying just about the default ids not - * localized ids - * 2. Make mapping between ids, again not worrying about localization. - * 3. When creating entries and assets, make the most localized version - * possible for each localized node i.e. get the localized field if it exists - * or the fallback field or the default field. - */ - -exports.sourceNodes = async ( - { - actions, - getNode, - getNodes, - getNodesByType, - createNodeId, - store, - cache, - getCache, - reporter, - schema, - parentSpan, - }, - pluginOptions -) => { - const { createNode, deleteNode, touchNode, createTypes } = actions - - let currentSyncData - let contentTypeItems - let tagItems - let defaultLocale - let locales - let space - if (process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE: Storing/loading remote data through \`` + - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE + - `\` so Remote changes CAN NOT be detected!` - ) - } - const forceCache = await fs.exists( - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE - ) - - const pluginConfig = createPluginConfig(pluginOptions) - const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( - `environment` - )}` - - const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` - const CACHE_SYNC_DATA = `contentful-sync-data-${sourceId}` - - /* - * Subsequent calls of Contentfuls sync API return only changed data. - * - * In some cases, especially when using rich-text fields, there can be data - * missing from referenced entries. This breaks the reference matching. - * - * To workround this, we cache the initial sync data and merge it - * with all data from subsequent syncs. Afterwards the references get - * resolved via the Contentful JS SDK. - */ - const syncToken = await cache.get(CACHE_SYNC_TOKEN) - let previousSyncData = { - assets: [], - entries: [], - } - const cachedData = await cache.get(CACHE_SYNC_DATA) - - if (cachedData) { - previousSyncData = cachedData - } - - const fetchActivity = reporter.activityTimer( - `Contentful: Fetch data (${sourceId})`, - { - parentSpan, - } - ) - - if (forceCache) { - // If the cache has data, use it. Otherwise do a remote fetch anyways and prime the cache now. - // If present, do NOT contact contentful, skip the round trips entirely - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Skipping remote fetch, using data stored in \`${process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE}\`` - ) - ;({ - currentSyncData, - contentTypeItems, - tagItems, - defaultLocale, - locales, - space, - } = v8.deserialize( - fs.readFileSync(process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) - )) - } else { - const online = await isOnline() - - // If the user knows they are offline, serve them cached result - // For prod builds though always fail if we can't get the latest data - if ( - !online && - process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && - process.env.NODE_ENV !== `production` - ) { - getNodes().forEach(node => { - if (node.internal.owner !== `gatsby-source-contentful`) { - return - } - touchNode(node) - if (node.localFile___NODE) { - // Prevent GraphQL type inference from crashing on this property - touchNode(getNode(node.localFile___NODE)) - } - }) - - reporter.info(`Using Contentful Offline cache ⚠️`) - reporter.info( - `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` - ) - - return - } - if (process.env.GATSBY_CONTENTFUL_OFFLINE) { - reporter.info( - `Note: \`GATSBY_CONTENTFUL_OFFLINE\` was set but it either was not \`true\`, we _are_ online, or we are in production mode, so the flag is ignored.` - ) - } - - fetchActivity.start() - ;({ - currentSyncData, - contentTypeItems, - tagItems, - defaultLocale, - locales, - space, - } = await fetchData({ - syncToken, - reporter, - pluginConfig, - parentSpan, - })) - - if (process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Writing v8 serialized glob of remote data to: ` + - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE - ) - fs.writeFileSync( - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE, - v8.serialize({ - currentSyncData, - contentTypeItems, - defaultLocale, - locales, - space, - }) - ) - } - } - - // Check for restricted content type names - const useNameForId = pluginConfig.get(`useNameForId`) - const restrictedContentTypes = [`entity`, `reference`, `asset`] - - if (pluginConfig.get(`enableTags`)) { - restrictedContentTypes.push(`tags`) - } - - contentTypeItems.forEach(contentTypeItem => { - // Establish identifier for content type - // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, - // but sometimes a base62 uuid generated by Contentful, hence the option) - let contentTypeItemId - if (useNameForId) { - contentTypeItemId = contentTypeItem.name.toLowerCase() - } else { - contentTypeItemId = contentTypeItem.sys.id.toLowerCase() - } - - if (restrictedContentTypes.includes(contentTypeItemId)) { - reporter.panic({ - id: CODES.FetchContentTypes, - context: { - sourceMessage: `Restricted ContentType name found. The name "${contentTypeItemId}" is not allowed.`, - }, - }) - } - }) - - const allLocales = locales - locales = locales.filter(pluginConfig.get(`localeFilter`)) - reporter.verbose( - `Default locale: ${defaultLocale}. All locales: ${allLocales - .map(({ code }) => code) - .join(`, `)}` - ) - if (allLocales.length !== locales.length) { - reporter.verbose( - `After plugin.options.localeFilter: ${locales - .map(({ code }) => code) - .join(`, `)}` - ) - } - if (locales.length === 0) { - reporter.panic({ - id: CODES.LocalesMissing, - context: { - sourceMessage: `Please check if your localeFilter is configured properly. Locales '${allLocales - .map(item => item.code) - .join(`,`)}' were found but were filtered down to none.`, - }, - }) - } - - if (pluginConfig.get(`enableTags`)) { - createTypes( - schema.buildObjectType({ - name: `ContentfulTag`, - fields: { - name: { type: `String!` }, - contentful_id: { type: `String!` }, - id: { type: `ID!` }, - }, - interfaces: [`Node`], - extensions: { dontInfer: {} }, - }) - ) - } - - createTypes(` - interface ContentfulEntry implements Node { - contentful_id: String! - id: ID! - node_locale: String! - } -`) - - createTypes(` - interface ContentfulReference { - contentful_id: String! - id: ID! - } -`) - - createTypes( - schema.buildObjectType({ - name: `ContentfulAsset`, - fields: { - contentful_id: { type: `String!` }, - id: { type: `ID!` }, - }, - interfaces: [`ContentfulReference`, `Node`], - }) - ) - - const gqlTypes = contentTypeItems.map(contentTypeItem => - schema.buildObjectType({ - name: _.upperFirst( - _.camelCase( - `Contentful ${ - pluginConfig.get(`useNameForId`) - ? contentTypeItem.name - : contentTypeItem.sys.id - }` - ) - ), - fields: { - contentful_id: { type: `String!` }, - id: { type: `ID!` }, - node_locale: { type: `String!` }, - }, - interfaces: [`ContentfulReference`, `ContentfulEntry`, `Node`], - }) - ) - - createTypes(gqlTypes) - - fetchActivity.end() - - const processingActivity = reporter.activityTimer( - `Contentful: Process data (${sourceId})`, - { - parentSpan, - } - ) - processingActivity.start() - - // Create a map of up to date entries and assets - function mergeSyncData(previous, current, deleted) { - const entryMap = new Map() - previous.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) - current.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) - return [...entryMap.values()] - } - - const deletedSet = new Set(currentSyncData.deletedEntries.map(e => e.sys.id)) - - const mergedSyncData = { - entries: mergeSyncData( - previousSyncData.entries, - currentSyncData.entries, - deletedSet - ), - assets: mergeSyncData( - previousSyncData.assets, - currentSyncData.assets, - deletedSet - ), - } - - // Store a raw and unresolved copy of the data for caching - const mergedSyncDataRaw = _.cloneDeep(mergedSyncData) - - // Use the JS-SDK to resolve the entries and assets - const res = createClient({ - space: `none`, - accessToken: `fake-access-token`, - }).parseEntries({ - items: mergedSyncData.entries, - includes: { - assets: mergedSyncData.assets, - entries: mergedSyncData.entries, - }, - }) - - mergedSyncData.entries = res.items - - // Inject raw API output to rich text fields - const richTextFieldMap = new Map() - contentTypeItems.forEach(contentType => { - richTextFieldMap.set( - contentType.sys.id, - contentType.fields - .filter(field => field.type === `RichText`) - .map(field => field.id) - ) - }) - - const rawEntries = new Map() - mergedSyncDataRaw.entries.forEach(rawEntry => - rawEntries.set(rawEntry.sys.id, rawEntry) - ) - - mergedSyncData.entries.forEach(entry => { - const contentTypeId = entry.sys.contentType.sys.id - const richTextFieldIds = richTextFieldMap.get(contentTypeId) - if (richTextFieldIds) { - richTextFieldIds.forEach(richTextFieldId => { - if (!entry.fields[richTextFieldId]) { - return - } - entry.fields[richTextFieldId] = rawEntries.get(entry.sys.id).fields[ - richTextFieldId - ] - }) - } - }) - - const entryList = normalize.buildEntryList({ - mergedSyncData, - contentTypeItems, - }) - - // Remove deleted entries & assets. - // TODO figure out if entries referencing now deleted entries/assets - // are "updated" so will get the now deleted reference removed. - - function deleteContentfulNode(node) { - const normalizedType = node.sys.type.startsWith(`Deleted`) - ? node.sys.type.substring(`Deleted`.length) - : node.sys.type - - const localizedNodes = locales - .map(locale => { - const nodeId = createNodeId( - normalize.makeId({ - spaceId: space.sys.id, - id: node.sys.id, - type: normalizedType, - currentLocale: locale.code, - defaultLocale, - }) - ) - return getNode(nodeId) - }) - .filter(node => node) - - localizedNodes.forEach(node => { - // touchNode first, to populate typeOwners & avoid erroring - touchNode(node) - deleteNode(node) - }) - } - - currentSyncData.deletedEntries.forEach(deleteContentfulNode) - currentSyncData.deletedAssets.forEach(deleteContentfulNode) - - const existingNodes = getNodes().filter( - n => - n.internal.owner === `gatsby-source-contentful` && - (n?.sys?.type === `Asset` || n?.sys?.type === `Entry`) - ) - existingNodes.forEach(n => touchNode(n)) - - const assets = mergedSyncData.assets - - reporter.info(`Updated entries ${currentSyncData.entries.length}`) - reporter.info(`Deleted entries ${currentSyncData.deletedEntries.length}`) - reporter.info(`Updated assets ${currentSyncData.assets.length}`) - reporter.info(`Deleted assets ${currentSyncData.deletedAssets.length}`) - - // Update syncToken - const nextSyncToken = currentSyncData.nextSyncToken - - await Promise.all([ - cache.set(CACHE_SYNC_DATA, mergedSyncDataRaw), - cache.set(CACHE_SYNC_TOKEN, nextSyncToken), - ]) - - reporter.verbose(`Building Contentful reference map`) - - // Create map of resolvable ids so we can check links against them while creating - // links. - const resolvable = normalize.buildResolvableSet({ - existingNodes, - entryList, - assets, - defaultLocale, - locales, - space, - }) - - // Build foreign reference map before starting to insert any nodes - const foreignReferenceMap = normalize.buildForeignReferenceMap({ - contentTypeItems, - entryList, - resolvable, - defaultLocale, - locales, - space, - useNameForId: pluginConfig.get(`useNameForId`), - }) - - reporter.verbose(`Resolving Contentful references`) - - const newOrUpdatedEntries = new Set() - entryList.forEach(entries => { - entries.forEach(entry => { - newOrUpdatedEntries.add(`${entry.sys.id}___${entry.sys.type}`) - }) - }) - - // Update existing entry nodes that weren't updated but that need reverse - // links added. - existingNodes - .filter(n => newOrUpdatedEntries.has(`${n.id}___${n.sys.type}`)) - .forEach(n => { - if (foreignReferenceMap[`${n.id}___${n.sys.type}`]) { - foreignReferenceMap[`${n.id}___${n.sys.type}`].forEach( - foreignReference => { - // Add reverse links - if (n[foreignReference.name]) { - n[foreignReference.name].push(foreignReference.id) - // It might already be there so we'll uniquify after pushing. - n[foreignReference.name] = _.uniq(n[foreignReference.name]) - } else { - // If is one foreign reference, there can always be many. - // Best to be safe and put it in an array to start with. - n[foreignReference.name] = [foreignReference.id] - } - } - ) - } - }) - - processingActivity.end() - - const creationActivity = reporter.activityTimer( - `Contentful: Create nodes (${sourceId})`, - { - parentSpan, - } - ) - creationActivity.start() - - for (let i = 0; i < contentTypeItems.length; i++) { - const contentTypeItem = contentTypeItems[i] - - if (entryList[i].length) { - reporter.info( - `Creating ${entryList[i].length} Contentful ${ - pluginConfig.get(`useNameForId`) - ? contentTypeItem.name - : contentTypeItem.sys.id - } nodes` - ) - } - - // A contentType can hold lots of entries which create nodes - // We wait until all nodes are created and processed until we handle the next one - // TODO add batching in gatsby-core - await Promise.all( - normalize.createNodesForContentType({ - contentTypeItem, - restrictedNodeFields, - conflictFieldPrefix, - entries: entryList[i], - createNode, - createNodeId, - getNode, - resolvable, - foreignReferenceMap, - defaultLocale, - locales, - space, - useNameForId: pluginConfig.get(`useNameForId`), - pluginConfig, - }) - ) - } - - if (assets.length) { - reporter.info(`Creating ${assets.length} Contentful Asset nodes`) - } - - for (let i = 0; i < assets.length; i++) { - // We wait for each asset to be process until handling the next one. - await Promise.all( - normalize.createAssetNodes({ - assetItem: assets[i], - createNode, - createNodeId, - defaultLocale, - locales, - space, - }) - ) - } - - // Create tags entities - if (tagItems.length) { - reporter.info(`Creating ${tagItems.length} Contentful Tag nodes`) - - for (const tag of tagItems) { - await createNode({ - id: createNodeId(`ContentfulTag__${space.sys.id}__${tag.sys.id}`), - name: tag.name, - contentful_id: tag.sys.id, - internal: { - type: `ContentfulTag`, - contentDigest: tag.sys.updatedAt, - }, - }) - } - } - - creationActivity.end() - - if (pluginConfig.get(`downloadLocal`)) { - reporter.info(`Download Contentful asset files`) - - await downloadContentfulAssets({ - actions, - createNodeId, - store, - cache, - getCache, - getNode, - getNodesByType, - reporter, - assetDownloadWorkers: pluginConfig.get(`assetDownloadWorkers`), - }) - } - - return -} - -// Check if there are any ContentfulAsset nodes and if gatsby-image is installed. If so, -// add fragments for ContentfulAsset and gatsby-image. The fragment will cause an error -// if there's not ContentfulAsset nodes and without gatsby-image, the fragment is useless. -exports.onPreExtractQueries = async ({ store }) => { - const program = store.getState().program - - const CACHE_DIR = path.resolve( - `${program.directory}/.cache/contentful/assets/` - ) - await fs.ensureDir(CACHE_DIR) -} diff --git a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js new file mode 100644 index 0000000000000..649a4b0d0aabe --- /dev/null +++ b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js @@ -0,0 +1,356 @@ +// @todo import syntax! +import normalize from "./normalize" +const isOnline = require(`is-online`) +const _ = require(`lodash`) +const fs = require(`fs-extra`) +const { createClient } = require(`contentful`) +const v8 = require(`v8`) +const { CODES } = require(`./report`) +const fetchData = require(`./fetch`) +const { createPluginConfig } = require(`./plugin-options`) + +export async function onPreBootstrap( + { reporter, cache, actions, parentSpan, getNode, getNodes, createNodeId }, + pluginOptions +) { + // Fetch data for sourceNodes & createSchemaCustomization + const { deleteNode, touchNode } = actions + + let currentSyncData + let contentTypeItems + let tagItems + let defaultLocale + let locales + let space + if (process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { + reporter.info( + `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE: Storing/loading remote data through \`` + + process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE + + `\` so Remote changes CAN NOT be detected!` + ) + } + const forceCache = await fs.exists( + process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE + ) + + const pluginConfig = createPluginConfig(pluginOptions) + const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( + `environment` + )}` + + const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` + const CACHE_SYNC_DATA = `contentful-sync-data-${sourceId}` + + /* + * Subsequent calls of Contentfuls sync API return only changed data. + * + * In some cases, especially when using rich-text fields, there can be data + * missing from referenced entries. This breaks the reference matching. + * + * To workround this, we cache the initial sync data and merge it + * with all data from subsequent syncs. Afterwards the references get + * resolved via the Contentful JS SDK. + */ + const syncToken = await cache.get(CACHE_SYNC_TOKEN) + let previousSyncData = { + assets: [], + entries: [], + } + const cachedData = await cache.get(CACHE_SYNC_DATA) + + if (cachedData) { + previousSyncData = cachedData + } + + const fetchActivity = reporter.activityTimer( + `Contentful: Fetch data (${sourceId})`, + { + parentSpan, + } + ) + + // If the cache has data, use it. Otherwise do a remote fetch anyways and prime the cache now. + // If present, do NOT contact contentful, skip the round trips entirely + if (forceCache) { + reporter.info( + `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Skipping remote fetch, using data stored in \`${process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE}\`` + ) + ;({ + currentSyncData, + contentTypeItems, + tagItems, + defaultLocale, + locales, + space, + } = v8.deserialize( + fs.readFileSync(process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) + )) + } else { + const online = await isOnline() + + // If the user knows they are offline, serve them cached result + // For prod builds though always fail if we can't get the latest data + if ( + !online && + process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && + process.env.NODE_ENV !== `production` + ) { + getNodes().forEach(node => { + if (node.internal.owner !== `gatsby-source-contentful`) { + return + } + touchNode(node) + if (node.localFile___NODE) { + // Prevent GraphQL type inference from crashing on this property + touchNode(getNode(node.localFile___NODE)) + } + }) + + reporter.info(`Using Contentful Offline cache ⚠️`) + reporter.info( + `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` + ) + + return + } + if (process.env.GATSBY_CONTENTFUL_OFFLINE) { + reporter.info( + `Note: \`GATSBY_CONTENTFUL_OFFLINE\` was set but it either was not \`true\`, we _are_ online, or we are in production mode, so the flag is ignored.` + ) + } + + fetchActivity.start() + ;({ + currentSyncData, + contentTypeItems, + tagItems, + defaultLocale, + locales, + space, + } = await fetchData({ + syncToken, + reporter, + pluginConfig, + parentSpan, + })) + + if (process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { + reporter.info( + `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Writing v8 serialized glob of remote data to: ` + + process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE + ) + fs.writeFileSync( + process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE, + v8.serialize({ + currentSyncData, + contentTypeItems, + defaultLocale, + locales, + space, + }) + ) + } + } + + // Check for restricted content type names + const useNameForId = pluginConfig.get(`useNameForId`) + const restrictedContentTypes = [`entity`, `reference`, `asset`] + + if (pluginConfig.get(`enableTags`)) { + restrictedContentTypes.push(`tags`) + } + + contentTypeItems.forEach(contentTypeItem => { + // Establish identifier for content type + // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, + // but sometimes a base62 uuid generated by Contentful, hence the option) + let contentTypeItemId + if (useNameForId) { + contentTypeItemId = contentTypeItem.name.toLowerCase() + } else { + contentTypeItemId = contentTypeItem.sys.id.toLowerCase() + } + + if (restrictedContentTypes.includes(contentTypeItemId)) { + reporter.panic({ + id: CODES.FetchContentTypes, + context: { + sourceMessage: `Restricted ContentType name found. The name "${contentTypeItemId}" is not allowed.`, + }, + }) + } + }) + + const allLocales = locales + locales = locales.filter(pluginConfig.get(`localeFilter`)) + reporter.verbose( + `Default locale: ${defaultLocale}. All locales: ${allLocales + .map(({ code }) => code) + .join(`, `)}` + ) + if (allLocales.length !== locales.length) { + reporter.verbose( + `After plugin.options.localeFilter: ${locales + .map(({ code }) => code) + .join(`, `)}` + ) + } + if (locales.length === 0) { + reporter.panic({ + id: CODES.LocalesMissing, + context: { + sourceMessage: `Please check if your localeFilter is configured properly. Locales '${allLocales + .map(item => item.code) + .join(`,`)}' were found but were filtered down to none.`, + }, + }) + } + + fetchActivity.end() + + // Process data fetch results and turn them into GraphQL entities + const processingActivity = reporter.activityTimer( + `Contentful: Process data (${sourceId})`, + { + parentSpan, + } + ) + processingActivity.start() + + // Create a map of up to date entries and assets + function mergeSyncData(previous, current, deleted) { + const entryMap = new Map() + previous.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) + current.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) + return [...entryMap.values()] + } + + const mergedSyncData = { + entries: mergeSyncData( + previousSyncData.entries, + currentSyncData.entries, + new Set(currentSyncData.deletedEntries.map(e => e.sys.id)) + ), + assets: mergeSyncData( + previousSyncData.assets, + currentSyncData.assets, + new Set(currentSyncData.deletedAssets.map(e => e.sys.id)) + ), + tagItems, + } + + // Store a raw and unresolved copy of the data for caching + const mergedSyncDataRaw = _.cloneDeep(mergedSyncData) + + // Use the JS-SDK to resolve the entries and assets + const res = createClient({ + space: `none`, + accessToken: `fake-access-token`, + }).parseEntries({ + items: mergedSyncData.entries, + includes: { + assets: mergedSyncData.assets, + entries: mergedSyncData.entries, + }, + }) + + mergedSyncData.entries = res.items + + // Inject raw API output to rich text fields + const richTextFieldMap = new Map() + contentTypeItems.forEach(contentType => { + richTextFieldMap.set( + contentType.sys.id, + contentType.fields + .filter(field => field.type === `RichText`) + .map(field => field.id) + ) + }) + + const rawEntries = new Map() + mergedSyncDataRaw.entries.forEach(rawEntry => + rawEntries.set(rawEntry.sys.id, rawEntry) + ) + + mergedSyncData.entries.forEach(entry => { + const contentTypeId = entry.sys.contentType.sys.id + const richTextFieldIds = richTextFieldMap.get(contentTypeId) + if (richTextFieldIds) { + richTextFieldIds.forEach(richTextFieldId => { + if (!entry.fields[richTextFieldId]) { + return + } + entry.fields[richTextFieldId] = rawEntries.get(entry.sys.id).fields[ + richTextFieldId + ] + }) + } + }) + + // @todo based on the sys metadata we should be able to differentiate new and updated entities + reporter.info( + `Contentful: ${currentSyncData.entries.length} new/updated entries` + ) + reporter.info( + `Contentful: ${currentSyncData.deletedEntries.length} deleted entries` + ) + reporter.info(`Contentful: ${previousSyncData.entries.length} cached entries`) + reporter.info( + `Contentful: ${currentSyncData.assets.length} new/updated assets` + ) + reporter.info(`Contentful: ${previousSyncData.assets.length} cached assets`) + reporter.info( + `Contentful: ${currentSyncData.deletedAssets.length} deleted assets` + ) + + // Remove deleted entries & assets + reporter.verbose(`Removing deleted Contentful entries & assets`) + + // @todo this should happen when sourcing? + function deleteContentfulNode(node) { + const normalizedType = node.sys.type.startsWith(`Deleted`) + ? node.sys.type.substring(`Deleted`.length) + : node.sys.type + + const localizedNodes = locales + .map(locale => { + const nodeId = createNodeId( + normalize.makeId({ + spaceId: space.sys.id, + id: node.sys.id, + type: normalizedType, + currentLocale: locale.code, + defaultLocale, + }) + ) + return getNode(nodeId) + }) + .filter(node => node) + + localizedNodes.forEach(node => { + // touchNode first, to populate typeOwners & avoid erroring + touchNode(node) + deleteNode(node) + }) + } + + currentSyncData.deletedEntries.forEach(deleteContentfulNode) + currentSyncData.deletedAssets.forEach(deleteContentfulNode) + + // Update syncToken + const nextSyncToken = currentSyncData.nextSyncToken + + await Promise.all([ + cache.set(CACHE_SYNC_DATA, mergedSyncDataRaw), + cache.set(CACHE_SYNC_TOKEN, nextSyncToken), + cache.set(`contentful-sync-result-${sourceId}`, { + mergedSyncData, + contentTypeItems, + locales, + space, + defaultLocale, + }), + ]) + + processingActivity.end() +} diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js new file mode 100644 index 0000000000000..28555d48e9698 --- /dev/null +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -0,0 +1,234 @@ +// @todo import syntax! +import _ from "lodash" +const path = require(`path`) +const fs = require(`fs-extra`) +import normalize from "./normalize" + +const { createPluginConfig } = require(`./plugin-options`) +import { downloadContentfulAssets } from "./download-contentful-assets" + +const conflictFieldPrefix = `contentful` + +// restrictedNodeFields from here https://www.gatsbyjs.org/docs/node-interface/ +const restrictedNodeFields = [ + `children`, + `contentful_id`, + `fields`, + `id`, + `internal`, + `parent`, +] + +/*** + * Localization algorithm + * + * 1. Make list of all resolvable IDs worrying just about the default ids not + * localized ids + * 2. Make mapping between ids, again not worrying about localization. + * 3. When creating entries and assets, make the most localized version + * possible for each localized node i.e. get the localized field if it exists + * or the fallback field or the default field. + */ + +export async function sourceNodes( + { + actions, + getNode, + getNodes, + getNodesByType, + createNodeId, + store, + cache, + getCache, + reporter, + parentSpan, + }, + pluginOptions +) { + const { createNode, touchNode } = actions + + const pluginConfig = createPluginConfig(pluginOptions) + const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( + `environment` + )}` + + const { mergedSyncData, contentTypeItems, locales, space, defaultLocale } = + await cache.get(`contentful-sync-result-${sourceId}`) + + const { assets, tagItems } = mergedSyncData + + const entryList = normalize.buildEntryList({ + mergedSyncData, + contentTypeItems, + }) + + const existingNodes = getNodes().filter( + n => n.internal.owner === `gatsby-source-contentful` + ) + existingNodes.forEach(n => touchNode(n)) + + reporter.verbose(`Building Contentful reference map`) + + // Create map of resolvable ids so we can check links against them while creating + // links. + const resolvable = normalize.buildResolvableSet({ + existingNodes, + entryList, + assets, + defaultLocale, + locales, + space, + }) + + // Build foreign reference map before starting to insert any nodes + const foreignReferenceMap = normalize.buildForeignReferenceMap({ + contentTypeItems, + entryList, + resolvable, + defaultLocale, + locales, + space, + useNameForId: pluginConfig.get(`useNameForId`), + }) + + reporter.verbose(`Resolving Contentful references`) + + const newOrUpdatedEntries = new Set() + entryList.forEach(entries => { + entries.forEach(entry => { + newOrUpdatedEntries.add(`${entry.sys.id}___${entry.sys.type}`) + }) + }) + + // Update existing entry nodes that weren't updated but that need reverse + // links added. + existingNodes + .filter(n => newOrUpdatedEntries.has(`${n.id}___${n.sys.type}`)) + .forEach(n => { + if (foreignReferenceMap[`${n.id}___${n.sys.type}`]) { + foreignReferenceMap[`${n.id}___${n.sys.type}`].forEach( + foreignReference => { + // Add reverse links + if (n[foreignReference.name]) { + n[foreignReference.name].push(foreignReference.id) + // It might already be there so we'll uniquify after pushing. + n[foreignReference.name] = _.uniq(n[foreignReference.name]) + } else { + // If is one foreign reference, there can always be many. + // Best to be safe and put it in an array to start with. + n[foreignReference.name] = [foreignReference.id] + } + } + ) + } + }) + + const creationActivity = reporter.activityTimer( + `Contentful: Create nodes (${sourceId})`, + { + parentSpan, + } + ) + creationActivity.start() + + for (let i = 0; i < contentTypeItems.length; i++) { + const contentTypeItem = contentTypeItems[i] + + if (entryList[i].length) { + reporter.info( + `Creating ${entryList[i].length} Contentful ${ + pluginConfig.get(`useNameForId`) + ? contentTypeItem.name + : contentTypeItem.sys.id + } nodes` + ) + } + + // A contentType can hold lots of entries which create nodes + // We wait until all nodes are created and processed until we handle the next one + // TODO add batching in gatsby-core + await Promise.all( + normalize.createNodesForContentType({ + contentTypeItem, + contentTypeItems, + restrictedNodeFields, + conflictFieldPrefix, + entries: entryList[i], + createNode, + createNodeId, + getNode, + resolvable, + foreignReferenceMap, + defaultLocale, + locales, + space, + useNameForId: pluginConfig.get(`useNameForId`), + pluginConfig, + }) + ) + } + + if (assets.length) { + reporter.info(`Creating ${assets.length} Contentful asset nodes`) + } + + for (let i = 0; i < assets.length; i++) { + // We wait for each asset to be process until handling the next one. + await Promise.all( + normalize.createAssetNodes({ + assetItem: assets[i], + createNode, + createNodeId, + defaultLocale, + locales, + space, + }) + ) + } + + // Create tags entities + if (tagItems.length) { + reporter.info(`Creating ${tagItems.length} Contentful Tag nodes`) + + for (const tag of tagItems) { + await createNode({ + id: createNodeId(`ContentfulTag__${space.sys.id}__${tag.sys.id}`), + name: tag.name, + contentful_id: tag.sys.id, + internal: { + type: `ContentfulTag`, + contentDigest: tag.sys.updatedAt, + }, + }) + } + } + + creationActivity.end() + + // @todo add own activity! + + if (pluginConfig.get(`downloadLocal`)) { + reporter.info(`Download Contentful asset files`) + + // Ensure cache dir exists for downloadLocal + const program = store.getState().program + + const CACHE_DIR = path.resolve( + `${program.directory}/.cache/contentful/assets/` + ) + + await fs.ensureDir(CACHE_DIR) + + await downloadContentfulAssets({ + actions, + createNodeId, + store, + cache, + getCache, + getNode, + getNodesByType, + reporter, + assetDownloadWorkers: pluginConfig.get(`assetDownloadWorkers`), + }) + } +} From 12daa97a2c43bef2309c91f372b77d0c204cb064 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Thu, 16 Sep 2021 13:30:06 +0200 Subject: [PATCH 02/27] move all node manipulating code into sourceNodes --- .../src/__tests__/gatsby-node.js | 1 - .../src/on-pre-bootstrap.js | 62 ++----------- .../src/source-nodes.js | 86 ++++++++++++++++++- 3 files changed, 90 insertions(+), 59 deletions(-) diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index 249560fa2864a..53b988ad34a0d 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -365,7 +365,6 @@ describe(`gatsby-node`, () => { { entries: startersBlogFixture.initialSync().currentSyncData.entries, assets: startersBlogFixture.initialSync().currentSyncData.assets, - tagItems: [], } ) expect(cache.set.mock.calls.map(v => v[0])).toMatchInlineSnapshot(` diff --git a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js index 649a4b0d0aabe..fd20ebcc5f4a9 100644 --- a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js +++ b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js @@ -1,5 +1,4 @@ // @todo import syntax! -import normalize from "./normalize" const isOnline = require(`is-online`) const _ = require(`lodash`) const fs = require(`fs-extra`) @@ -10,11 +9,10 @@ const fetchData = require(`./fetch`) const { createPluginConfig } = require(`./plugin-options`) export async function onPreBootstrap( - { reporter, cache, actions, parentSpan, getNode, getNodes, createNodeId }, + { reporter, cache, parentSpan }, pluginOptions ) { // Fetch data for sourceNodes & createSchemaCustomization - const { deleteNode, touchNode } = actions let currentSyncData let contentTypeItems @@ -95,22 +93,6 @@ export async function onPreBootstrap( process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && process.env.NODE_ENV !== `production` ) { - getNodes().forEach(node => { - if (node.internal.owner !== `gatsby-source-contentful`) { - return - } - touchNode(node) - if (node.localFile___NODE) { - // Prevent GraphQL type inference from crashing on this property - touchNode(getNode(node.localFile___NODE)) - } - }) - - reporter.info(`Using Contentful Offline cache ⚠️`) - reporter.info( - `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` - ) - return } if (process.env.GATSBY_CONTENTFUL_OFFLINE) { @@ -218,7 +200,8 @@ export async function onPreBootstrap( processingActivity.start() // Create a map of up to date entries and assets - function mergeSyncData(previous, current, deleted) { + function mergeSyncData(previous, current, deletedEntities) { + const deleted = new Set(deletedEntities.map(e => e.sys.id)) const entryMap = new Map() previous.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) current.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) @@ -229,14 +212,13 @@ export async function onPreBootstrap( entries: mergeSyncData( previousSyncData.entries, currentSyncData.entries, - new Set(currentSyncData.deletedEntries.map(e => e.sys.id)) + currentSyncData.deletedEntries ), assets: mergeSyncData( previousSyncData.assets, currentSyncData.assets, - new Set(currentSyncData.deletedAssets.map(e => e.sys.id)) + currentSyncData.deletedAssets ), - tagItems, } // Store a raw and unresolved copy of the data for caching @@ -306,37 +288,6 @@ export async function onPreBootstrap( // Remove deleted entries & assets reporter.verbose(`Removing deleted Contentful entries & assets`) - // @todo this should happen when sourcing? - function deleteContentfulNode(node) { - const normalizedType = node.sys.type.startsWith(`Deleted`) - ? node.sys.type.substring(`Deleted`.length) - : node.sys.type - - const localizedNodes = locales - .map(locale => { - const nodeId = createNodeId( - normalize.makeId({ - spaceId: space.sys.id, - id: node.sys.id, - type: normalizedType, - currentLocale: locale.code, - defaultLocale, - }) - ) - return getNode(nodeId) - }) - .filter(node => node) - - localizedNodes.forEach(node => { - // touchNode first, to populate typeOwners & avoid erroring - touchNode(node) - deleteNode(node) - }) - } - - currentSyncData.deletedEntries.forEach(deleteContentfulNode) - currentSyncData.deletedAssets.forEach(deleteContentfulNode) - // Update syncToken const nextSyncToken = currentSyncData.nextSyncToken @@ -349,6 +300,9 @@ export async function onPreBootstrap( locales, space, defaultLocale, + tagItems, + deletedEntries: currentSyncData.deletedEntries, + deletedAssets: currentSyncData.deletedAssets, }), ]) diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index 28555d48e9698..92d09b70f09e7 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -45,17 +45,55 @@ export async function sourceNodes( }, pluginOptions ) { - const { createNode, touchNode } = actions + const isOnline = require(`is-online`) + const online = await isOnline() + const forceCache = await fs.exists( + process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE + ) + if ( + !online && + !forceCache && + process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && + process.env.NODE_ENV !== `production` + ) { + getNodes().forEach(node => { + if (node.internal.owner !== `gatsby-source-contentful`) { + return + } + touchNode(node) + if (node.localFile___NODE) { + // Prevent GraphQL type inference from crashing on this property + touchNode(getNode(node.localFile___NODE)) + } + }) + + reporter.info(`Using Contentful Offline cache ⚠️`) + reporter.info( + `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` + ) + + return + } + + const { createNode, touchNode, deleteNode } = actions const pluginConfig = createPluginConfig(pluginOptions) const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( `environment` )}` - const { mergedSyncData, contentTypeItems, locales, space, defaultLocale } = - await cache.get(`contentful-sync-result-${sourceId}`) + const { + mergedSyncData, + contentTypeItems, + locales, + space, + defaultLocale, + tagItems, + deletedEntries, + deletedAssets, + } = await cache.get(`contentful-sync-result-${sourceId}`) - const { assets, tagItems } = mergedSyncData + const { assets } = mergedSyncData const entryList = normalize.buildEntryList({ mergedSyncData, @@ -123,6 +161,46 @@ export async function sourceNodes( } }) + function deleteContentfulNode(node) { + const normalizedType = node.sys.type.startsWith(`Deleted`) + ? node.sys.type.substring(`Deleted`.length) + : node.sys.type + + const localizedNodes = locales + .map(locale => { + const nodeId = createNodeId( + normalize.makeId({ + spaceId: space.sys.id, + id: node.sys.id, + type: normalizedType, + currentLocale: locale.code, + defaultLocale, + }) + ) + return getNode(nodeId) + }) + .filter(node => node) + + localizedNodes.forEach(node => { + // touchNode first, to populate typeOwners & avoid erroring + touchNode(node) + deleteNode(node) + }) + } + + if (deletedEntries.length || deletedAssets.length) { + const deletionActivity = reporter.activityTimer( + `Contentful: Deleting ${deletedEntries.length} nodes and ${deletedAssets.length} assets (${sourceId})`, + { + parentSpan, + } + ) + deletionActivity.start() + deletedEntries.forEach(deleteContentfulNode) + deletedAssets.forEach(deleteContentfulNode) + deletionActivity.end() + } + const creationActivity = reporter.activityTimer( `Contentful: Create nodes (${sourceId})`, { From 53dc7f87a56c9c6cd36a2ed407c4958e1b1527ea Mon Sep 17 00:00:00 2001 From: axe312ger Date: Thu, 16 Sep 2021 15:01:58 +0200 Subject: [PATCH 03/27] move asset cache dir creation to pre bootstrap --- .../src/__tests__/gatsby-node.js | 6 +- .../src/on-pre-bootstrap.js | 81 +++---------------- .../src/source-nodes.js | 70 +++++++++++++--- 3 files changed, 78 insertions(+), 79 deletions(-) diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index 53b988ad34a0d..1dc2e38e9d63b 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -10,6 +10,7 @@ jest.mock(`gatsby-core-utils`, () => { const gatsbyNode = require(`../gatsby-node`) const fetch = require(`../fetch`) const normalize = require(`../normalize`) +const _ = require(`lodash`) const startersBlogFixture = require(`../__fixtures__/starter-blog-data`) const richTextFixture = require(`../__fixtures__/rich-text-data`) @@ -20,7 +21,7 @@ const defaultPluginOptions = { spaceId: `testSpaceId` } const createMockCache = () => { const actualCacheMap = new Map() return { - get: jest.fn(key => actualCacheMap.get(key)), + get: jest.fn(key => _.cloneDeep(actualCacheMap.get(key))), set: jest.fn((key, value) => actualCacheMap.set(key, value)), directory: __dirname, actualMap: actualCacheMap, @@ -349,6 +350,9 @@ describe(`gatsby-node`, () => { Array [ "contentful-sync-result-testSpaceId-master", ], + Array [ + "contentful-sync-data-testSpaceId-master", + ], Array [ "contentful-sync-result-testSpaceId-master", ], diff --git a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js index fd20ebcc5f4a9..ce42826ab7b67 100644 --- a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js +++ b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js @@ -2,18 +2,26 @@ const isOnline = require(`is-online`) const _ = require(`lodash`) const fs = require(`fs-extra`) -const { createClient } = require(`contentful`) +const path = require(`path`) const v8 = require(`v8`) const { CODES } = require(`./report`) const fetchData = require(`./fetch`) const { createPluginConfig } = require(`./plugin-options`) export async function onPreBootstrap( - { reporter, cache, parentSpan }, + { reporter, cache, parentSpan, store }, pluginOptions ) { - // Fetch data for sourceNodes & createSchemaCustomization + // Ensure cache dir exists for downloadLocal + const program = store.getState().program + + const CACHE_DIR = path.resolve( + `${program.directory}/.cache/contentful/assets/` + ) + + await fs.ensureDir(CACHE_DIR) + // Fetch data for sourceNodes & createSchemaCustomization let currentSyncData let contentTypeItems let tagItems @@ -188,17 +196,6 @@ export async function onPreBootstrap( }) } - fetchActivity.end() - - // Process data fetch results and turn them into GraphQL entities - const processingActivity = reporter.activityTimer( - `Contentful: Process data (${sourceId})`, - { - parentSpan, - } - ) - processingActivity.start() - // Create a map of up to date entries and assets function mergeSyncData(previous, current, deletedEntities) { const deleted = new Set(deletedEntities.map(e => e.sys.id)) @@ -221,54 +218,6 @@ export async function onPreBootstrap( ), } - // Store a raw and unresolved copy of the data for caching - const mergedSyncDataRaw = _.cloneDeep(mergedSyncData) - - // Use the JS-SDK to resolve the entries and assets - const res = createClient({ - space: `none`, - accessToken: `fake-access-token`, - }).parseEntries({ - items: mergedSyncData.entries, - includes: { - assets: mergedSyncData.assets, - entries: mergedSyncData.entries, - }, - }) - - mergedSyncData.entries = res.items - - // Inject raw API output to rich text fields - const richTextFieldMap = new Map() - contentTypeItems.forEach(contentType => { - richTextFieldMap.set( - contentType.sys.id, - contentType.fields - .filter(field => field.type === `RichText`) - .map(field => field.id) - ) - }) - - const rawEntries = new Map() - mergedSyncDataRaw.entries.forEach(rawEntry => - rawEntries.set(rawEntry.sys.id, rawEntry) - ) - - mergedSyncData.entries.forEach(entry => { - const contentTypeId = entry.sys.contentType.sys.id - const richTextFieldIds = richTextFieldMap.get(contentTypeId) - if (richTextFieldIds) { - richTextFieldIds.forEach(richTextFieldId => { - if (!entry.fields[richTextFieldId]) { - return - } - entry.fields[richTextFieldId] = rawEntries.get(entry.sys.id).fields[ - richTextFieldId - ] - }) - } - }) - // @todo based on the sys metadata we should be able to differentiate new and updated entities reporter.info( `Contentful: ${currentSyncData.entries.length} new/updated entries` @@ -285,17 +234,13 @@ export async function onPreBootstrap( `Contentful: ${currentSyncData.deletedAssets.length} deleted assets` ) - // Remove deleted entries & assets - reporter.verbose(`Removing deleted Contentful entries & assets`) - // Update syncToken const nextSyncToken = currentSyncData.nextSyncToken await Promise.all([ - cache.set(CACHE_SYNC_DATA, mergedSyncDataRaw), + cache.set(CACHE_SYNC_DATA, mergedSyncData), cache.set(CACHE_SYNC_TOKEN, nextSyncToken), cache.set(`contentful-sync-result-${sourceId}`, { - mergedSyncData, contentTypeItems, locales, space, @@ -306,5 +251,5 @@ export async function onPreBootstrap( }), ]) - processingActivity.end() + fetchActivity.end() } diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index 92d09b70f09e7..0aeff73189a14 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -3,6 +3,7 @@ import _ from "lodash" const path = require(`path`) const fs = require(`fs-extra`) import normalize from "./normalize" +const { createClient } = require(`contentful`) const { createPluginConfig } = require(`./plugin-options`) import { downloadContentfulAssets } from "./download-contentful-assets" @@ -82,8 +83,9 @@ export async function sourceNodes( `environment` )}` + const mergedSyncData = await cache.get(`contentful-sync-data-${sourceId}`) + const { - mergedSyncData, contentTypeItems, locales, space, @@ -93,6 +95,63 @@ export async function sourceNodes( deletedAssets, } = await cache.get(`contentful-sync-result-${sourceId}`) + // Process data fetch results and turn them into GraphQL entities + const processingActivity = reporter.activityTimer( + `Contentful: Process data (${sourceId})`, + { + parentSpan, + } + ) + processingActivity.start() + + // Store a raw and unresolved copy of the data for caching + const mergedSyncDataRaw = _.cloneDeep(mergedSyncData) + + // Use the JS-SDK to resolve the entries and assets + const res = createClient({ + space: `none`, + accessToken: `fake-access-token`, + }).parseEntries({ + items: mergedSyncData.entries, + includes: { + assets: mergedSyncData.assets, + entries: mergedSyncData.entries, + }, + }) + + mergedSyncData.entries = res.items + + // Inject raw API output to rich text fields + const richTextFieldMap = new Map() + contentTypeItems.forEach(contentType => { + richTextFieldMap.set( + contentType.sys.id, + contentType.fields + .filter(field => field.type === `RichText`) + .map(field => field.id) + ) + }) + + const rawEntries = new Map() + mergedSyncDataRaw.entries.forEach(rawEntry => + rawEntries.set(rawEntry.sys.id, rawEntry) + ) + + mergedSyncData.entries.forEach(entry => { + const contentTypeId = entry.sys.contentType.sys.id + const richTextFieldIds = richTextFieldMap.get(contentTypeId) + if (richTextFieldIds) { + richTextFieldIds.forEach(richTextFieldId => { + if (!entry.fields[richTextFieldId]) { + return + } + entry.fields[richTextFieldId] = rawEntries.get(entry.sys.id).fields[ + richTextFieldId + ] + }) + } + }) + const { assets } = mergedSyncData const entryList = normalize.buildEntryList({ @@ -288,15 +347,6 @@ export async function sourceNodes( if (pluginConfig.get(`downloadLocal`)) { reporter.info(`Download Contentful asset files`) - // Ensure cache dir exists for downloadLocal - const program = store.getState().program - - const CACHE_DIR = path.resolve( - `${program.directory}/.cache/contentful/assets/` - ) - - await fs.ensureDir(CACHE_DIR) - await downloadContentfulAssets({ actions, createNodeId, From 2ed4927b7f364b2a8efcbe3cbd59681c3525aed1 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Thu, 16 Sep 2021 15:03:14 +0200 Subject: [PATCH 04/27] fix(contentful): GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE now works with tags --- packages/gatsby-source-contentful/src/on-pre-bootstrap.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js index ce42826ab7b67..31c58843543b7 100644 --- a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js +++ b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js @@ -134,6 +134,7 @@ export async function onPreBootstrap( v8.serialize({ currentSyncData, contentTypeItems, + tagItems, defaultLocale, locales, space, From e5e685e6293d3ec00a8ed2c5b8be70bca1f9dd08 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Thu, 16 Sep 2021 15:13:11 +0200 Subject: [PATCH 05/27] fix GATSBY_CONTENTFUL_OFFLINE --- packages/gatsby-source-contentful/src/on-pre-bootstrap.js | 5 +++++ packages/gatsby-source-contentful/src/source-nodes.js | 8 +------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js index 31c58843543b7..344c937595126 100644 --- a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js +++ b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js @@ -101,6 +101,11 @@ export async function onPreBootstrap( process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && process.env.NODE_ENV !== `production` ) { + reporter.info(`Using Contentful Offline cache ⚠️`) + reporter.info( + `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` + ) + return } if (process.env.GATSBY_CONTENTFUL_OFFLINE) { diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index 0aeff73189a14..ae54ec775f1a9 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -46,6 +46,7 @@ export async function sourceNodes( }, pluginOptions ) { + const { createNode, touchNode, deleteNode } = actions const isOnline = require(`is-online`) const online = await isOnline() const forceCache = await fs.exists( @@ -68,16 +69,9 @@ export async function sourceNodes( } }) - reporter.info(`Using Contentful Offline cache ⚠️`) - reporter.info( - `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` - ) - return } - const { createNode, touchNode, deleteNode } = actions - const pluginConfig = createPluginConfig(pluginOptions) const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( `environment` From ca28e3a484ea6abd2f621a7a17e60bb4605f949d Mon Sep 17 00:00:00 2001 From: axe312ger Date: Thu, 16 Sep 2021 15:34:29 +0200 Subject: [PATCH 06/27] fix(contentful): add enableTags to plugin options --- packages/gatsby-source-contentful/src/gatsby-node.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index 249287e7b1857..dcf3b397dbff9 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -127,6 +127,11 @@ List of locales and their codes can be found in Contentful app -> Settings -> Lo If you are confident your Content Types will have natural-language IDs (e.g. \`blogPost\`), then you should set this option to \`false\`. If you are unable to ensure this, then you should leave this option set to \`true\` (the default).` ) .default(true), + enableTags: Joi.boolean() + .description( + `Enable the new tags feature. This will disallow the content type name "tags" till the next major version of this plugin.` + ) + .default(true), contentfulClientConfig: Joi.object() .description( `Additional config which will get passed to [Contentfuls JS SDK](https://github.com/contentful/contentful.js#configuration). From cbceb73fdfd58af1b8b8b82c0e0842c1c6ac2018 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Thu, 16 Sep 2021 15:34:56 +0200 Subject: [PATCH 07/27] fix(contentful): ensure enabled tags do not break warm builds --- packages/gatsby-source-contentful/src/source-nodes.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index ae54ec775f1a9..c66d9d39da974 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -154,7 +154,9 @@ export async function sourceNodes( }) const existingNodes = getNodes().filter( - n => n.internal.owner === `gatsby-source-contentful` + n => + n.internal.owner === `gatsby-source-contentful` && + n.internal.type !== `ContentfulTag` ) existingNodes.forEach(n => touchNode(n)) From d6a012f8ec17746e89dc2cba7cc104b0dd7a9e84 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Thu, 16 Sep 2021 15:45:09 +0200 Subject: [PATCH 08/27] fix unit tests --- .../src/__tests__/gatsby-node.js | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index 1dc2e38e9d63b..b15ff8bd9abf0 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -31,7 +31,11 @@ const createMockCache = () => { describe(`gatsby-node`, () => { const actions = { createTypes: jest.fn() } const schema = { buildObjectType: jest.fn() } - const store = {} + const store = { + getState: jest.fn(() => { + return { program: { directory: process.cwd() }, status: {} } + }), + } const cache = createMockCache() const getCache = jest.fn(() => cache) const reporter = { @@ -55,15 +59,7 @@ describe(`gatsby-node`, () => { pluginOptions = defaultPluginOptions ) { await gatsbyNode.onPreBootstrap( - { - reporter, - cache, - actions, - parentSpan, - getNode, - getNodes, - createNodeId, - }, + { reporter, cache, parentSpan, store }, pluginOptions ) @@ -304,11 +300,7 @@ describe(`gatsby-node`, () => { } actions.touchNode = jest.fn() actions.setPluginStatus = jest.fn() - store.getState = jest.fn(() => { - return { - status: {}, - } - }) + store.getState.mockClear() cache.actualMap.clear() }) From e5e752658627682afb6366495e3c5d095c0e9461 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Fri, 17 Sep 2021 13:57:41 +0200 Subject: [PATCH 09/27] refactor(contentful): fetch content types for schema and data for node sourcing --- .../__fixtures__/restricted-content-type.js | 83 +- .../src/__fixtures__/rich-text-data.js | 175 +-- .../src/__fixtures__/starter-blog-data.js | 1387 +++-------------- .../src/__tests__/fetch-backoff.js | 17 +- .../src/__tests__/fetch-network-errors.js | 17 +- .../src/__tests__/fetch.js | 24 +- .../src/__tests__/gatsby-node.js | 108 +- .../src/create-schema-customization.js | 71 +- .../gatsby-source-contentful/src/fetch.js | 144 +- .../gatsby-source-contentful/src/fs-cache.js | 26 + .../src/on-pre-bootstrap.js | 251 +-- .../src/source-nodes.js | 213 ++- 12 files changed, 820 insertions(+), 1696 deletions(-) create mode 100644 packages/gatsby-source-contentful/src/fs-cache.js diff --git a/packages/gatsby-source-contentful/src/__fixtures__/restricted-content-type.js b/packages/gatsby-source-contentful/src/__fixtures__/restricted-content-type.js index 86c5f7a2a02d2..8adda16238442 100644 --- a/packages/gatsby-source-contentful/src/__fixtures__/restricted-content-type.js +++ b/packages/gatsby-source-contentful/src/__fixtures__/restricted-content-type.js @@ -1,3 +1,45 @@ +exports.contentTypeItems = () => [ + { + sys: { + space: { + sys: { + type: `Link`, + linkType: `Space`, + id: `uzfinxahlog0`, + contentful_id: `uzfinxahlog0`, + }, + }, + id: `reference`, + type: `ContentType`, + createdAt: `2020-06-03T14:17:18.696Z`, + updatedAt: `2020-06-03T14:17:18.696Z`, + environment: { + sys: { + id: `master`, + type: `Link`, + linkType: `Environment`, + }, + }, + revision: 1, + contentful_id: `person`, + }, + displayField: `name`, + name: `Reference`, + description: ``, + fields: [ + { + id: `name`, + name: `Name`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + ], + }, +] + exports.initialSync = () => { return { currentSyncData: { @@ -7,47 +49,6 @@ exports.initialSync = () => { deletedAssets: [], nextSyncToken: `12345`, }, - contentTypeItems: [ - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `reference`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:18.696Z`, - updatedAt: `2020-06-03T14:17:18.696Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `person`, - }, - displayField: `name`, - name: `Reference`, - description: ``, - fields: [ - { - id: `name`, - name: `Name`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - ], - }, - ], defaultLocale: `en-US`, locales: [ { diff --git a/packages/gatsby-source-contentful/src/__fixtures__/rich-text-data.js b/packages/gatsby-source-contentful/src/__fixtures__/rich-text-data.js index 6a0200de2dc4e..0be1eaa13ee55 100644 --- a/packages/gatsby-source-contentful/src/__fixtures__/rich-text-data.js +++ b/packages/gatsby-source-contentful/src/__fixtures__/rich-text-data.js @@ -1,3 +1,61 @@ +exports.contentTypeItems = () => [ + { + sys: { + space: { + sys: { + type: `Link`, + linkType: `Space`, + id: `ahntqop9oi7x`, + }, + }, + id: `page`, + type: `ContentType`, + createdAt: `2020-10-16T11:43:48.221Z`, + updatedAt: `2020-10-16T11:44:25.392Z`, + environment: { + sys: { + id: `master`, + type: `Link`, + linkType: `Environment`, + }, + }, + revision: 2, + }, + displayField: `title`, + name: `Page`, + description: ``, + fields: [ + { + id: `title`, + name: `Title`, + type: `Symbol`, + localized: false, + required: false, + disabled: false, + omitted: false, + }, + { + id: `slug`, + name: `Slug`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `content`, + name: `Content`, + type: `RichText`, + localized: true, + required: false, + disabled: false, + omitted: false, + }, + ], + }, +] + exports.initialSync = () => { return { currentSyncData: { @@ -692,63 +750,7 @@ exports.initialSync = () => { deletedAssets: [], nextSyncToken: `FEnChMOBwr1Yw4TCqsK2LcKpCH3CjsORIyLDrGbDtgozw6xreMKCwpjCtlxATw3CmxolIsOxF10EMMOGCXM-IFrCrhc0LUPDvkjDkms7w5gLw4sqw4_CvxsiZMOFFsOawpM8R8OVPAhMJ8O1w6zCmg`, }, - contentTypeItems: [ - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `ahntqop9oi7x`, - }, - }, - id: `page`, - type: `ContentType`, - createdAt: `2020-10-16T11:43:48.221Z`, - updatedAt: `2020-10-16T11:44:25.392Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 2, - }, - displayField: `title`, - name: `Page`, - description: ``, - fields: [ - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `slug`, - name: `Slug`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `content`, - name: `Content`, - type: `RichText`, - localized: true, - required: false, - disabled: false, - omitted: false, - }, - ], - }, - ], + defaultLocale: `en-US`, locales: [ { @@ -792,6 +794,8 @@ exports.initialSync = () => { tagItems: [], } } + +// @todo this fixture is unused exports.deleteLinkedPage = () => { return { currentSyncData: { @@ -826,63 +830,6 @@ exports.deleteLinkedPage = () => { deletedAssets: [], nextSyncToken: `FEnChMOBwr1Yw4TCqsK2LcKpCH3CjsORIyLDrGbDtgozw6xreMKCwpjCtlxATw3CqcO3w6XCrMKuITDDiEoQSMKvIMOYwrzCn3sHPH3CvsK3w4A9w6LCjsOVwrjCjGwbw4rCl0fDl8OhU8Oqw67DhMOCwozDmxrChsOtRD4`, }, - contentTypeItems: [ - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `ahntqop9oi7x`, - }, - }, - id: `page`, - type: `ContentType`, - createdAt: `2020-10-16T11:43:48.221Z`, - updatedAt: `2020-10-16T11:44:25.392Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 2, - }, - displayField: `title`, - name: `Page`, - description: ``, - fields: [ - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `slug`, - name: `Slug`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `content`, - name: `Content`, - type: `RichText`, - localized: true, - required: false, - disabled: false, - omitted: false, - }, - ], - }, - ], defaultLocale: `en-US`, locales: [ { diff --git a/packages/gatsby-source-contentful/src/__fixtures__/starter-blog-data.js b/packages/gatsby-source-contentful/src/__fixtures__/starter-blog-data.js index eafa7a2723a4d..5808f74fe837a 100644 --- a/packages/gatsby-source-contentful/src/__fixtures__/starter-blog-data.js +++ b/packages/gatsby-source-contentful/src/__fixtures__/starter-blog-data.js @@ -1,3 +1,235 @@ +exports.contentTypeItems = () => [ + { + sys: { + space: { + sys: { + type: `Link`, + linkType: `Space`, + id: `uzfinxahlog0`, + contentful_id: `uzfinxahlog0`, + }, + }, + id: `person`, + type: `ContentType`, + createdAt: `2020-06-03T14:17:18.696Z`, + updatedAt: `2020-06-03T14:17:18.696Z`, + environment: { + sys: { + id: `master`, + type: `Link`, + linkType: `Environment`, + }, + }, + revision: 1, + contentful_id: `person`, + }, + displayField: `name`, + name: `Person`, + description: ``, + fields: [ + { + id: `name`, + name: `Name`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `title`, + name: `Title`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `company`, + name: `Company`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `shortBio`, + name: `Short Bio`, + type: `Text`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `email`, + name: `Email`, + type: `Symbol`, + localized: false, + required: false, + disabled: false, + omitted: false, + }, + { + id: `phone`, + name: `Phone`, + type: `Symbol`, + localized: false, + required: false, + disabled: false, + omitted: false, + }, + { + id: `facebook`, + name: `Facebook`, + type: `Symbol`, + localized: false, + required: false, + disabled: false, + omitted: false, + }, + { + id: `twitter`, + name: `Twitter`, + type: `Symbol`, + localized: false, + required: false, + disabled: false, + omitted: false, + }, + { + id: `github`, + name: `Github`, + type: `Symbol`, + localized: false, + required: false, + disabled: false, + omitted: false, + }, + { + id: `image`, + name: `Image`, + type: `Link`, + localized: false, + required: false, + disabled: false, + omitted: false, + linkType: `Asset`, + }, + ], + }, + { + sys: { + space: { + sys: { + type: `Link`, + linkType: `Space`, + id: `uzfinxahlog0`, + contentful_id: `uzfinxahlog0`, + }, + }, + id: `blogPost`, + type: `ContentType`, + createdAt: `2020-06-03T14:17:19.068Z`, + updatedAt: `2020-06-03T14:17:19.068Z`, + environment: { + sys: { + id: `master`, + type: `Link`, + linkType: `Environment`, + }, + }, + revision: 1, + contentful_id: `blogPost`, + }, + displayField: `title`, + name: `Blog Post`, + description: ``, + fields: [ + { + id: `title`, + name: `Title`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `slug`, + name: `Slug`, + type: `Symbol`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `heroImage`, + name: `Hero Image`, + type: `Link`, + localized: false, + required: true, + disabled: false, + omitted: false, + linkType: `Asset`, + }, + { + id: `description`, + name: `Description`, + type: `Text`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `body`, + name: `Body`, + type: `Text`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `author`, + name: `Author`, + type: `Link`, + localized: false, + required: false, + disabled: false, + omitted: false, + linkType: `Entry`, + }, + { + id: `publishDate`, + name: `Publish Date`, + type: `Date`, + localized: false, + required: true, + disabled: false, + omitted: false, + }, + { + id: `tags`, + name: `Tags`, + type: `Array`, + localized: false, + required: false, + disabled: false, + omitted: false, + items: { + type: `Symbol`, + validations: [{ in: [`general`, `javascript`, `static-sites`] }], + }, + }, + ], + }, +] + exports.initialSync = () => { return { currentSyncData: { @@ -420,237 +652,6 @@ exports.initialSync = () => { deletedAssets: [], nextSyncToken: `FEnChMOBwr1Yw 4TCqsK2LcKpCH3CjsORIyLDrGbDtgozw6xreMKCwpjCtlxATw3CswoUY34WECLCh152KsOLQcKwH8Kfw4kOQcOlw6TCr8OmEcKiwrhBZ8KhwrLCrcOsA8KYAMOFwo1kBMOZwrHDgCbDllcXVA`, }, - contentTypeItems: [ - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `person`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:18.696Z`, - updatedAt: `2020-06-03T14:17:18.696Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `person`, - }, - displayField: `name`, - name: `Person`, - description: ``, - fields: [ - { - id: `name`, - name: `Name`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `company`, - name: `Company`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `shortBio`, - name: `Short Bio`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `email`, - name: `Email`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `phone`, - name: `Phone`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `facebook`, - name: `Facebook`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `twitter`, - name: `Twitter`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `github`, - name: `Github`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `image`, - name: `Image`, - type: `Link`, - localized: false, - required: false, - disabled: false, - omitted: false, - linkType: `Asset`, - }, - ], - }, - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `blogPost`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:19.068Z`, - updatedAt: `2020-06-03T14:17:19.068Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `blogPost`, - }, - displayField: `title`, - name: `Blog Post`, - description: ``, - fields: [ - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `slug`, - name: `Slug`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `heroImage`, - name: `Hero Image`, - type: `Link`, - localized: false, - required: true, - disabled: false, - omitted: false, - linkType: `Asset`, - }, - { - id: `description`, - name: `Description`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `body`, - name: `Body`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `author`, - name: `Author`, - type: `Link`, - localized: false, - required: false, - disabled: false, - omitted: false, - linkType: `Entry`, - }, - { - id: `publishDate`, - name: `Publish Date`, - type: `Date`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `tags`, - name: `Tags`, - type: `Array`, - localized: false, - required: false, - disabled: false, - omitted: false, - items: { - type: `Symbol`, - validations: [{ in: [`general`, `javascript`, `static-sites`] }], - }, - }, - ], - }, - ], defaultLocale: `en-US`, locales: [ { @@ -810,237 +811,6 @@ exports.createBlogPost = () => { deletedAssets: [], nextSyncToken: `FEnChMOBwr1Yw4TCqsK2LcKpCH3CjsORIyLDrGbDtgozw6xreMKCwpjCtlxATw0YQMOfwrtZBsOQw41ww7xhJj8Ew4TChcOow6ZPPVVZaMOfOlFEwp7CpcOxwpd_YcKBw5jCkznDgMO6w4lsw73CrcOmwpILwqTClg`, }, - contentTypeItems: [ - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `person`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:18.696Z`, - updatedAt: `2020-06-03T14:17:18.696Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `person`, - }, - displayField: `name`, - name: `Person`, - description: ``, - fields: [ - { - id: `name`, - name: `Name`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `company`, - name: `Company`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `shortBio`, - name: `Short Bio`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `email`, - name: `Email`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `phone`, - name: `Phone`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `facebook`, - name: `Facebook`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `twitter`, - name: `Twitter`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `github`, - name: `Github`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `image`, - name: `Image`, - type: `Link`, - localized: false, - required: false, - disabled: false, - omitted: false, - linkType: `Asset`, - }, - ], - }, - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `blogPost`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:19.068Z`, - updatedAt: `2020-06-03T14:17:19.068Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `blogPost`, - }, - displayField: `title`, - name: `Blog Post`, - description: ``, - fields: [ - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `slug`, - name: `Slug`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `heroImage`, - name: `Hero Image`, - type: `Link`, - localized: false, - required: true, - disabled: false, - omitted: false, - linkType: `Asset`, - }, - { - id: `description`, - name: `Description`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `body`, - name: `Body`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `author`, - name: `Author`, - type: `Link`, - localized: false, - required: false, - disabled: false, - omitted: false, - linkType: `Entry`, - }, - { - id: `publishDate`, - name: `Publish Date`, - type: `Date`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `tags`, - name: `Tags`, - type: `Array`, - localized: false, - required: false, - disabled: false, - omitted: false, - items: { - type: `Symbol`, - validations: [{ in: [`general`, `javascript`, `static-sites`] }], - }, - }, - ], - }, - ], defaultLocale: `en-US`, locales: [ { @@ -1162,237 +932,6 @@ exports.updateBlogPost = () => { deletedAssets: [], nextSyncToken: `FEnChMOBwr1Yw4TCqsK2LcKpCH3CjsORIyLDrGbDtgozw6xreMKCwpjCtlxATw0OwpjDkMOywrZewqlOAMK_wp_DmcOzLRXDmlJ3wp5VextfJMKtw43CngvCrw7Cn07CgnNJw4XDscOsw5zCuXbDrMKnw7rCsTPCpxE`, }, - contentTypeItems: [ - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `person`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:18.696Z`, - updatedAt: `2020-06-03T14:17:18.696Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `person`, - }, - displayField: `name`, - name: `Person`, - description: ``, - fields: [ - { - id: `name`, - name: `Name`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `company`, - name: `Company`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `shortBio`, - name: `Short Bio`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `email`, - name: `Email`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `phone`, - name: `Phone`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `facebook`, - name: `Facebook`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `twitter`, - name: `Twitter`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `github`, - name: `Github`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `image`, - name: `Image`, - type: `Link`, - localized: false, - required: false, - disabled: false, - omitted: false, - linkType: `Asset`, - }, - ], - }, - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `blogPost`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:19.068Z`, - updatedAt: `2020-06-03T14:17:19.068Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `blogPost`, - }, - displayField: `title`, - name: `Blog Post`, - description: ``, - fields: [ - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `slug`, - name: `Slug`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `heroImage`, - name: `Hero Image`, - type: `Link`, - localized: false, - required: true, - disabled: false, - omitted: false, - linkType: `Asset`, - }, - { - id: `description`, - name: `Description`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `body`, - name: `Body`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `author`, - name: `Author`, - type: `Link`, - localized: false, - required: false, - disabled: false, - omitted: false, - linkType: `Entry`, - }, - { - id: `publishDate`, - name: `Publish Date`, - type: `Date`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `tags`, - name: `Tags`, - type: `Array`, - localized: false, - required: false, - disabled: false, - omitted: false, - items: { - type: `Symbol`, - validations: [{ in: [`general`, `javascript`, `static-sites`] }], - }, - }, - ], - }, - ], defaultLocale: `en-US`, locales: [ { @@ -1476,237 +1015,6 @@ exports.removeBlogPost = () => { deletedAssets: [], nextSyncToken: `FEnChMOBwr1Yw4TCqsK2LcKpCH3CjsORIyLDrGbDtgozw6xreMKCwpjCtlxATw13woJkDMK6fDpuN014SMKXw4MowpNDLcKVGQlXJiNSw53DlcKow4Fjw5HDqjthfQQrwo5MBlfDr3UfZjjCiMKi`, }, - contentTypeItems: [ - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `person`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:18.696Z`, - updatedAt: `2020-06-03T14:17:18.696Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `person`, - }, - displayField: `name`, - name: `Person`, - description: ``, - fields: [ - { - id: `name`, - name: `Name`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `company`, - name: `Company`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `shortBio`, - name: `Short Bio`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `email`, - name: `Email`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `phone`, - name: `Phone`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `facebook`, - name: `Facebook`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `twitter`, - name: `Twitter`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `github`, - name: `Github`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `image`, - name: `Image`, - type: `Link`, - localized: false, - required: false, - disabled: false, - omitted: false, - linkType: `Asset`, - }, - ], - }, - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `blogPost`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:19.068Z`, - updatedAt: `2020-06-03T14:17:19.068Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `blogPost`, - }, - displayField: `title`, - name: `Blog Post`, - description: ``, - fields: [ - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `slug`, - name: `Slug`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `heroImage`, - name: `Hero Image`, - type: `Link`, - localized: false, - required: true, - disabled: false, - omitted: false, - linkType: `Asset`, - }, - { - id: `description`, - name: `Description`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `body`, - name: `Body`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `author`, - name: `Author`, - type: `Link`, - localized: false, - required: false, - disabled: false, - omitted: false, - linkType: `Entry`, - }, - { - id: `publishDate`, - name: `Publish Date`, - type: `Date`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `tags`, - name: `Tags`, - type: `Array`, - localized: false, - required: false, - disabled: false, - omitted: false, - items: { - type: `Symbol`, - validations: [{ in: [`general`, `javascript`, `static-sites`] }], - }, - }, - ], - }, - ], defaultLocale: `en-US`, locales: [ { @@ -1790,237 +1098,6 @@ exports.removeAsset = () => { ], nextSyncToken: `FEnChMOBwr1Yw4TCqsK2LcKpCH3CjsORIyLDrGbDtgozw6xreMKCwpjCtlxATw0LNMOLwow1KMKwAW_Ci8OIwoPDgcK-Hn5Rw5XDvwXCsMK7wpPDk2jDtywiw6lyU8KEwprCojzDscOMwollMCbCicK_XTUEw7wZ`, }, - contentTypeItems: [ - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `person`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:18.696Z`, - updatedAt: `2020-06-03T14:17:18.696Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `person`, - }, - displayField: `name`, - name: `Person`, - description: ``, - fields: [ - { - id: `name`, - name: `Name`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `company`, - name: `Company`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `shortBio`, - name: `Short Bio`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `email`, - name: `Email`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `phone`, - name: `Phone`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `facebook`, - name: `Facebook`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `twitter`, - name: `Twitter`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `github`, - name: `Github`, - type: `Symbol`, - localized: false, - required: false, - disabled: false, - omitted: false, - }, - { - id: `image`, - name: `Image`, - type: `Link`, - localized: false, - required: false, - disabled: false, - omitted: false, - linkType: `Asset`, - }, - ], - }, - { - sys: { - space: { - sys: { - type: `Link`, - linkType: `Space`, - id: `uzfinxahlog0`, - contentful_id: `uzfinxahlog0`, - }, - }, - id: `blogPost`, - type: `ContentType`, - createdAt: `2020-06-03T14:17:19.068Z`, - updatedAt: `2020-06-03T14:17:19.068Z`, - environment: { - sys: { - id: `master`, - type: `Link`, - linkType: `Environment`, - }, - }, - revision: 1, - contentful_id: `blogPost`, - }, - displayField: `title`, - name: `Blog Post`, - description: ``, - fields: [ - { - id: `title`, - name: `Title`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `slug`, - name: `Slug`, - type: `Symbol`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `heroImage`, - name: `Hero Image`, - type: `Link`, - localized: false, - required: true, - disabled: false, - omitted: false, - linkType: `Asset`, - }, - { - id: `description`, - name: `Description`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `body`, - name: `Body`, - type: `Text`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `author`, - name: `Author`, - type: `Link`, - localized: false, - required: false, - disabled: false, - omitted: false, - linkType: `Entry`, - }, - { - id: `publishDate`, - name: `Publish Date`, - type: `Date`, - localized: false, - required: true, - disabled: false, - omitted: false, - }, - { - id: `tags`, - name: `Tags`, - type: `Array`, - localized: false, - required: false, - disabled: false, - omitted: false, - items: { - type: `Symbol`, - validations: [{ in: [`general`, `javascript`, `static-sites`] }], - }, - }, - ], - }, - ], defaultLocale: `en-US`, locales: [ { diff --git a/packages/gatsby-source-contentful/src/__tests__/fetch-backoff.js b/packages/gatsby-source-contentful/src/__tests__/fetch-backoff.js index cc3658bcf5407..a05ddb7b41e52 100644 --- a/packages/gatsby-source-contentful/src/__tests__/fetch-backoff.js +++ b/packages/gatsby-source-contentful/src/__tests__/fetch-backoff.js @@ -3,7 +3,7 @@ */ import nock from "nock" -import fetchData from "../fetch" +import { fetchContent } from "../fetch" import { createPluginConfig } from "../plugin-options" const host = `localhost` @@ -82,13 +82,7 @@ describe(`fetch-backoff`, () => { `/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=444` ) .reply(200, { items: [] }) - // Content types - .get( - `/spaces/${options.spaceId}/environments/master/content_types?skip=0&limit=1000&order=sys.createdAt` - ) - .reply(200, { items: [] }) - - await fetchData({ pluginConfig, reporter }) + await fetchContent({ pluginConfig, reporter }) expect(reporter.panic).not.toBeCalled() expect(reporter.warn.mock.calls).toMatchInlineSnapshot(` @@ -125,13 +119,8 @@ describe(`fetch-backoff`, () => { `/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=1000` ) .reply(200, { items: [] }) - // Content types - .get( - `/spaces/${options.spaceId}/environments/master/content_types?skip=0&limit=1000&order=sys.createdAt` - ) - .reply(200, { items: [] }) - await fetchData({ pluginConfig, reporter }) + await fetchContent({ pluginConfig, reporter }) expect(reporter.panic).not.toBeCalled() expect(reporter.warn).not.toBeCalled() diff --git a/packages/gatsby-source-contentful/src/__tests__/fetch-network-errors.js b/packages/gatsby-source-contentful/src/__tests__/fetch-network-errors.js index 8f3d0a2db687a..6b22c5f47f3de 100644 --- a/packages/gatsby-source-contentful/src/__tests__/fetch-network-errors.js +++ b/packages/gatsby-source-contentful/src/__tests__/fetch-network-errors.js @@ -3,7 +3,7 @@ */ import nock from "nock" -import fetchData from "../fetch" +import { fetchContent } from "../fetch" import { createPluginConfig } from "../plugin-options" nock.disableNetConnect() @@ -65,13 +65,8 @@ describe(`fetch-retry`, () => { `/spaces/${options.spaceId}/environments/master/sync?initial=true&limit=1000` ) .reply(200, { items: [] }) - // Content types - .get( - `/spaces/${options.spaceId}/environments/master/content_types?skip=0&limit=1000&order=sys.createdAt` - ) - .reply(200, { items: [] }) - await fetchData({ pluginConfig, reporter }) + await fetchContent({ pluginConfig, reporter }) expect(reporter.panic).not.toBeCalled() expect(scope.isDone()).toBeTruthy() @@ -106,7 +101,7 @@ describe(`fetch-retry`, () => { ) try { - await fetchData({ pluginConfig, reporter }) + await fetchContent({ pluginConfig, reporter }) jest.fail() } catch (e) { const msg = expect(e.context.sourceMessage) @@ -132,7 +127,7 @@ describe(`fetch-network-errors`, () => { .get(`/spaces/${options.spaceId}/`) .replyWithError({ code: `ECONNRESET` }) try { - await fetchData({ + await fetchContent({ pluginConfig: createPluginConfig({ ...options, contentfulClientConfig: { retryOnError: false }, @@ -159,7 +154,7 @@ describe(`fetch-network-errors`, () => { .reply(502, `Bad Gateway`) try { - await fetchData({ + await fetchContent({ pluginConfig: createPluginConfig({ ...options, contentfulClientConfig: { retryOnError: false }, @@ -193,7 +188,7 @@ describe(`fetch-network-errors`, () => { }) try { - await fetchData({ + await fetchContent({ pluginConfig: createPluginConfig({ ...options, contentfulClientConfig: { retryOnError: false }, diff --git a/packages/gatsby-source-contentful/src/__tests__/fetch.js b/packages/gatsby-source-contentful/src/__tests__/fetch.js index d9bbbd4479047..6bc03c7300de6 100644 --- a/packages/gatsby-source-contentful/src/__tests__/fetch.js +++ b/packages/gatsby-source-contentful/src/__tests__/fetch.js @@ -59,7 +59,7 @@ jest.mock(`../plugin-options`, () => { global.console = { log: jest.fn(), time: jest.fn(), timeEnd: jest.fn() } const contentful = require(`contentful`) -const fetchData = require(`../fetch`) +const { fetchContent, fetchContentTypes } = require(`../fetch`) const { formatPluginOptionsForCLI, createPluginConfig, @@ -119,7 +119,7 @@ afterAll(() => { }) it(`calls contentful.createClient with expected params`, async () => { - await fetchData({ pluginConfig, reporter }) + await fetchContent({ pluginConfig, reporter }) expect(reporter.panic).not.toBeCalled() expect(contentful.createClient).toBeCalledWith( expect.objectContaining({ @@ -133,7 +133,7 @@ it(`calls contentful.createClient with expected params`, async () => { }) it(`calls contentful.createClient with expected params and default fallbacks`, async () => { - await fetchData({ + await fetchContent({ pluginConfig: createPluginConfig({ accessToken: `6f35edf0db39085e9b9c19bd92943e4519c77e72c852d961968665f1324bfc94`, spaceId: `rocybtov1ozk`, @@ -153,7 +153,7 @@ it(`calls contentful.createClient with expected params and default fallbacks`, a }) it(`calls contentful.getContentTypes with default page limit`, async () => { - await fetchData({ + await fetchContentTypes({ pluginConfig: createPluginConfig({ accessToken: `6f35edf0db39085e9b9c19bd92943e4519c77e72c852d961968665f1324bfc94`, spaceId: `rocybtov1ozk`, @@ -170,7 +170,7 @@ it(`calls contentful.getContentTypes with default page limit`, async () => { }) it(`calls contentful.getContentTypes with custom plugin option page limit`, async () => { - await fetchData({ + await fetchContentTypes({ pluginConfig: createPluginConfig({ accessToken: `6f35edf0db39085e9b9c19bd92943e4519c77e72c852d961968665f1324bfc94`, spaceId: `rocybtov1ozk`, @@ -189,7 +189,7 @@ it(`calls contentful.getContentTypes with custom plugin option page limit`, asyn describe(`Tags feature`, () => { it(`tags are disabled by default`, async () => { - await fetchData({ + await fetchContent({ pluginConfig: createPluginConfig({ accessToken: `6f35edf0db39085e9b9c19bd92943e4519c77e72c852d961968665f1324bfc94`, spaceId: `rocybtov1ozk`, @@ -202,7 +202,7 @@ describe(`Tags feature`, () => { expect(mockClient.getTags).not.toBeCalled() }) it(`calls contentful.getTags when enabled`, async () => { - await fetchData({ + await fetchContent({ pluginConfig: createPluginConfig({ accessToken: `6f35edf0db39085e9b9c19bd92943e4519c77e72c852d961968665f1324bfc94`, spaceId: `rocybtov1ozk`, @@ -227,7 +227,7 @@ describe(`Displays troubleshooting tips and detailed plugin options on contentfu throw new Error(`error`) }) - await fetchData({ pluginConfig, reporter }) + await fetchContent({ pluginConfig, reporter }) expect(reporter.panic).toBeCalledWith( expect.objectContaining({ @@ -264,7 +264,7 @@ describe(`Displays troubleshooting tips and detailed plugin options on contentfu throw err }) - await fetchData({ pluginConfig, reporter }) + await fetchContent({ pluginConfig, reporter }) expect(reporter.panic).toBeCalledWith( expect.objectContaining({ @@ -301,7 +301,7 @@ describe(`Displays troubleshooting tips and detailed plugin options on contentfu const masterOptions = { ...options, environment: `master` } const masterConfig = createPluginConfig(masterOptions) - await fetchData({ + await fetchContent({ pluginConfig: masterConfig, reporter, }) @@ -344,7 +344,7 @@ describe(`Displays troubleshooting tips and detailed plugin options on contentfu throw err }) - await fetchData({ pluginConfig, reporter }) + await fetchContent({ pluginConfig, reporter }) expect(reporter.panic).toBeCalledWith( expect.objectContaining({ @@ -384,7 +384,7 @@ describe(`Displays troubleshooting tips and detailed plugin options on contentfu throw err }) - await fetchData({ pluginConfig, reporter }) + await fetchContent({ pluginConfig, reporter }) expect(reporter.panic).toBeCalledWith( expect.objectContaining({ diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index b15ff8bd9abf0..8a694be620c08 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -8,7 +8,7 @@ jest.mock(`gatsby-core-utils`, () => { }) const gatsbyNode = require(`../gatsby-node`) -const fetch = require(`../fetch`) +const { fetchContent, fetchContentTypes } = require(`../fetch`) const normalize = require(`../normalize`) const _ = require(`lodash`) @@ -18,6 +18,10 @@ const restrictedContentTypeFixture = require(`../__fixtures__/restricted-content const defaultPluginOptions = { spaceId: `testSpaceId` } +fetchContentTypes.mockImplementation(() => + startersBlogFixture.contentTypeItems() +) + const createMockCache = () => { const actualCacheMap = new Map() return { @@ -51,6 +55,7 @@ describe(`gatsby-node`, () => { let currentNodeMap const getNodes = () => Array.from(currentNodeMap.values()) const getNode = id => currentNodeMap.get(id) + const getNodesByType = jest.fn() const getFieldValue = (value, locale, defaultLocale) => value[locale] ?? value[defaultLocale] @@ -58,27 +63,25 @@ describe(`gatsby-node`, () => { const simulateGatsbyBuild = async function ( pluginOptions = defaultPluginOptions ) { - await gatsbyNode.onPreBootstrap( - { reporter, cache, parentSpan, store }, - pluginOptions - ) + await gatsbyNode.onPreBootstrap({ store }) await gatsbyNode.createSchemaCustomization( - { schema, actions, cache }, + { schema, actions, cache, reporter }, pluginOptions ) await gatsbyNode.sourceNodes( { actions, - store, - getNodes, getNode, - reporter, + getNodes, + getNodesByType, createNodeId, + store, cache, getCache, - schema, + reporter, + parentSpan, }, pluginOptions ) @@ -288,7 +291,8 @@ describe(`gatsby-node`, () => { } beforeEach(() => { - fetch.mockClear() + fetchContent.mockClear() + fetchContentTypes.mockClear() currentNodeMap = new Map() actions.createNode = jest.fn(async node => { node.internal.owner = `gatsby-source-contentful` @@ -302,20 +306,20 @@ describe(`gatsby-node`, () => { actions.setPluginStatus = jest.fn() store.getState.mockClear() cache.actualMap.clear() + cache.get.mockClear() + cache.set.mockClear() }) it(`should create nodes from initial payload`, async () => { - cache.get.mockClear() - cache.set.mockClear() - fetch.mockImplementationOnce(startersBlogFixture.initialSync) + fetchContent.mockImplementationOnce(startersBlogFixture.initialSync) const locales = [`en-US`, `nl`] await simulateGatsbyBuild() - testIfContentTypesExists(startersBlogFixture.initialSync().contentTypeItems) + testIfContentTypesExists(startersBlogFixture.contentTypeItems()) testIfEntriesExists( startersBlogFixture.initialSync().currentSyncData.entries, - startersBlogFixture.initialSync().contentTypeItems, + startersBlogFixture.contentTypeItems(), locales ) testIfAssetsExistsAndMatch( @@ -337,17 +341,11 @@ describe(`gatsby-node`, () => { "contentful-sync-token-testSpaceId-master", ], Array [ - "contentful-sync-data-testSpaceId-master", - ], - Array [ - "contentful-sync-result-testSpaceId-master", + "contentful-content-types-testSpaceId-master", ], Array [ "contentful-sync-data-testSpaceId-master", ], - Array [ - "contentful-sync-result-testSpaceId-master", - ], ] `) @@ -356,18 +354,25 @@ describe(`gatsby-node`, () => { `contentful-sync-token-testSpaceId-master`, startersBlogFixture.initialSync().currentSyncData.nextSyncToken ) - expect(cache.set).toHaveBeenCalledWith( - `contentful-sync-data-testSpaceId-master`, - { - entries: startersBlogFixture.initialSync().currentSyncData.entries, - assets: startersBlogFixture.initialSync().currentSyncData.assets, - } + + // Check for valid cache data + const cacheCall = cache.set.mock.calls.filter( + args => args[0] === `contentful-sync-data-testSpaceId-master` ) + + expect(cacheCall).toBeTruthy() + expect(cacheCall[0][1].entries).toHaveLength( + startersBlogFixture.initialSync().currentSyncData.entries.length + ) + expect(cacheCall[0][1].assets).toHaveLength( + startersBlogFixture.initialSync().currentSyncData.assets.length + ) + expect(cache.set.mock.calls.map(v => v[0])).toMatchInlineSnapshot(` Array [ + "contentful-content-types-testSpaceId-master", "contentful-sync-data-testSpaceId-master", "contentful-sync-token-testSpaceId-master", - "contentful-sync-result-testSpaceId-master", ] `) }) @@ -375,7 +380,7 @@ describe(`gatsby-node`, () => { it(`should add a new blogpost and update linkedNodes`, async () => { const locales = [`en-US`, `nl`] - fetch + fetchContent .mockImplementationOnce(startersBlogFixture.initialSync) .mockImplementationOnce(startersBlogFixture.createBlogPost) @@ -402,12 +407,10 @@ describe(`gatsby-node`, () => { // add new blog post await simulateGatsbyBuild() - testIfContentTypesExists( - startersBlogFixture.createBlogPost().contentTypeItems - ) + testIfContentTypesExists(startersBlogFixture.contentTypeItems()) testIfEntriesExists( startersBlogFixture.createBlogPost().currentSyncData.entries, - startersBlogFixture.createBlogPost().contentTypeItems, + startersBlogFixture.contentTypeItems(), locales ) testIfAssetsExistsAndMatch( @@ -424,7 +427,7 @@ describe(`gatsby-node`, () => { it(`should update a blogpost`, async () => { const locales = [`en-US`, `nl`] - fetch + fetchContent .mockImplementationOnce(startersBlogFixture.initialSync) .mockImplementationOnce(startersBlogFixture.createBlogPost) .mockImplementationOnce(startersBlogFixture.updateBlogPost) @@ -454,12 +457,10 @@ describe(`gatsby-node`, () => { // updated blog post await simulateGatsbyBuild() - testIfContentTypesExists( - startersBlogFixture.updateBlogPost().contentTypeItems - ) + testIfContentTypesExists(startersBlogFixture.contentTypeItems()) testIfEntriesExists( startersBlogFixture.updateBlogPost().currentSyncData.entries, - startersBlogFixture.updateBlogPost().contentTypeItems, + startersBlogFixture.contentTypeItems(), locales ) testIfAssetsExistsAndMatch( @@ -477,7 +478,7 @@ describe(`gatsby-node`, () => { it(`should remove a blogpost and update linkedNodes`, async () => { const locales = [`en-US`, `nl`] - fetch + fetchContent .mockImplementationOnce(startersBlogFixture.initialSync) .mockImplementationOnce(startersBlogFixture.createBlogPost) .mockImplementationOnce(startersBlogFixture.removeBlogPost) @@ -514,9 +515,7 @@ describe(`gatsby-node`, () => { // remove blog post await simulateGatsbyBuild() - testIfContentTypesExists( - startersBlogFixture.removeBlogPost().contentTypeItems - ) + testIfContentTypesExists(startersBlogFixture.contentTypeItems()) testIfEntriesDeleted( startersBlogFixture.removeBlogPost().currentSyncData.assets, locales @@ -531,7 +530,7 @@ describe(`gatsby-node`, () => { it(`should remove an asset`, async () => { const locales = [`en-US`, `nl`] - fetch + fetchContent .mockImplementationOnce(startersBlogFixture.initialSync) .mockImplementationOnce(startersBlogFixture.createBlogPost) .mockImplementationOnce(startersBlogFixture.removeAsset) @@ -568,7 +567,7 @@ describe(`gatsby-node`, () => { // remove asset await simulateGatsbyBuild() - testIfContentTypesExists(startersBlogFixture.removeAsset().contentTypeItems) + testIfContentTypesExists(startersBlogFixture.contentTypeItems()) testIfEntriesExists( startersBlogFixture.removeAsset().currentSyncData.entries, startersBlogFixture.removeAsset().contentTypeItems, @@ -581,7 +580,8 @@ describe(`gatsby-node`, () => { }) it(`stores rich text as raw with references attached`, async () => { - fetch.mockImplementationOnce(richTextFixture.initialSync) + fetchContent.mockImplementationOnce(richTextFixture.initialSync) + fetchContentTypes.mockImplementationOnce(richTextFixture.contentTypeItems) // initial sync await simulateGatsbyBuild() @@ -591,6 +591,7 @@ describe(`gatsby-node`, () => { const homeNodes = initNodes.filter( ({ contentful_id: id }) => id === `6KpLS2NZyB3KAvDzWf4Ukh` ) + expect(homeNodes).toHaveLength(2) homeNodes.forEach(homeNode => { expect(homeNode.content.references___NODE).toStrictEqual([ ...new Set(homeNode.content.references___NODE), @@ -600,9 +601,7 @@ describe(`gatsby-node`, () => { }) it(`panics when localeFilter reduces locale list to 0`, async () => { - cache.get.mockClear() - cache.set.mockClear() - fetch.mockImplementationOnce(startersBlogFixture.initialSync) + fetchContent.mockImplementationOnce(startersBlogFixture.initialSync) const locales = [`en-US`, `nl`] await simulateGatsbyBuild({ @@ -622,9 +621,12 @@ describe(`gatsby-node`, () => { }) it(`panics when response contains restricted content types`, async () => { - cache.get.mockClear() - cache.set.mockClear() - fetch.mockImplementationOnce(restrictedContentTypeFixture.initialSync) + fetchContent.mockImplementationOnce( + restrictedContentTypeFixture.initialSync + ) + fetchContentTypes.mockImplementationOnce( + restrictedContentTypeFixture.contentTypeItems + ) await simulateGatsbyBuild() diff --git a/packages/gatsby-source-contentful/src/create-schema-customization.js b/packages/gatsby-source-contentful/src/create-schema-customization.js index 585561d086d88..1d12efa70505e 100644 --- a/packages/gatsby-source-contentful/src/create-schema-customization.js +++ b/packages/gatsby-source-contentful/src/create-schema-customization.js @@ -1,21 +1,82 @@ +// @ts-check const _ = require(`lodash`) +const v8 = require(`v8`) +const fs = require(`fs-extra`) const { createPluginConfig } = require(`./plugin-options`) +const { fetchContentTypes } = require(`./fetch`) +const { CODES } = require(`./report`) +import { getFileSystemCachePath } from "./fs-cache" export async function createSchemaCustomization( - { schema, actions, cache }, + { schema, actions, reporter, cache }, pluginOptions ) { + const { fsForceCache, fsCacheFileExists, fsCacheFilePath } = + await getFileSystemCachePath({ suffix: `content-type` }) const { createTypes } = actions const pluginConfig = createPluginConfig(pluginOptions) + + // Get content type items from Contentful + let contentTypeItems + if (!fsCacheFileExists) { + // Fetch content types as long fs cache is disabled or the fs cache file does not exist + contentTypeItems = await fetchContentTypes({ pluginConfig, reporter }) + + // Cache file to FS if required + if (fsForceCache) { + reporter.info( + `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Writing v8 serialized glob of remote content type data to: ` + + fsCacheFilePath + ) + await fs.writeFile(fsCacheFilePath, v8.serialize(contentTypeItems)) + } + } else { + // Load the content type item data from FS + reporter.info( + `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Reading v8 serialized glob of remote content type data from: ` + + fsCacheFilePath + ) + const contentTypeItemsCacheBuffer = await fs.readFile(fsCacheFilePath) + contentTypeItems = v8.deserialize(contentTypeItemsCacheBuffer) + } + + // Check for restricted content type names and set id based on useNameForId + const useNameForId = pluginConfig.get(`useNameForId`) + const restrictedContentTypes = [`entity`, `reference`, `asset`] + + if (pluginConfig.get(`enableTags`)) { + restrictedContentTypes.push(`tags`) + } + + contentTypeItems.forEach(contentTypeItem => { + // Establish identifier for content type + // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, + // but sometimes a base62 uuid generated by Contentful, hence the option) + let contentTypeItemId + if (useNameForId) { + contentTypeItemId = contentTypeItem.name.toLowerCase() + } else { + contentTypeItemId = contentTypeItem.sys.id.toLowerCase() + } + + if (restrictedContentTypes.includes(contentTypeItemId)) { + reporter.panic({ + id: CODES.FetchContentTypes, + context: { + sourceMessage: `Restricted ContentType name found. The name "${contentTypeItemId}" is not allowed.`, + }, + }) + } + }) + + // Store processed content types in cache for sourceNodes const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( `environment` )}` - - const { contentTypeItems } = await cache.get( - `contentful-sync-result-${sourceId}` - ) + const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}` + await cache.set(CACHE_CONTENT_TYPES, contentTypeItems) createTypes(` interface ContentfulEntry implements Node { diff --git a/packages/gatsby-source-contentful/src/fetch.js b/packages/gatsby-source-contentful/src/fetch.js index 136031150d68e..3c8dbad11530c 100644 --- a/packages/gatsby-source-contentful/src/fetch.js +++ b/packages/gatsby-source-contentful/src/fetch.js @@ -72,15 +72,13 @@ const createContentfulErrorMessage = e => { return errorMessage } -module.exports = async function contentfulFetch({ - syncToken, +function createContentfulClientOptions({ pluginConfig, reporter, + syncProgress = { total: 0, tick: a => a }, }) { - // Fetch articles. - let syncProgress let syncItemCount = 0 - const pageLimit = pluginConfig.get(`pageLimit`) + const contentfulClientOptions = { space: pluginConfig.get(`spaceId`), accessToken: pluginConfig.get(`accessToken`), @@ -133,6 +131,17 @@ module.exports = async function contentfulFetch({ ...(pluginConfig.get(`contentfulClientConfig`) || {}), } + return contentfulClientOptions +} + +async function fetchContent({ syncToken, pluginConfig, reporter }) { + // Fetch articles. + let syncProgress + const pageLimit = pluginConfig.get(`pageLimit`) + const contentfulClientOptions = createContentfulClientOptions({ + pluginConfig, + reporter, + }) const client = contentful.createClient(contentfulClientOptions) // The sync API puts the locale in all fields in this format { fieldName: @@ -278,25 +287,6 @@ ${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`, syncProgress.done() } - // We need to fetch content types with the non-sync API as the sync API - // doesn't support this. - let contentTypes - try { - contentTypes = await pagedGet(client, `getContentTypes`, pageLimit) - } catch (e) { - reporter.panic({ - id: CODES.FetchContentTypes, - context: { - sourceMessage: `Error fetching content types: ${createContentfulErrorMessage( - e - )}`, - }, - }) - } - reporter.verbose(`Content types fetched ${contentTypes.items.length}`) - - const contentTypeItems = contentTypes.items - // We need to fetch tags with the non-sync API as the sync API doesn't support this. let tagItems = [] if (pluginConfig.get(`enableTags`)) { @@ -318,7 +308,6 @@ ${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`, const result = { currentSyncData, - contentTypeItems, tagItems, defaultLocale, locales, @@ -328,6 +317,111 @@ ${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`, return result } +module.exports.fetchContent = fetchContent + +async function fetchContentTypes({ pluginConfig, reporter }) { + const contentfulClientOptions = createContentfulClientOptions({ + pluginConfig, + reporter, + }) + const client = contentful.createClient(contentfulClientOptions) + const pageLimit = pluginConfig.get(`pageLimit`) + const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( + `environment` + )}` + let contentTypes = null + + try { + reporter.verbose(`Fetching content types (${sourceId})`) + + // Fetch content types from CDA API + try { + contentTypes = await pagedGet(client, `getContentTypes`, pageLimit) + } catch (e) { + reporter.panic({ + id: CODES.FetchContentTypes, + context: { + sourceMessage: `Error fetching content types: ${createContentfulErrorMessage( + e + )}`, + }, + }) + } + reporter.verbose( + `Content types fetched ${contentTypes.items.length} (${sourceId})` + ) + + contentTypes = contentTypes.items + } catch (e) { + let details + let errors + if (e.code === `ENOTFOUND`) { + details = `You seem to be offline` + } else if (e.code === `SELF_SIGNED_CERT_IN_CHAIN`) { + reporter.panic({ + id: CODES.SelfSignedCertificate, + context: { + sourceMessage: `We couldn't make a secure connection to your contentful space. Please check if you have any self-signed SSL certificates installed.`, + }, + }) + } else if (e.responseData) { + if ( + e.responseData.status === 404 && + contentfulClientOptions.environment && + contentfulClientOptions.environment !== `master` + ) { + // environments need to have access to master + details = `Unable to access your space. Check if ${chalk.yellow( + `environment` + )} is correct and your ${chalk.yellow( + `accessToken` + )} has access to the ${chalk.yellow( + contentfulClientOptions.environment + )} and the ${chalk.yellow(`master`)} environments.` + errors = { + accessToken: `Check if setting is correct`, + environment: `Check if setting is correct`, + } + } else if (e.responseData.status === 404) { + // host and space used to generate url + details = `Endpoint not found. Check if ${chalk.yellow( + `host` + )} and ${chalk.yellow(`spaceId`)} settings are correct` + errors = { + host: `Check if setting is correct`, + spaceId: `Check if setting is correct`, + } + } else if (e.responseData.status === 401) { + // authorization error + details = `Authorization error. Check if ${chalk.yellow( + `accessToken` + )} and ${chalk.yellow(`environment`)} are correct` + errors = { + accessToken: `Check if setting is correct`, + environment: `Check if setting is correct`, + } + } + } + + reporter.panic({ + context: { + sourceMessage: `Accessing your Contentful space failed: ${createContentfulErrorMessage( + e + )} + +Try setting GATSBY_CONTENTFUL_OFFLINE=true to see if we can serve from cache. +${details ? `\n${details}\n` : ``} +Used options: +${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`, + }, + }) + } + + return contentTypes +} + +module.exports.fetchContentTypes = fetchContentTypes + /** * Gets all the existing entities based on pagination parameters. * The first call will have no aggregated response. Subsequent calls will diff --git a/packages/gatsby-source-contentful/src/fs-cache.js b/packages/gatsby-source-contentful/src/fs-cache.js new file mode 100644 index 0000000000000..b3a58929fc0f4 --- /dev/null +++ b/packages/gatsby-source-contentful/src/fs-cache.js @@ -0,0 +1,26 @@ +import fs from "fs-extra" + +export async function getFileSystemCachePath({ suffix = null } = {}) { + if (!process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { + return { + fsForceCache: false, + fsCacheFileExists: false, + fsCacheFilePath: null, + } + } + + const fsCacheFilePath = [ + process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE, + suffix, + ] + .filter(Boolean) + .join(`-`) + + const fsCacheFileExists = await fs.exists(fsCacheFilePath) + + return { + fsForceCache: true, + fsCacheFileExists, + fsCacheFilePath, + } +} diff --git a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js index 344c937595126..63df556b0718a 100644 --- a/packages/gatsby-source-contentful/src/on-pre-bootstrap.js +++ b/packages/gatsby-source-contentful/src/on-pre-bootstrap.js @@ -1,17 +1,8 @@ -// @todo import syntax! -const isOnline = require(`is-online`) -const _ = require(`lodash`) +// @ts-check const fs = require(`fs-extra`) const path = require(`path`) -const v8 = require(`v8`) -const { CODES } = require(`./report`) -const fetchData = require(`./fetch`) -const { createPluginConfig } = require(`./plugin-options`) -export async function onPreBootstrap( - { reporter, cache, parentSpan, store }, - pluginOptions -) { +export async function onPreBootstrap({ store }) { // Ensure cache dir exists for downloadLocal const program = store.getState().program @@ -20,242 +11,4 @@ export async function onPreBootstrap( ) await fs.ensureDir(CACHE_DIR) - - // Fetch data for sourceNodes & createSchemaCustomization - let currentSyncData - let contentTypeItems - let tagItems - let defaultLocale - let locales - let space - if (process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE: Storing/loading remote data through \`` + - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE + - `\` so Remote changes CAN NOT be detected!` - ) - } - const forceCache = await fs.exists( - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE - ) - - const pluginConfig = createPluginConfig(pluginOptions) - const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( - `environment` - )}` - - const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` - const CACHE_SYNC_DATA = `contentful-sync-data-${sourceId}` - - /* - * Subsequent calls of Contentfuls sync API return only changed data. - * - * In some cases, especially when using rich-text fields, there can be data - * missing from referenced entries. This breaks the reference matching. - * - * To workround this, we cache the initial sync data and merge it - * with all data from subsequent syncs. Afterwards the references get - * resolved via the Contentful JS SDK. - */ - const syncToken = await cache.get(CACHE_SYNC_TOKEN) - let previousSyncData = { - assets: [], - entries: [], - } - const cachedData = await cache.get(CACHE_SYNC_DATA) - - if (cachedData) { - previousSyncData = cachedData - } - - const fetchActivity = reporter.activityTimer( - `Contentful: Fetch data (${sourceId})`, - { - parentSpan, - } - ) - - // If the cache has data, use it. Otherwise do a remote fetch anyways and prime the cache now. - // If present, do NOT contact contentful, skip the round trips entirely - if (forceCache) { - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Skipping remote fetch, using data stored in \`${process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE}\`` - ) - ;({ - currentSyncData, - contentTypeItems, - tagItems, - defaultLocale, - locales, - space, - } = v8.deserialize( - fs.readFileSync(process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) - )) - } else { - const online = await isOnline() - - // If the user knows they are offline, serve them cached result - // For prod builds though always fail if we can't get the latest data - if ( - !online && - process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && - process.env.NODE_ENV !== `production` - ) { - reporter.info(`Using Contentful Offline cache ⚠️`) - reporter.info( - `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` - ) - - return - } - if (process.env.GATSBY_CONTENTFUL_OFFLINE) { - reporter.info( - `Note: \`GATSBY_CONTENTFUL_OFFLINE\` was set but it either was not \`true\`, we _are_ online, or we are in production mode, so the flag is ignored.` - ) - } - - fetchActivity.start() - ;({ - currentSyncData, - contentTypeItems, - tagItems, - defaultLocale, - locales, - space, - } = await fetchData({ - syncToken, - reporter, - pluginConfig, - parentSpan, - })) - - if (process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Writing v8 serialized glob of remote data to: ` + - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE - ) - fs.writeFileSync( - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE, - v8.serialize({ - currentSyncData, - contentTypeItems, - tagItems, - defaultLocale, - locales, - space, - }) - ) - } - } - - // Check for restricted content type names - const useNameForId = pluginConfig.get(`useNameForId`) - const restrictedContentTypes = [`entity`, `reference`, `asset`] - - if (pluginConfig.get(`enableTags`)) { - restrictedContentTypes.push(`tags`) - } - - contentTypeItems.forEach(contentTypeItem => { - // Establish identifier for content type - // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, - // but sometimes a base62 uuid generated by Contentful, hence the option) - let contentTypeItemId - if (useNameForId) { - contentTypeItemId = contentTypeItem.name.toLowerCase() - } else { - contentTypeItemId = contentTypeItem.sys.id.toLowerCase() - } - - if (restrictedContentTypes.includes(contentTypeItemId)) { - reporter.panic({ - id: CODES.FetchContentTypes, - context: { - sourceMessage: `Restricted ContentType name found. The name "${contentTypeItemId}" is not allowed.`, - }, - }) - } - }) - - const allLocales = locales - locales = locales.filter(pluginConfig.get(`localeFilter`)) - reporter.verbose( - `Default locale: ${defaultLocale}. All locales: ${allLocales - .map(({ code }) => code) - .join(`, `)}` - ) - if (allLocales.length !== locales.length) { - reporter.verbose( - `After plugin.options.localeFilter: ${locales - .map(({ code }) => code) - .join(`, `)}` - ) - } - if (locales.length === 0) { - reporter.panic({ - id: CODES.LocalesMissing, - context: { - sourceMessage: `Please check if your localeFilter is configured properly. Locales '${allLocales - .map(item => item.code) - .join(`,`)}' were found but were filtered down to none.`, - }, - }) - } - - // Create a map of up to date entries and assets - function mergeSyncData(previous, current, deletedEntities) { - const deleted = new Set(deletedEntities.map(e => e.sys.id)) - const entryMap = new Map() - previous.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) - current.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) - return [...entryMap.values()] - } - - const mergedSyncData = { - entries: mergeSyncData( - previousSyncData.entries, - currentSyncData.entries, - currentSyncData.deletedEntries - ), - assets: mergeSyncData( - previousSyncData.assets, - currentSyncData.assets, - currentSyncData.deletedAssets - ), - } - - // @todo based on the sys metadata we should be able to differentiate new and updated entities - reporter.info( - `Contentful: ${currentSyncData.entries.length} new/updated entries` - ) - reporter.info( - `Contentful: ${currentSyncData.deletedEntries.length} deleted entries` - ) - reporter.info(`Contentful: ${previousSyncData.entries.length} cached entries`) - reporter.info( - `Contentful: ${currentSyncData.assets.length} new/updated assets` - ) - reporter.info(`Contentful: ${previousSyncData.assets.length} cached assets`) - reporter.info( - `Contentful: ${currentSyncData.deletedAssets.length} deleted assets` - ) - - // Update syncToken - const nextSyncToken = currentSyncData.nextSyncToken - - await Promise.all([ - cache.set(CACHE_SYNC_DATA, mergedSyncData), - cache.set(CACHE_SYNC_TOKEN, nextSyncToken), - cache.set(`contentful-sync-result-${sourceId}`, { - contentTypeItems, - locales, - space, - defaultLocale, - tagItems, - deletedEntries: currentSyncData.deletedEntries, - deletedAssets: currentSyncData.deletedAssets, - }), - ]) - - fetchActivity.end() } diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index c66d9d39da974..729134b7a7ff9 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -1,12 +1,15 @@ // @todo import syntax! import _ from "lodash" -const path = require(`path`) const fs = require(`fs-extra`) -import normalize from "./normalize" +const v8 = require(`v8`) const { createClient } = require(`contentful`) +import normalize from "./normalize" const { createPluginConfig } = require(`./plugin-options`) +const { fetchContent } = require(`./fetch`) +const { CODES } = require(`./report`) import { downloadContentfulAssets } from "./download-contentful-assets" +import { getFileSystemCachePath } from "./fs-cache" const conflictFieldPrefix = `contentful` @@ -49,12 +52,12 @@ export async function sourceNodes( const { createNode, touchNode, deleteNode } = actions const isOnline = require(`is-online`) const online = await isOnline() - const forceCache = await fs.exists( - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE - ) + const { fsForceCache, fsCacheFileExists, fsCacheFilePath } = + await getFileSystemCachePath() + if ( !online && - !forceCache && + !fsForceCache && process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && process.env.NODE_ENV !== `production` ) { @@ -77,17 +80,191 @@ export async function sourceNodes( `environment` )}` - const mergedSyncData = await cache.get(`contentful-sync-data-${sourceId}`) + const fetchActivity = reporter.activityTimer( + `Contentful: Fetch data (${sourceId})`, + { + parentSpan, + } + ) + fetchActivity.start() + + let currentSyncData + let contentTypeItems + let tagItems + let defaultLocale + let locales + let space + + const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` + const CACHE_SYNC_DATA = `contentful-sync-data-${sourceId}` + const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}` + + /* + * Subsequent calls of Contentfuls sync API return only changed data. + * + * In some cases, especially when using rich-text fields, there can be data + * missing from referenced entries. This breaks the reference matching. + * + * To workround this, we cache the initial sync data and merge it + * with all data from subsequent syncs. Afterwards the references get + * resolved via the Contentful JS SDK. + */ + const syncToken = await cache.get(CACHE_SYNC_TOKEN) + + // If the cache has data, use it. Otherwise do a remote fetch anyways and prime the cache now. + // If present, do NOT contact contentful, skip the round trips entirely + if (fsCacheFileExists) { + reporter.info( + `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Skipping remote fetch, using data stored in \`${fsCacheFilePath}\`` + ) + const dataCacheBuffer = await fs.readFile(fsCacheFilePath) + ;({ + currentSyncData, + contentTypeItems, + tagItems, + defaultLocale, + locales, + space, + } = v8.deserialize(dataCacheBuffer)) + console.log({ + currentSyncData, + contentTypeItems, + tagItems, + defaultLocale, + locales, + space, + }) + } else { + const online = await isOnline() + + // If the user knows they are offline, serve them cached result + // For prod builds though always fail if we can't get the latest data + if ( + !online && + process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && + process.env.NODE_ENV !== `production` + ) { + reporter.info(`Using Contentful Offline cache ⚠️`) + reporter.info( + `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` + ) - const { - contentTypeItems, - locales, - space, - defaultLocale, - tagItems, - deletedEntries, - deletedAssets, - } = await cache.get(`contentful-sync-result-${sourceId}`) + return + } + if (process.env.GATSBY_CONTENTFUL_OFFLINE) { + reporter.info( + `Note: \`GATSBY_CONTENTFUL_OFFLINE\` was set but it either was not \`true\`, we _are_ online, or we are in production mode, so the flag is ignored.` + ) + } + + // Actual fetch of data from Contentful + ;({ currentSyncData, tagItems, defaultLocale, locales, space } = + await fetchContent({ syncToken, pluginConfig, reporter })) + + contentTypeItems = await cache.get(CACHE_CONTENT_TYPES) + + // Write to FS cache if desired + if (fsForceCache) { + reporter.info( + `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Writing v8 serialized glob of remote data to: ` + + fsCacheFilePath + ) + await fs.writeFile( + fsCacheFilePath, + v8.serialize({ + currentSyncData, + contentTypeItems, + tagItems, + defaultLocale, + locales, + space, + }) + ) + } + } + + const allLocales = locales + locales = locales.filter(pluginConfig.get(`localeFilter`)) + reporter.verbose( + `Default locale: ${defaultLocale}. All locales: ${allLocales + .map(({ code }) => code) + .join(`, `)}` + ) + if (allLocales.length !== locales.length) { + reporter.verbose( + `After plugin.options.localeFilter: ${locales + .map(({ code }) => code) + .join(`, `)}` + ) + } + if (locales.length === 0) { + reporter.panic({ + id: CODES.LocalesMissing, + context: { + sourceMessage: `Please check if your localeFilter is configured properly. Locales '${allLocales + .map(item => item.code) + .join(`,`)}' were found but were filtered down to none.`, + }, + }) + } + + // Create a map of up to date entries and assets + function mergeSyncData(previous, current, deletedEntities) { + const deleted = new Set(deletedEntities.map(e => e.sys.id)) + const entryMap = new Map() + previous.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) + current.forEach(e => !deleted.has(e.sys.id) && entryMap.set(e.sys.id, e)) + return [...entryMap.values()] + } + + let previousSyncData = { + assets: [], + entries: [], + } + const cachedData = await cache.get(CACHE_SYNC_DATA) + + if (cachedData) { + previousSyncData = cachedData + } + + const mergedSyncData = { + entries: mergeSyncData( + previousSyncData.entries, + currentSyncData.entries, + currentSyncData.deletedEntries + ), + assets: mergeSyncData( + previousSyncData.assets, + currentSyncData.assets, + currentSyncData.deletedAssets + ), + } + + // @todo based on the sys metadata we should be able to differentiate new and updated entities + reporter.info( + `Contentful: ${currentSyncData.entries.length} new/updated entries` + ) + reporter.info( + `Contentful: ${currentSyncData.deletedEntries.length} deleted entries` + ) + reporter.info(`Contentful: ${previousSyncData.entries.length} cached entries`) + reporter.info( + `Contentful: ${currentSyncData.assets.length} new/updated assets` + ) + reporter.info(`Contentful: ${previousSyncData.assets.length} cached assets`) + reporter.info( + `Contentful: ${currentSyncData.deletedAssets.length} deleted assets` + ) + + // Update syncToken + const nextSyncToken = currentSyncData.nextSyncToken + + await Promise.all([ + cache.set(CACHE_SYNC_DATA, mergedSyncData), + cache.set(CACHE_SYNC_TOKEN, nextSyncToken), + ]) + + fetchActivity.end() // Process data fetch results and turn them into GraphQL entities const processingActivity = reporter.activityTimer( @@ -102,7 +279,7 @@ export async function sourceNodes( const mergedSyncDataRaw = _.cloneDeep(mergedSyncData) // Use the JS-SDK to resolve the entries and assets - const res = createClient({ + const res = await createClient({ space: `none`, accessToken: `fake-access-token`, }).parseEntries({ @@ -243,6 +420,8 @@ export async function sourceNodes( }) } + const { deletedEntries, deletedAssets } = currentSyncData + if (deletedEntries.length || deletedAssets.length) { const deletionActivity = reporter.activityTimer( `Contentful: Deleting ${deletedEntries.length} nodes and ${deletedAssets.length} assets (${sourceId})`, From 845e19fa67bd4aa302db47ca28c9a4bcdafab458 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Fri, 17 Sep 2021 14:41:39 +0200 Subject: [PATCH 10/27] refactor(contentful): simplify fetch code and add more comments --- .../gatsby-source-contentful/src/fetch.js | 219 ++++++++---------- 1 file changed, 91 insertions(+), 128 deletions(-) diff --git a/packages/gatsby-source-contentful/src/fetch.js b/packages/gatsby-source-contentful/src/fetch.js index 3c8dbad11530c..5e797d1c0eefe 100644 --- a/packages/gatsby-source-contentful/src/fetch.js +++ b/packages/gatsby-source-contentful/src/fetch.js @@ -134,10 +134,84 @@ function createContentfulClientOptions({ return contentfulClientOptions } +function handleContentfulError({ + e, + reporter, + contentfulClientOptions, + pluginConfig, +}) { + let details + let errors + if (e.code === `ENOTFOUND`) { + details = `You seem to be offline` + } else if (e.code === `SELF_SIGNED_CERT_IN_CHAIN`) { + reporter.panic({ + id: CODES.SelfSignedCertificate, + context: { + sourceMessage: `We couldn't make a secure connection to your contentful space. Please check if you have any self-signed SSL certificates installed.`, + }, + }) + } else if (e.responseData) { + if ( + e.responseData.status === 404 && + contentfulClientOptions.environment && + contentfulClientOptions.environment !== `master` + ) { + // environments need to have access to master + details = `Unable to access your space. Check if ${chalk.yellow( + `environment` + )} is correct and your ${chalk.yellow( + `accessToken` + )} has access to the ${chalk.yellow( + contentfulClientOptions.environment + )} and the ${chalk.yellow(`master`)} environments.` + errors = { + accessToken: `Check if setting is correct`, + environment: `Check if setting is correct`, + } + } else if (e.responseData.status === 404) { + // host and space used to generate url + details = `Endpoint not found. Check if ${chalk.yellow( + `host` + )} and ${chalk.yellow(`spaceId`)} settings are correct` + errors = { + host: `Check if setting is correct`, + spaceId: `Check if setting is correct`, + } + } else if (e.responseData.status === 401) { + // authorization error + details = `Authorization error. Check if ${chalk.yellow( + `accessToken` + )} and ${chalk.yellow(`environment`)} are correct` + errors = { + accessToken: `Check if setting is correct`, + environment: `Check if setting is correct`, + } + } + } + + reporter.panic({ + context: { + sourceMessage: `Accessing your Contentful space failed: ${createContentfulErrorMessage( + e + )} + +Try setting GATSBY_CONTENTFUL_OFFLINE=true to see if we can serve from cache. +${details ? `\n${details}\n` : ``} +Used options: +${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`, + }, + }) +} + +/** + * Fetches: + * * Locales with default locale + * * Entries and assets + * * Tags + */ async function fetchContent({ syncToken, pluginConfig, reporter }) { - // Fetch articles. - let syncProgress - const pageLimit = pluginConfig.get(`pageLimit`) + // Fetch locales and check connectivity const contentfulClientOptions = createContentfulClientOptions({ pluginConfig, reporter, @@ -146,8 +220,6 @@ async function fetchContent({ syncToken, pluginConfig, reporter }) { // The sync API puts the locale in all fields in this format { fieldName: // {'locale': value} } so we need to get the space and its default local. - // - // We'll extend this soon to support multiple locales. let space let locales let defaultLocale = `en-US` @@ -160,70 +232,18 @@ async function fetchContent({ syncToken, pluginConfig, reporter }) { `Default locale is: ${defaultLocale}. There are ${locales.length} locales in total.` ) } catch (e) { - let details - let errors - if (e.code === `ENOTFOUND`) { - details = `You seem to be offline` - } else if (e.code === `SELF_SIGNED_CERT_IN_CHAIN`) { - reporter.panic({ - id: CODES.SelfSignedCertificate, - context: { - sourceMessage: `We couldn't make a secure connection to your contentful space. Please check if you have any self-signed SSL certificates installed.`, - }, - }) - } else if (e.responseData) { - if ( - e.responseData.status === 404 && - contentfulClientOptions.environment && - contentfulClientOptions.environment !== `master` - ) { - // environments need to have access to master - details = `Unable to access your space. Check if ${chalk.yellow( - `environment` - )} is correct and your ${chalk.yellow( - `accessToken` - )} has access to the ${chalk.yellow( - contentfulClientOptions.environment - )} and the ${chalk.yellow(`master`)} environments.` - errors = { - accessToken: `Check if setting is correct`, - environment: `Check if setting is correct`, - } - } else if (e.responseData.status === 404) { - // host and space used to generate url - details = `Endpoint not found. Check if ${chalk.yellow( - `host` - )} and ${chalk.yellow(`spaceId`)} settings are correct` - errors = { - host: `Check if setting is correct`, - spaceId: `Check if setting is correct`, - } - } else if (e.responseData.status === 401) { - // authorization error - details = `Authorization error. Check if ${chalk.yellow( - `accessToken` - )} and ${chalk.yellow(`environment`)} are correct` - errors = { - accessToken: `Check if setting is correct`, - environment: `Check if setting is correct`, - } - } - } - - reporter.panic({ - context: { - sourceMessage: `Accessing your Contentful space failed: ${createContentfulErrorMessage( - e - )} - -Try setting GATSBY_CONTENTFUL_OFFLINE=true to see if we can serve from cache. -${details ? `\n${details}\n` : ``} -Used options: -${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`, - }, + handleContentfulError({ + e, + reporter, + contentfulClientOptions, + pluginConfig, }) } + // Fetch entries and assets via Contentful CDA sync API + let syncProgress + const pageLimit = pluginConfig.get(`pageLimit`) + let currentSyncData let currentPageLimit = pageLimit let lastCurrentPageLimit @@ -319,6 +339,10 @@ ${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`, module.exports.fetchContent = fetchContent +/** + * Fetches: + * * Content types + */ async function fetchContentTypes({ pluginConfig, reporter }) { const contentfulClientOptions = createContentfulClientOptions({ pluginConfig, @@ -353,68 +377,7 @@ async function fetchContentTypes({ pluginConfig, reporter }) { contentTypes = contentTypes.items } catch (e) { - let details - let errors - if (e.code === `ENOTFOUND`) { - details = `You seem to be offline` - } else if (e.code === `SELF_SIGNED_CERT_IN_CHAIN`) { - reporter.panic({ - id: CODES.SelfSignedCertificate, - context: { - sourceMessage: `We couldn't make a secure connection to your contentful space. Please check if you have any self-signed SSL certificates installed.`, - }, - }) - } else if (e.responseData) { - if ( - e.responseData.status === 404 && - contentfulClientOptions.environment && - contentfulClientOptions.environment !== `master` - ) { - // environments need to have access to master - details = `Unable to access your space. Check if ${chalk.yellow( - `environment` - )} is correct and your ${chalk.yellow( - `accessToken` - )} has access to the ${chalk.yellow( - contentfulClientOptions.environment - )} and the ${chalk.yellow(`master`)} environments.` - errors = { - accessToken: `Check if setting is correct`, - environment: `Check if setting is correct`, - } - } else if (e.responseData.status === 404) { - // host and space used to generate url - details = `Endpoint not found. Check if ${chalk.yellow( - `host` - )} and ${chalk.yellow(`spaceId`)} settings are correct` - errors = { - host: `Check if setting is correct`, - spaceId: `Check if setting is correct`, - } - } else if (e.responseData.status === 401) { - // authorization error - details = `Authorization error. Check if ${chalk.yellow( - `accessToken` - )} and ${chalk.yellow(`environment`)} are correct` - errors = { - accessToken: `Check if setting is correct`, - environment: `Check if setting is correct`, - } - } - } - - reporter.panic({ - context: { - sourceMessage: `Accessing your Contentful space failed: ${createContentfulErrorMessage( - e - )} - -Try setting GATSBY_CONTENTFUL_OFFLINE=true to see if we can serve from cache. -${details ? `\n${details}\n` : ``} -Used options: -${formatPluginOptionsForCLI(pluginConfig.getOriginalPluginOptions(), errors)}`, - }, - }) + handleContentfulError(e) } return contentTypes From 2b6f64ffd0cd9a7fa31981bc5fc92de9a80649ce Mon Sep 17 00:00:00 2001 From: axe312ger Date: Fri, 17 Sep 2021 16:14:02 +0200 Subject: [PATCH 11/27] refactor(contentful): remove GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE --- .../src/create-schema-customization.js | 28 +---- .../gatsby-source-contentful/src/fs-cache.js | 26 ---- .../src/source-nodes.js | 118 +++++------------- 3 files changed, 33 insertions(+), 139 deletions(-) delete mode 100644 packages/gatsby-source-contentful/src/fs-cache.js diff --git a/packages/gatsby-source-contentful/src/create-schema-customization.js b/packages/gatsby-source-contentful/src/create-schema-customization.js index 1d12efa70505e..18a2d7db1f3fc 100644 --- a/packages/gatsby-source-contentful/src/create-schema-customization.js +++ b/packages/gatsby-source-contentful/src/create-schema-customization.js @@ -1,46 +1,20 @@ // @ts-check const _ = require(`lodash`) -const v8 = require(`v8`) -const fs = require(`fs-extra`) const { createPluginConfig } = require(`./plugin-options`) const { fetchContentTypes } = require(`./fetch`) const { CODES } = require(`./report`) -import { getFileSystemCachePath } from "./fs-cache" export async function createSchemaCustomization( { schema, actions, reporter, cache }, pluginOptions ) { - const { fsForceCache, fsCacheFileExists, fsCacheFilePath } = - await getFileSystemCachePath({ suffix: `content-type` }) const { createTypes } = actions const pluginConfig = createPluginConfig(pluginOptions) // Get content type items from Contentful - let contentTypeItems - if (!fsCacheFileExists) { - // Fetch content types as long fs cache is disabled or the fs cache file does not exist - contentTypeItems = await fetchContentTypes({ pluginConfig, reporter }) - - // Cache file to FS if required - if (fsForceCache) { - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Writing v8 serialized glob of remote content type data to: ` + - fsCacheFilePath - ) - await fs.writeFile(fsCacheFilePath, v8.serialize(contentTypeItems)) - } - } else { - // Load the content type item data from FS - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Reading v8 serialized glob of remote content type data from: ` + - fsCacheFilePath - ) - const contentTypeItemsCacheBuffer = await fs.readFile(fsCacheFilePath) - contentTypeItems = v8.deserialize(contentTypeItemsCacheBuffer) - } + const contentTypeItems = await fetchContentTypes({ pluginConfig, reporter }) // Check for restricted content type names and set id based on useNameForId const useNameForId = pluginConfig.get(`useNameForId`) diff --git a/packages/gatsby-source-contentful/src/fs-cache.js b/packages/gatsby-source-contentful/src/fs-cache.js deleted file mode 100644 index b3a58929fc0f4..0000000000000 --- a/packages/gatsby-source-contentful/src/fs-cache.js +++ /dev/null @@ -1,26 +0,0 @@ -import fs from "fs-extra" - -export async function getFileSystemCachePath({ suffix = null } = {}) { - if (!process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE) { - return { - fsForceCache: false, - fsCacheFileExists: false, - fsCacheFilePath: null, - } - } - - const fsCacheFilePath = [ - process.env.GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE, - suffix, - ] - .filter(Boolean) - .join(`-`) - - const fsCacheFileExists = await fs.exists(fsCacheFilePath) - - return { - fsForceCache: true, - fsCacheFileExists, - fsCacheFilePath, - } -} diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index 729134b7a7ff9..ba23a011f6dfb 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -1,7 +1,5 @@ // @todo import syntax! import _ from "lodash" -const fs = require(`fs-extra`) -const v8 = require(`v8`) const { createClient } = require(`contentful`) import normalize from "./normalize" @@ -9,7 +7,6 @@ const { createPluginConfig } = require(`./plugin-options`) const { fetchContent } = require(`./fetch`) const { CODES } = require(`./report`) import { downloadContentfulAssets } from "./download-contentful-assets" -import { getFileSystemCachePath } from "./fs-cache" const conflictFieldPrefix = `contentful` @@ -52,12 +49,9 @@ export async function sourceNodes( const { createNode, touchNode, deleteNode } = actions const isOnline = require(`is-online`) const online = await isOnline() - const { fsForceCache, fsCacheFileExists, fsCacheFilePath } = - await getFileSystemCachePath() if ( !online && - !fsForceCache && process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && process.env.NODE_ENV !== `production` ) { @@ -86,14 +80,28 @@ export async function sourceNodes( parentSpan, } ) - fetchActivity.start() - let currentSyncData - let contentTypeItems - let tagItems - let defaultLocale - let locales - let space + // If the user knows they are offline, serve them cached result + // For prod builds though always fail if we can't get the latest data + if ( + !online && + process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && + process.env.NODE_ENV !== `production` + ) { + reporter.info(`Using Contentful Offline cache ⚠️`) + reporter.info( + `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` + ) + + return + } + if (process.env.GATSBY_CONTENTFUL_OFFLINE) { + reporter.info( + `Note: \`GATSBY_CONTENTFUL_OFFLINE\` was set but it either was not \`true\`, we _are_ online, or we are in production mode, so the flag is ignored.` + ) + } + + fetchActivity.start() const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` const CACHE_SYNC_DATA = `contentful-sync-data-${sourceId}` @@ -111,82 +119,20 @@ export async function sourceNodes( */ const syncToken = await cache.get(CACHE_SYNC_TOKEN) - // If the cache has data, use it. Otherwise do a remote fetch anyways and prime the cache now. - // If present, do NOT contact contentful, skip the round trips entirely - if (fsCacheFileExists) { - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Skipping remote fetch, using data stored in \`${fsCacheFilePath}\`` - ) - const dataCacheBuffer = await fs.readFile(fsCacheFilePath) - ;({ - currentSyncData, - contentTypeItems, - tagItems, - defaultLocale, - locales, - space, - } = v8.deserialize(dataCacheBuffer)) - console.log({ - currentSyncData, - contentTypeItems, - tagItems, - defaultLocale, - locales, - space, - }) - } else { - const online = await isOnline() - - // If the user knows they are offline, serve them cached result - // For prod builds though always fail if we can't get the latest data - if ( - !online && - process.env.GATSBY_CONTENTFUL_OFFLINE === `true` && - process.env.NODE_ENV !== `production` - ) { - reporter.info(`Using Contentful Offline cache ⚠️`) - reporter.info( - `Cache may be invalidated if you edit package.json, gatsby-node.js or gatsby-config.js files` - ) - - return - } - if (process.env.GATSBY_CONTENTFUL_OFFLINE) { - reporter.info( - `Note: \`GATSBY_CONTENTFUL_OFFLINE\` was set but it either was not \`true\`, we _are_ online, or we are in production mode, so the flag is ignored.` - ) - } - - // Actual fetch of data from Contentful - ;({ currentSyncData, tagItems, defaultLocale, locales, space } = - await fetchContent({ syncToken, pluginConfig, reporter })) - - contentTypeItems = await cache.get(CACHE_CONTENT_TYPES) + // Actual fetch of data from Contentful + const { + currentSyncData, + tagItems, + defaultLocale, + locales: allLocales, + space, + } = await fetchContent({ syncToken, pluginConfig, reporter }) - // Write to FS cache if desired - if (fsForceCache) { - reporter.info( - `GATSBY_CONTENTFUL_EXPERIMENTAL_FORCE_CACHE was set. Writing v8 serialized glob of remote data to: ` + - fsCacheFilePath - ) - await fs.writeFile( - fsCacheFilePath, - v8.serialize({ - currentSyncData, - contentTypeItems, - tagItems, - defaultLocale, - locales, - space, - }) - ) - } - } + const contentTypeItems = await cache.get(CACHE_CONTENT_TYPES) - const allLocales = locales - locales = locales.filter(pluginConfig.get(`localeFilter`)) + const locales = allLocales.filter(pluginConfig.get(`localeFilter`)) reporter.verbose( - `Default locale: ${defaultLocale}. All locales: ${allLocales + `Default locale: ${defaultLocale}. All locales: ${allLocales .map(({ code }) => code) .join(`, `)}` ) From 8ed2235a7b301b34098cca0ddfbbb61e92336376 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Fri, 17 Sep 2021 17:03:04 +0200 Subject: [PATCH 12/27] refactor(contentful): get latest content types on every schema customization and node sourcing --- .../src/__tests__/gatsby-node.js | 15 ++----- .../src/create-schema-customization.js | 43 +++---------------- .../gatsby-source-contentful/src/normalize.js | 37 ++++++++++++++++ .../src/source-nodes.js | 13 ++++-- 4 files changed, 58 insertions(+), 50 deletions(-) diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index 8a694be620c08..839f5103f7c19 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -18,10 +18,6 @@ const restrictedContentTypeFixture = require(`../__fixtures__/restricted-content const defaultPluginOptions = { spaceId: `testSpaceId` } -fetchContentTypes.mockImplementation(() => - startersBlogFixture.contentTypeItems() -) - const createMockCache = () => { const actualCacheMap = new Map() return { @@ -66,7 +62,7 @@ describe(`gatsby-node`, () => { await gatsbyNode.onPreBootstrap({ store }) await gatsbyNode.createSchemaCustomization( - { schema, actions, cache, reporter }, + { schema, actions, reporter }, pluginOptions ) @@ -293,6 +289,7 @@ describe(`gatsby-node`, () => { beforeEach(() => { fetchContent.mockClear() fetchContentTypes.mockClear() + fetchContentTypes.mockImplementation(startersBlogFixture.contentTypeItems) currentNodeMap = new Map() actions.createNode = jest.fn(async node => { node.internal.owner = `gatsby-source-contentful` @@ -340,9 +337,6 @@ describe(`gatsby-node`, () => { Array [ "contentful-sync-token-testSpaceId-master", ], - Array [ - "contentful-content-types-testSpaceId-master", - ], Array [ "contentful-sync-data-testSpaceId-master", ], @@ -370,7 +364,6 @@ describe(`gatsby-node`, () => { expect(cache.set.mock.calls.map(v => v[0])).toMatchInlineSnapshot(` Array [ - "contentful-content-types-testSpaceId-master", "contentful-sync-data-testSpaceId-master", "contentful-sync-token-testSpaceId-master", ] @@ -581,7 +574,7 @@ describe(`gatsby-node`, () => { it(`stores rich text as raw with references attached`, async () => { fetchContent.mockImplementationOnce(richTextFixture.initialSync) - fetchContentTypes.mockImplementationOnce(richTextFixture.contentTypeItems) + fetchContentTypes.mockImplementation(richTextFixture.contentTypeItems) // initial sync await simulateGatsbyBuild() @@ -624,7 +617,7 @@ describe(`gatsby-node`, () => { fetchContent.mockImplementationOnce( restrictedContentTypeFixture.initialSync ) - fetchContentTypes.mockImplementationOnce( + fetchContentTypes.mockImplementation( restrictedContentTypeFixture.contentTypeItems ) diff --git a/packages/gatsby-source-contentful/src/create-schema-customization.js b/packages/gatsby-source-contentful/src/create-schema-customization.js index 18a2d7db1f3fc..b91007dc5e179 100644 --- a/packages/gatsby-source-contentful/src/create-schema-customization.js +++ b/packages/gatsby-source-contentful/src/create-schema-customization.js @@ -1,12 +1,12 @@ // @ts-check const _ = require(`lodash`) +const { normalizeContentTypeItems } = require(`./normalize`) const { createPluginConfig } = require(`./plugin-options`) const { fetchContentTypes } = require(`./fetch`) -const { CODES } = require(`./report`) export async function createSchemaCustomization( - { schema, actions, reporter, cache }, + { schema, actions, reporter }, pluginOptions ) { const { createTypes } = actions @@ -16,42 +16,13 @@ export async function createSchemaCustomization( // Get content type items from Contentful const contentTypeItems = await fetchContentTypes({ pluginConfig, reporter }) - // Check for restricted content type names and set id based on useNameForId - const useNameForId = pluginConfig.get(`useNameForId`) - const restrictedContentTypes = [`entity`, `reference`, `asset`] - - if (pluginConfig.get(`enableTags`)) { - restrictedContentTypes.push(`tags`) - } - - contentTypeItems.forEach(contentTypeItem => { - // Establish identifier for content type - // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, - // but sometimes a base62 uuid generated by Contentful, hence the option) - let contentTypeItemId - if (useNameForId) { - contentTypeItemId = contentTypeItem.name.toLowerCase() - } else { - contentTypeItemId = contentTypeItem.sys.id.toLowerCase() - } - - if (restrictedContentTypes.includes(contentTypeItemId)) { - reporter.panic({ - id: CODES.FetchContentTypes, - context: { - sourceMessage: `Restricted ContentType name found. The name "${contentTypeItemId}" is not allowed.`, - }, - }) - } + // Prepare content types + normalizeContentTypeItems({ + contentTypeItems, + pluginConfig, + reporter, }) - // Store processed content types in cache for sourceNodes - const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( - `environment` - )}` - const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}` - await cache.set(CACHE_CONTENT_TYPES, contentTypeItems) - createTypes(` interface ContentfulEntry implements Node { contentful_id: String! diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 82a31c9afadd7..d07aa52d124f9 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -1,6 +1,8 @@ const _ = require(`lodash`) const stringify = require(`json-stringify-safe`) +const { CODES } = require(`./report`) + const typePrefix = `Contentful` const makeTypeName = type => _.upperFirst(_.camelCase(`${typePrefix} ${type}`)) @@ -179,6 +181,41 @@ exports.buildForeignReferenceMap = ({ return foreignReferenceMap } +// Check for restricted content type names and set id based on useNameForId +exports.normalizeContentTypeItems = ({ + contentTypeItems, + pluginConfig, + reporter, +}) => { + const useNameForId = pluginConfig.get(`useNameForId`) + const restrictedContentTypes = [`entity`, `reference`, `asset`] + + if (pluginConfig.get(`enableTags`)) { + restrictedContentTypes.push(`tags`) + } + + contentTypeItems.forEach(contentTypeItem => { + // Establish identifier for content type + // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, + // but sometimes a base62 uuid generated by Contentful, hence the option) + let contentTypeItemId + if (useNameForId) { + contentTypeItemId = contentTypeItem.name.toLowerCase() + } else { + contentTypeItemId = contentTypeItem.sys.id.toLowerCase() + } + + if (restrictedContentTypes.includes(contentTypeItemId)) { + reporter.panic({ + id: CODES.FetchContentTypes, + context: { + sourceMessage: `Restricted ContentType name found. The name "${contentTypeItemId}" is not allowed.`, + }, + }) + } + }) +} + function prepareTextNode(id, node, key, text) { const str = _.isString(text) ? text : `` const textNode = { diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index ba23a011f6dfb..b5e2f9fefede5 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -4,7 +4,7 @@ const { createClient } = require(`contentful`) import normalize from "./normalize" const { createPluginConfig } = require(`./plugin-options`) -const { fetchContent } = require(`./fetch`) +const { fetchContent, fetchContentTypes } = require(`./fetch`) const { CODES } = require(`./report`) import { downloadContentfulAssets } from "./download-contentful-assets" @@ -105,7 +105,6 @@ export async function sourceNodes( const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` const CACHE_SYNC_DATA = `contentful-sync-data-${sourceId}` - const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}` /* * Subsequent calls of Contentfuls sync API return only changed data. @@ -128,7 +127,15 @@ export async function sourceNodes( space, } = await fetchContent({ syncToken, pluginConfig, reporter }) - const contentTypeItems = await cache.get(CACHE_CONTENT_TYPES) + // Get content type items from Contentful + const contentTypeItems = await fetchContentTypes({ pluginConfig, reporter }) + + // Prepare content types + normalize.normalizeContentTypeItems({ + contentTypeItems, + pluginConfig, + reporter, + }) const locales = allLocales.filter(pluginConfig.get(`localeFilter`)) reporter.verbose( From 00e2c24397fda487776f77b0e549a9faf221e7a0 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Fri, 17 Sep 2021 18:13:51 +0200 Subject: [PATCH 13/27] docs(contentful): plugin options - remove unused forceFullSync and properly add enableTags --- .../gatsby-source-contentful/src/gatsby-node.js | 15 +++++---------- .../src/plugin-options.js | 1 - 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index dcf3b397dbff9..dbc76acae4b6e 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -88,11 +88,6 @@ For example, to filter locales on only germany \`localeFilter: locale => locale. List of locales and their codes can be found in Contentful app -> Settings -> Locales` ) .default(() => () => true), - forceFullSync: Joi.boolean() - .description( - `Prevents the use of sync tokens when accessing the Contentful API.` - ) - .default(false), pageLimit: Joi.number() .integer() .description( @@ -117,6 +112,11 @@ List of locales and their codes can be found in Contentful app -> Settings -> Lo .description( `Axios proxy configuration. See the [axios request config documentation](https://github.com/mzabriskie/axios#request-config) for further information about the supported values.` ), + enableTags: Joi.boolean() + .description( + `Enable the new tags feature. This will disallow the content type name "tags" till the next major version of this plugin.` + ) + .default(false), useNameForId: Joi.boolean() .description( `Use the content's \`name\` when generating the GraphQL schema e.g. a Content Type called \`[Component] Navigation bar\` will be named \`contentfulComponentNavigationBar\`. @@ -127,11 +127,6 @@ List of locales and their codes can be found in Contentful app -> Settings -> Lo If you are confident your Content Types will have natural-language IDs (e.g. \`blogPost\`), then you should set this option to \`false\`. If you are unable to ensure this, then you should leave this option set to \`true\` (the default).` ) .default(true), - enableTags: Joi.boolean() - .description( - `Enable the new tags feature. This will disallow the content type name "tags" till the next major version of this plugin.` - ) - .default(true), contentfulClientConfig: Joi.object() .description( `Additional config which will get passed to [Contentfuls JS SDK](https://github.com/contentful/contentful.js#configuration). diff --git a/packages/gatsby-source-contentful/src/plugin-options.js b/packages/gatsby-source-contentful/src/plugin-options.js index 5fe9d96fa3453..1a150c054d73f 100644 --- a/packages/gatsby-source-contentful/src/plugin-options.js +++ b/packages/gatsby-source-contentful/src/plugin-options.js @@ -9,7 +9,6 @@ const defaultOptions = { environment: `master`, downloadLocal: false, localeFilter: () => true, - forceFullSync: false, pageLimit: DEFAULT_PAGE_LIMIT, useNameForId: true, enableTags: false, From 6bd317e14906fa731f8c608f2ede1e2e334c1607 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Sat, 25 Sep 2021 13:32:55 +0200 Subject: [PATCH 14/27] test(contentful): update e2e test snapshots for https --- e2e-tests/contentful/snapshots.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/contentful/snapshots.js b/e2e-tests/contentful/snapshots.js index eebeb902df067..2207ca7c22e03 100644 --- a/e2e-tests/contentful/snapshots.js +++ b/e2e-tests/contentful/snapshots.js @@ -28,7 +28,7 @@ module.exports = { }, "rich-text": { "rich-text: All Features": { - "1": "
\n

Rich Text: All Features

\n

The European languages

\n

are members of the same family. Their separate existence is a myth. For:

\n
    \n
  • \n

    science

    \n
    • \n
  • \n

    music

    \n
    • \n
  • \n

    sport

    \n
    • \n
  • \n

    etc

    \n
    • \n
\n

Europe uses the same vocabulary.

\n
\n
\"\"\n\n \n \n \n
\n

\n
\n

The languages only differ in:

\n
    \n
  1. \n

    their grammar

    \n
    2. \n
  2. \n

    their pronunciation

    \n
    3. \n
  3. \n

    their most common words

    \n
    4. \n
  4. \n

    [Inline-ContentfulText]\n Text: Short\n :\n The quick brown fox jumps over the lazy dog.

    \n
    5. \n
\n

Everyone realizes why a new common language would be desirable: one could\n refuse to pay expensive translators.

\n

{\n \"userId\": 1,\n \"id\": 1,\n \"title\": \"delectus aut autem\",\n \"completed\": false\n }

\n

To achieve this, it would be necessary to have uniform grammar,\n pronunciation and more common words.

\n

[ContentfulLocation] Lat:\n 52.51627\n , Long:\n 13.3777

\n
\n

If several languages coalesce, the grammar of the resulting language is\n more simple and regular than that of the individual languages.

\n
\n

The new common language will be more simple and regular than the existing\n European languages. It will be as simple as Occidental; in fact, it will be\n

\n
\n
" + "1": "
\n

Rich Text: All Features

\n

The European languages

\n

are members of the same family. Their separate existence is a myth. For:

\n
    \n
  • \n

    science

    \n
    • \n
  • \n

    music

    \n
    • \n
  • \n

    sport

    \n
    • \n
  • \n

    etc

    \n
    • \n
\n

Europe uses the same vocabulary.

\n
\n
\"\"\n\n \n \n \n
\n

\n
\n

The languages only differ in:

\n
    \n
  1. \n

    their grammar

    \n
    2. \n
  2. \n

    their pronunciation

    \n
    3. \n
  3. \n

    their most common words

    \n
    4. \n
  4. \n

    [Inline-ContentfulText]\n Text: Short\n :\n The quick brown fox jumps over the lazy dog.

    \n
    5. \n
\n

Everyone realizes why a new common language would be desirable: one could\n refuse to pay expensive translators.

\n

{\n \"userId\": 1,\n \"id\": 1,\n \"title\": \"delectus aut autem\",\n \"completed\": false\n }

\n

To achieve this, it would be necessary to have uniform grammar,\n pronunciation and more common words.

\n

[ContentfulLocation] Lat:\n 52.51627\n , Long:\n 13.3777

\n
\n

If several languages coalesce, the grammar of the resulting language is\n more simple and regular than that of the individual languages.

\n
\n

The new common language will be more simple and regular than the existing\n European languages. It will be as simple as Occidental; in fact, it will be\n

\n
\n
" }, "rich-text: Basic": { "1": "
\n

Rich Text: Basic

\n

The European languages

\n

are members of the same family. Their separate existence is a myth. For:

\n
    \n
  • \n

    science

    \n
    • \n
  • \n

    music

    \n
    • \n
  • \n

    sport

    \n
    • \n
  • \n

    etc

    \n
    • \n
\n

Europe uses the same vocabulary.

\n
\n

The languages only differ in:

\n
    \n
  1. \n

    their grammar

    \n
    2. \n
  2. \n

    their pronunciation

    \n
    3. \n
  3. \n

    their most common words

    \n
    4. \n
\n

Everyone realizes why a new common language would be desirable: one could\n refuse to pay expensive translators.

\n

{\n \"userId\": 1,\n \"id\": 1,\n \"title\": \"delectus aut autem\",\n \"completed\": false\n }

\n

To achieve this, it would be necessary to have uniform grammar,\n pronunciation and more common words.

\n
\n

If several languages coalesce, the grammar of the resulting language is\n more simple and regular than that of the individual languages.

\n
\n

The new common language will be more simple and regular than the existing\n European languages. It will be as simple as Occidental; in fact, it will be\n

\n
\n
" @@ -37,7 +37,7 @@ module.exports = { "1": "
\n

Rich Text: Embedded Entry

\n

Embedded Entry

\n

[ContentfulText]\n The quick brown fox jumps over the lazy dog.

\n

\n

\n
\n
" }, "rich-text: Embedded Asset": { - "1": "
\n

Rich Text: Embedded asset

\n

Embedded Asset

\n
\n
\n \n \n \n
\n

\n

\n

\n
\n
" + "1": "
\n

Rich Text: Embedded asset

\n

Embedded Asset

\n
\n
\n \n \n \n
\n

\n

\n

\n
\n
" }, "rich-text: Embedded Entry With Deep Reference Loop": { "1": "
\n

Rich Text: Embedded entry with deep reference loop

\n

Embedded entry with deep reference loop

\n

[ContentfulReference]\n Content Reference: Many (2nd level loop)\n : [\n Number: Integer, Text: Short, Content Reference: One (Loop A ->\n B)\n ]

\n

\n

\n
\n
" From 8eed1daa997932e7455bc0955928f35ce3316c61 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Sat, 25 Sep 2021 13:37:10 +0200 Subject: [PATCH 15/27] chore(contentful): update e2e test schema for comparision --- e2e-tests/contentful/schema.gql | 193 ++++++++++++++++++-------------- 1 file changed, 110 insertions(+), 83 deletions(-) diff --git a/e2e-tests/contentful/schema.gql b/e2e-tests/contentful/schema.gql index 2a8e30403f33d..7a501577d38cb 100644 --- a/e2e-tests/contentful/schema.gql +++ b/e2e-tests/contentful/schema.gql @@ -1,4 +1,4 @@ -### Type definitions saved at 2021-05-21T17:02:49.951Z ### +### Type definitions saved at 2021-09-25T11:33:25.217Z ### type File implements Node @dontInfer { sourceInstanceName: String! @@ -96,43 +96,23 @@ type SitePage implements Node @dontInfer { internalComponentName: String! componentChunkName: String! matchPath: String + pageContext: JSON } -type MarkdownHeading { - id: String - value: String - depth: Int -} - -enum MarkdownHeadingLevels { - h1 - h2 - h3 - h4 - h5 - h6 -} - -enum MarkdownExcerptFormats { - PLAIN - HTML - MARKDOWN -} - -type MarkdownWordCount { - paragraphs: Int - sentences: Int - words: Int -} - -type MarkdownRemark implements Node @childOf(mimeTypes: ["text/markdown", "text/x-markdown"], types: ["contentfulTextLongPlainTextNode", "contentfulTextLongMarkdownTextNode", "contentfulTextLongLocalizedTextNode"]) @derivedTypes @dontInfer { - frontmatter: MarkdownRemarkFrontmatter - excerpt: String - rawMarkdownBody: String +type SitePlugin implements Node @dontInfer { + resolve: String + name: String + version: String + nodeAPIs: [String] + browserAPIs: [String] + ssrAPIs: [String] + pluginFilepath: String + pluginOptions: JSON + packageJson: JSON } -type MarkdownRemarkFrontmatter { - title: String +type SiteBuildMetadata implements Node @dontInfer { + buildTime: Date @dateformat } interface ContentfulEntry implements Node { @@ -184,15 +164,16 @@ type ContentfulNumber implements ContentfulReference & ContentfulEntry & Node @d contentful_id: String! node_locale: String! title: String - decimal: Float + integerLocalized: Int spaceId: String createdAt: Date @dateformat updatedAt: Date @dateformat sys: ContentfulNumberSys + metadata: ContentfulNumberMetadata + decimal: Float integer: Int content_reference: [ContentfulContentReference] @link(by: "id", from: "content reference___NODE") @proxy(from: "content reference___NODE") decimalLocalized: Float - integerLocalized: Int } type ContentfulNumberSys @derivedTypes { @@ -211,22 +192,31 @@ type ContentfulNumberSysContentTypeSys { id: String } +type ContentfulNumberMetadata { + tags: [ContentfulTag] @link(by: "id", from: "tags___NODE") +} + +type ContentfulTag implements Node @dontInfer { + name: String! + contentful_id: String! +} + type ContentfulContentReference implements ContentfulReference & ContentfulEntry & Node @derivedTypes @dontInfer { contentful_id: String! node_locale: String! title: String - manyLocalized: [ContentfulNumberContentfulTextUnion] @link(by: "id", from: "manyLocalized___NODE") + one: ContentfulContentReferenceContentfulTextUnion @link(by: "id", from: "one___NODE") + content_reference: [ContentfulContentReference] @link(by: "id", from: "content reference___NODE") @proxy(from: "content reference___NODE") spaceId: String createdAt: Date @dateformat updatedAt: Date @dateformat sys: ContentfulContentReferenceSys oneLocalized: ContentfulNumber @link(by: "id", from: "oneLocalized___NODE") - one: ContentfulContentReferenceContentfulTextUnion @link(by: "id", from: "one___NODE") - content_reference: [ContentfulContentReference] @link(by: "id", from: "content reference___NODE") @proxy(from: "content reference___NODE") many: [ContentfulContentReferenceContentfulNumberContentfulTextUnion] @link(by: "id", from: "many___NODE") + manyLocalized: [ContentfulNumberContentfulTextUnion] @link(by: "id", from: "manyLocalized___NODE") } -union ContentfulNumberContentfulTextUnion = ContentfulNumber | ContentfulText +union ContentfulContentReferenceContentfulTextUnion = ContentfulContentReference | ContentfulText type ContentfulContentReferenceSys @derivedTypes { type: String @@ -244,10 +234,10 @@ type ContentfulContentReferenceSysContentTypeSys { id: String } -union ContentfulContentReferenceContentfulTextUnion = ContentfulContentReference | ContentfulText - union ContentfulContentReferenceContentfulNumberContentfulTextUnion = ContentfulContentReference | ContentfulNumber | ContentfulText +union ContentfulNumberContentfulTextUnion = ContentfulNumber | ContentfulText + type ContentfulText implements ContentfulReference & ContentfulEntry & Node @derivedTypes @dontInfer { contentful_id: String! node_locale: String! @@ -257,12 +247,12 @@ type ContentfulText implements ContentfulReference & ContentfulEntry & Node @der createdAt: Date @dateformat updatedAt: Date @dateformat sys: ContentfulTextSys - shortLocalized: String longMarkdown: contentfulTextLongMarkdownTextNode @link(by: "id", from: "longMarkdown___NODE") + shortLocalized: String longPlain: contentfulTextLongPlainTextNode @link(by: "id", from: "longPlain___NODE") + shortList: [String] short: String content_reference: [ContentfulContentReference] @link(by: "id", from: "content reference___NODE") @proxy(from: "content reference___NODE") - shortList: [String] } type contentfulTextLongLocalizedTextNode implements Node @derivedTypes @childOf(types: ["ContentfulText"]) @dontInfer { @@ -312,14 +302,14 @@ type ContentfulMediaReference implements ContentfulReference & ContentfulEntry & contentful_id: String! node_locale: String! title: String - manyLocalized: [ContentfulAsset] @link(by: "id", from: "manyLocalized___NODE") + one: ContentfulAsset @link(by: "id", from: "one___NODE") spaceId: String createdAt: Date @dateformat updatedAt: Date @dateformat sys: ContentfulMediaReferenceSys - many: [ContentfulAsset] @link(by: "id", from: "many___NODE") oneLocalized: ContentfulAsset @link(by: "id", from: "oneLocalized___NODE") - one: ContentfulAsset @link(by: "id", from: "one___NODE") + many: [ContentfulAsset] @link(by: "id", from: "many___NODE") + manyLocalized: [ContentfulAsset] @link(by: "id", from: "manyLocalized___NODE") } type ContentfulMediaReferenceSys @derivedTypes { @@ -370,14 +360,14 @@ type ContentfulDate implements ContentfulReference & ContentfulEntry & Node @der contentful_id: String! node_locale: String! title: String - dateLocalized: Date @dateformat + dateTimeTimezone: Date @dateformat spaceId: String createdAt: Date @dateformat updatedAt: Date @dateformat sys: ContentfulDateSys - dateTime: Date @dateformat - dateTimeTimezone: Date @dateformat date: Date @dateformat + dateLocalized: Date @dateformat + dateTime: Date @dateformat } type ContentfulDateSys @derivedTypes { @@ -409,8 +399,8 @@ type ContentfulLocation implements ContentfulReference & ContentfulEntry & Node } type ContentfulLocationLocationLocalized { - lat: Float lon: Float + lat: Float } type ContentfulLocationSys @derivedTypes { @@ -438,25 +428,39 @@ type ContentfulJson implements ContentfulReference & ContentfulEntry & Node @der contentful_id: String! node_locale: String! title: String - jsonLocalized: contentfulJsonJsonLocalizedJsonNode @link(by: "id", from: "jsonLocalized___NODE") + json: contentfulJsonJsonJsonNode @link(by: "id", from: "json___NODE") spaceId: String createdAt: Date @dateformat updatedAt: Date @dateformat sys: ContentfulJsonSys - json: contentfulJsonJsonJsonNode @link(by: "id", from: "json___NODE") + jsonLocalized: contentfulJsonJsonLocalizedJsonNode @link(by: "id", from: "jsonLocalized___NODE") } -type contentfulJsonJsonLocalizedJsonNode implements Node @derivedTypes @childOf(types: ["ContentfulJson"]) @dontInfer { +type contentfulJsonJsonJsonNode implements Node @derivedTypes @childOf(types: ["ContentfulJson"]) @dontInfer { age: Int city: String name: String - sys: contentfulJsonJsonLocalizedJsonNodeSys + sys: contentfulJsonJsonJsonNodeSys + Actors: [contentfulJsonJsonJsonNodeActors] } -type contentfulJsonJsonLocalizedJsonNodeSys { +type contentfulJsonJsonJsonNodeSys { type: String } +type contentfulJsonJsonJsonNodeActors { + age: Int + name: String + wife: String + photo: String + weight: Float + Born_At: String @proxy(from: "Born At") + children: [String] + Birthdate: String + hasChildren: Boolean + hasGreyHair: Boolean +} + type ContentfulJsonSys @derivedTypes { type: String revision: Int @@ -473,28 +477,14 @@ type ContentfulJsonSysContentTypeSys { id: String } -type contentfulJsonJsonJsonNode implements Node @derivedTypes @childOf(types: ["ContentfulJson"]) @dontInfer { - Actors: [contentfulJsonJsonJsonNodeActors] - sys: contentfulJsonJsonJsonNodeSys +type contentfulJsonJsonLocalizedJsonNode implements Node @derivedTypes @childOf(types: ["ContentfulJson"]) @dontInfer { name: String age: Int city: String + sys: contentfulJsonJsonLocalizedJsonNodeSys } -type contentfulJsonJsonJsonNodeActors { - name: String - age: Int - Born_At: String @proxy(from: "Born At") - Birthdate: String - photo: String - wife: String - weight: Float - hasChildren: Boolean - hasGreyHair: Boolean - children: [String] -} - -type contentfulJsonJsonJsonNodeSys { +type contentfulJsonJsonLocalizedJsonNodeSys { type: String } @@ -502,21 +492,21 @@ type ContentfulRichText implements ContentfulReference & ContentfulEntry & Node contentful_id: String! node_locale: String! title: String - richTextValidated: ContentfulRichTextRichTextValidated + richText: ContentfulRichTextRichText spaceId: String createdAt: Date @dateformat updatedAt: Date @dateformat sys: ContentfulRichTextSys + richTextValidated: ContentfulRichTextRichTextValidated richTextLocalized: ContentfulRichTextRichTextLocalized - richText: ContentfulRichTextRichText } -type ContentfulRichTextRichTextValidated { +type ContentfulRichTextRichText { raw: String - references: [ContentfulAssetContentfulLocationContentfulNumberContentfulTextUnion] @link(by: "id", from: "references___NODE") + references: [ContentfulAssetContentfulContentReferenceContentfulLocationContentfulTextUnion] @link(by: "id", from: "references___NODE") } -union ContentfulAssetContentfulLocationContentfulNumberContentfulTextUnion = ContentfulAsset | ContentfulLocation | ContentfulNumber | ContentfulText +union ContentfulAssetContentfulContentReferenceContentfulLocationContentfulTextUnion = ContentfulAsset | ContentfulContentReference | ContentfulLocation | ContentfulText type ContentfulRichTextSys @derivedTypes { type: String @@ -534,22 +524,59 @@ type ContentfulRichTextSysContentTypeSys { id: String } -type ContentfulRichTextRichTextLocalized { +type ContentfulRichTextRichTextValidated { raw: String + references: [ContentfulAssetContentfulLocationContentfulNumberContentfulTextUnion] @link(by: "id", from: "references___NODE") } -type ContentfulRichTextRichText { +union ContentfulAssetContentfulLocationContentfulNumberContentfulTextUnion = ContentfulAsset | ContentfulLocation | ContentfulNumber | ContentfulText + +type ContentfulRichTextRichTextLocalized { raw: String - references: [ContentfulAssetContentfulContentReferenceContentfulLocationContentfulTextUnion] @link(by: "id", from: "references___NODE") } -union ContentfulAssetContentfulContentReferenceContentfulLocationContentfulTextUnion = ContentfulAsset | ContentfulContentReference | ContentfulLocation | ContentfulText - type ContentfulValidatedContentReference implements ContentfulReference & ContentfulEntry & Node @dontInfer { contentful_id: String! node_locale: String! } +type MarkdownHeading { + id: String + value: String + depth: Int +} + +enum MarkdownHeadingLevels { + h1 + h2 + h3 + h4 + h5 + h6 +} + +enum MarkdownExcerptFormats { + PLAIN + HTML + MARKDOWN +} + +type MarkdownWordCount { + paragraphs: Int + sentences: Int + words: Int +} + +type MarkdownRemark implements Node @childOf(mimeTypes: ["text/markdown", "text/x-markdown"], types: ["contentfulTextLongPlainTextNode", "contentfulTextLongMarkdownTextNode", "contentfulTextLongLocalizedTextNode"]) @derivedTypes @dontInfer { + frontmatter: MarkdownRemarkFrontmatter + excerpt: String + rawMarkdownBody: String +} + +type MarkdownRemarkFrontmatter { + title: String +} + type ContentfulContentType implements Node @derivedTypes @dontInfer { name: String displayField: String From ea2e8406f82b2ccabcf7a45bb1555a9032662e46 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Fri, 8 Oct 2021 16:03:08 +0200 Subject: [PATCH 16/27] chore: add dontInfer to interfaces and align code style of type creation --- .../src/create-schema-customization.js | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/packages/gatsby-source-contentful/src/create-schema-customization.js b/packages/gatsby-source-contentful/src/create-schema-customization.js index b91007dc5e179..919978429ff1e 100644 --- a/packages/gatsby-source-contentful/src/create-schema-customization.js +++ b/packages/gatsby-source-contentful/src/create-schema-customization.js @@ -23,20 +23,29 @@ export async function createSchemaCustomization( reporter, }) - createTypes(` - interface ContentfulEntry implements Node { - contentful_id: String! - id: ID! - node_locale: String! - } - `) + createTypes( + schema.buildObjectType({ + name: `ContentfulEntry`, + fields: { + contentful_id: { type: `String!` }, + id: { type: `ID!` }, + node_locale: { type: `String!` }, + }, + extensions: { dontInfer: {} }, + interfaces: [`Node`], + }) + ) - createTypes(` - interface ContentfulReference { - contentful_id: String! - id: ID! - } - `) + createTypes( + schema.buildObjectType({ + name: `ContentfulReference`, + fields: { + contentful_id: { type: `String!` }, + id: { type: `ID!` }, + }, + extensions: { dontInfer: {} }, + }) + ) createTypes( schema.buildObjectType({ From bbebda5e8a7aef6bb9771d8eceebdaff70f34802 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Fri, 8 Oct 2021 17:49:31 +0200 Subject: [PATCH 17/27] Revert "refactor(contentful): get latest content types on every schema customization and node sourcing" This reverts commit 59af4101f46c7b4e9c0d623aceb1ef36ac5167fb. --- .../src/__tests__/gatsby-node.js | 15 +++++-- .../src/create-schema-customization.js | 43 ++++++++++++++++--- .../gatsby-source-contentful/src/normalize.js | 37 ---------------- .../src/source-nodes.js | 13 ++---- 4 files changed, 50 insertions(+), 58 deletions(-) diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index 839f5103f7c19..8a694be620c08 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -18,6 +18,10 @@ const restrictedContentTypeFixture = require(`../__fixtures__/restricted-content const defaultPluginOptions = { spaceId: `testSpaceId` } +fetchContentTypes.mockImplementation(() => + startersBlogFixture.contentTypeItems() +) + const createMockCache = () => { const actualCacheMap = new Map() return { @@ -62,7 +66,7 @@ describe(`gatsby-node`, () => { await gatsbyNode.onPreBootstrap({ store }) await gatsbyNode.createSchemaCustomization( - { schema, actions, reporter }, + { schema, actions, cache, reporter }, pluginOptions ) @@ -289,7 +293,6 @@ describe(`gatsby-node`, () => { beforeEach(() => { fetchContent.mockClear() fetchContentTypes.mockClear() - fetchContentTypes.mockImplementation(startersBlogFixture.contentTypeItems) currentNodeMap = new Map() actions.createNode = jest.fn(async node => { node.internal.owner = `gatsby-source-contentful` @@ -337,6 +340,9 @@ describe(`gatsby-node`, () => { Array [ "contentful-sync-token-testSpaceId-master", ], + Array [ + "contentful-content-types-testSpaceId-master", + ], Array [ "contentful-sync-data-testSpaceId-master", ], @@ -364,6 +370,7 @@ describe(`gatsby-node`, () => { expect(cache.set.mock.calls.map(v => v[0])).toMatchInlineSnapshot(` Array [ + "contentful-content-types-testSpaceId-master", "contentful-sync-data-testSpaceId-master", "contentful-sync-token-testSpaceId-master", ] @@ -574,7 +581,7 @@ describe(`gatsby-node`, () => { it(`stores rich text as raw with references attached`, async () => { fetchContent.mockImplementationOnce(richTextFixture.initialSync) - fetchContentTypes.mockImplementation(richTextFixture.contentTypeItems) + fetchContentTypes.mockImplementationOnce(richTextFixture.contentTypeItems) // initial sync await simulateGatsbyBuild() @@ -617,7 +624,7 @@ describe(`gatsby-node`, () => { fetchContent.mockImplementationOnce( restrictedContentTypeFixture.initialSync ) - fetchContentTypes.mockImplementation( + fetchContentTypes.mockImplementationOnce( restrictedContentTypeFixture.contentTypeItems ) diff --git a/packages/gatsby-source-contentful/src/create-schema-customization.js b/packages/gatsby-source-contentful/src/create-schema-customization.js index 919978429ff1e..68ed72aeaa8cf 100644 --- a/packages/gatsby-source-contentful/src/create-schema-customization.js +++ b/packages/gatsby-source-contentful/src/create-schema-customization.js @@ -1,12 +1,12 @@ // @ts-check const _ = require(`lodash`) -const { normalizeContentTypeItems } = require(`./normalize`) const { createPluginConfig } = require(`./plugin-options`) const { fetchContentTypes } = require(`./fetch`) +const { CODES } = require(`./report`) export async function createSchemaCustomization( - { schema, actions, reporter }, + { schema, actions, reporter, cache }, pluginOptions ) { const { createTypes } = actions @@ -16,13 +16,42 @@ export async function createSchemaCustomization( // Get content type items from Contentful const contentTypeItems = await fetchContentTypes({ pluginConfig, reporter }) - // Prepare content types - normalizeContentTypeItems({ - contentTypeItems, - pluginConfig, - reporter, + // Check for restricted content type names and set id based on useNameForId + const useNameForId = pluginConfig.get(`useNameForId`) + const restrictedContentTypes = [`entity`, `reference`, `asset`] + + if (pluginConfig.get(`enableTags`)) { + restrictedContentTypes.push(`tags`) + } + + contentTypeItems.forEach(contentTypeItem => { + // Establish identifier for content type + // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, + // but sometimes a base62 uuid generated by Contentful, hence the option) + let contentTypeItemId + if (useNameForId) { + contentTypeItemId = contentTypeItem.name.toLowerCase() + } else { + contentTypeItemId = contentTypeItem.sys.id.toLowerCase() + } + + if (restrictedContentTypes.includes(contentTypeItemId)) { + reporter.panic({ + id: CODES.FetchContentTypes, + context: { + sourceMessage: `Restricted ContentType name found. The name "${contentTypeItemId}" is not allowed.`, + }, + }) + } }) + // Store processed content types in cache for sourceNodes + const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( + `environment` + )}` + const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}` + await cache.set(CACHE_CONTENT_TYPES, contentTypeItems) + createTypes( schema.buildObjectType({ name: `ContentfulEntry`, diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index d07aa52d124f9..82a31c9afadd7 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -1,8 +1,6 @@ const _ = require(`lodash`) const stringify = require(`json-stringify-safe`) -const { CODES } = require(`./report`) - const typePrefix = `Contentful` const makeTypeName = type => _.upperFirst(_.camelCase(`${typePrefix} ${type}`)) @@ -181,41 +179,6 @@ exports.buildForeignReferenceMap = ({ return foreignReferenceMap } -// Check for restricted content type names and set id based on useNameForId -exports.normalizeContentTypeItems = ({ - contentTypeItems, - pluginConfig, - reporter, -}) => { - const useNameForId = pluginConfig.get(`useNameForId`) - const restrictedContentTypes = [`entity`, `reference`, `asset`] - - if (pluginConfig.get(`enableTags`)) { - restrictedContentTypes.push(`tags`) - } - - contentTypeItems.forEach(contentTypeItem => { - // Establish identifier for content type - // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, - // but sometimes a base62 uuid generated by Contentful, hence the option) - let contentTypeItemId - if (useNameForId) { - contentTypeItemId = contentTypeItem.name.toLowerCase() - } else { - contentTypeItemId = contentTypeItem.sys.id.toLowerCase() - } - - if (restrictedContentTypes.includes(contentTypeItemId)) { - reporter.panic({ - id: CODES.FetchContentTypes, - context: { - sourceMessage: `Restricted ContentType name found. The name "${contentTypeItemId}" is not allowed.`, - }, - }) - } - }) -} - function prepareTextNode(id, node, key, text) { const str = _.isString(text) ? text : `` const textNode = { diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index b5e2f9fefede5..ba23a011f6dfb 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -4,7 +4,7 @@ const { createClient } = require(`contentful`) import normalize from "./normalize" const { createPluginConfig } = require(`./plugin-options`) -const { fetchContent, fetchContentTypes } = require(`./fetch`) +const { fetchContent } = require(`./fetch`) const { CODES } = require(`./report`) import { downloadContentfulAssets } from "./download-contentful-assets" @@ -105,6 +105,7 @@ export async function sourceNodes( const CACHE_SYNC_TOKEN = `contentful-sync-token-${sourceId}` const CACHE_SYNC_DATA = `contentful-sync-data-${sourceId}` + const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}` /* * Subsequent calls of Contentfuls sync API return only changed data. @@ -127,15 +128,7 @@ export async function sourceNodes( space, } = await fetchContent({ syncToken, pluginConfig, reporter }) - // Get content type items from Contentful - const contentTypeItems = await fetchContentTypes({ pluginConfig, reporter }) - - // Prepare content types - normalize.normalizeContentTypeItems({ - contentTypeItems, - pluginConfig, - reporter, - }) + const contentTypeItems = await cache.get(CACHE_CONTENT_TYPES) const locales = allLocales.filter(pluginConfig.get(`localeFilter`)) reporter.verbose( From 053eb577daa64c3461c5e73d5927118d303a6cd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benedikt=20R=C3=B6tsch?= Date: Fri, 8 Oct 2021 17:53:07 +0200 Subject: [PATCH 18/27] store sync token in plugin status Co-authored-by: Ward Peeters --- packages/gatsby-source-contentful/src/source-nodes.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index ba23a011f6dfb..39eb024666281 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -117,7 +117,7 @@ export async function sourceNodes( * with all data from subsequent syncs. Afterwards the references get * resolved via the Contentful JS SDK. */ - const syncToken = await cache.get(CACHE_SYNC_TOKEN) + const syncToken = store.getState().status.plugins?.[`gatsby-source-contentful`]?.[CACHE_SYNC_TOKEN] // Actual fetch of data from Contentful const { @@ -205,10 +205,10 @@ export async function sourceNodes( // Update syncToken const nextSyncToken = currentSyncData.nextSyncToken - await Promise.all([ - cache.set(CACHE_SYNC_DATA, mergedSyncData), - cache.set(CACHE_SYNC_TOKEN, nextSyncToken), - ]) + await cache.set(CACHE_SYNC_DATA, mergedSyncData) + actions.setPluginStatus({ + [CACHE_SYNC_TOKEN]: nextSyncToken + }) fetchActivity.end() From e6639b85e4f67d86600a2cfa7e4a54adb9e906de Mon Sep 17 00:00:00 2001 From: axe312ger Date: Fri, 8 Oct 2021 18:16:25 +0200 Subject: [PATCH 19/27] lint --- packages/gatsby-source-contentful/src/source-nodes.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index 39eb024666281..b6217eb8a9f95 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -117,7 +117,10 @@ export async function sourceNodes( * with all data from subsequent syncs. Afterwards the references get * resolved via the Contentful JS SDK. */ - const syncToken = store.getState().status.plugins?.[`gatsby-source-contentful`]?.[CACHE_SYNC_TOKEN] + const syncToken = + store.getState().status.plugins?.[`gatsby-source-contentful`]?.[ + CACHE_SYNC_TOKEN + ] // Actual fetch of data from Contentful const { @@ -207,7 +210,7 @@ export async function sourceNodes( await cache.set(CACHE_SYNC_DATA, mergedSyncData) actions.setPluginStatus({ - [CACHE_SYNC_TOKEN]: nextSyncToken + [CACHE_SYNC_TOKEN]: nextSyncToken, }) fetchActivity.end() From 56f026f649b5011d30412e618c16864363c3cd3b Mon Sep 17 00:00:00 2001 From: axe312ger Date: Fri, 8 Oct 2021 18:51:52 +0200 Subject: [PATCH 20/27] fix tests for pluginStatus change --- .../src/__tests__/gatsby-node.js | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index 8a694be620c08..611bb35d76056 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -33,7 +33,7 @@ const createMockCache = () => { } describe(`gatsby-node`, () => { - const actions = { createTypes: jest.fn() } + const actions = { createTypes: jest.fn(), setPluginStatus: jest.fn() } const schema = { buildObjectType: jest.fn() } const store = { getState: jest.fn(() => { @@ -327,19 +327,15 @@ describe(`gatsby-node`, () => { locales ) + expect(store.getState).toHaveBeenCalled() + // Tries to load data from cache - expect(cache.get).toHaveBeenCalledWith( - `contentful-sync-token-testSpaceId-master` - ) expect(cache.get).toHaveBeenCalledWith( `contentful-sync-data-testSpaceId-master` ) expect(cache.get.mock.calls).toMatchInlineSnapshot(` Array [ - Array [ - "contentful-sync-token-testSpaceId-master", - ], Array [ "contentful-content-types-testSpaceId-master", ], @@ -350,10 +346,10 @@ describe(`gatsby-node`, () => { `) // Stores sync token and raw/unparsed data to the cache - expect(cache.set).toHaveBeenCalledWith( - `contentful-sync-token-testSpaceId-master`, - startersBlogFixture.initialSync().currentSyncData.nextSyncToken - ) + expect(actions.setPluginStatus).toHaveBeenCalledWith({ + [`contentful-sync-token-testSpaceId-master`]: + startersBlogFixture.initialSync().currentSyncData.nextSyncToken, + }) // Check for valid cache data const cacheCall = cache.set.mock.calls.filter( @@ -372,7 +368,6 @@ describe(`gatsby-node`, () => { Array [ "contentful-content-types-testSpaceId-master", "contentful-sync-data-testSpaceId-master", - "contentful-sync-token-testSpaceId-master", ] `) }) From 40eb0dfe8bb3539e544c94899a534ab1a04bcbe9 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Sun, 10 Oct 2021 10:37:06 +0200 Subject: [PATCH 21/27] ensure syncProgress does tick and finish --- .../gatsby-source-contentful/src/fetch.js | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/gatsby-source-contentful/src/fetch.js b/packages/gatsby-source-contentful/src/fetch.js index 5e797d1c0eefe..6c8eae5d75743 100644 --- a/packages/gatsby-source-contentful/src/fetch.js +++ b/packages/gatsby-source-contentful/src/fetch.js @@ -241,22 +241,26 @@ async function fetchContent({ syncToken, pluginConfig, reporter }) { } // Fetch entries and assets via Contentful CDA sync API - let syncProgress const pageLimit = pluginConfig.get(`pageLimit`) + reporter.verbose(`Contentful: Sync ${pageLimit} items per page.`) + const syncProgress = reporter.createProgress( + `Contentful: ${syncToken ? `Sync changed items` : `Sync all items`}`, + pageLimit, + 0 + ) + syncProgress.start() + const contentfulSyncClientOptions = createContentfulClientOptions({ + pluginConfig, + reporter, + syncProgress, + }) + const syncClient = contentful.createClient(contentfulSyncClientOptions) let currentSyncData let currentPageLimit = pageLimit let lastCurrentPageLimit let syncSuccess = false try { - syncProgress = reporter.createProgress( - `Contentful: ${syncToken ? `Sync changed items` : `Sync all items`}`, - currentPageLimit, - 0 - ) - syncProgress.start() - reporter.verbose(`Contentful: Sync ${currentPageLimit} items per page.`) - while (!syncSuccess) { try { const basicSyncConfig = { @@ -266,7 +270,7 @@ async function fetchContent({ syncToken, pluginConfig, reporter }) { const query = syncToken ? { nextSyncToken: syncToken, ...basicSyncConfig } : { initial: true, ...basicSyncConfig } - currentSyncData = await client.sync(query) + currentSyncData = await syncClient.sync(query) syncSuccess = true } catch (e) { // Back off page limit if responses content length exceeds Contentfuls limits. From 978ac43bb17d9fdfdc0bc36d6dde4ceb3b7095e7 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Sun, 10 Oct 2021 10:40:58 +0200 Subject: [PATCH 22/27] properly create interfaces --- .../src/create-schema-customization.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-contentful/src/create-schema-customization.js b/packages/gatsby-source-contentful/src/create-schema-customization.js index 68ed72aeaa8cf..d84940966ea11 100644 --- a/packages/gatsby-source-contentful/src/create-schema-customization.js +++ b/packages/gatsby-source-contentful/src/create-schema-customization.js @@ -53,7 +53,7 @@ export async function createSchemaCustomization( await cache.set(CACHE_CONTENT_TYPES, contentTypeItems) createTypes( - schema.buildObjectType({ + schema.buildInterfaceType({ name: `ContentfulEntry`, fields: { contentful_id: { type: `String!` }, @@ -66,7 +66,7 @@ export async function createSchemaCustomization( ) createTypes( - schema.buildObjectType({ + schema.buildInterfaceType({ name: `ContentfulReference`, fields: { contentful_id: { type: `String!` }, From 1c394ef0ba99ac0699f3ad68aa1826c41309f32a Mon Sep 17 00:00:00 2001 From: axe312ger Date: Sun, 10 Oct 2021 10:57:53 +0200 Subject: [PATCH 23/27] ensure syncProgress reports properly when no data was available for sync --- packages/gatsby-source-contentful/src/fetch.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/gatsby-source-contentful/src/fetch.js b/packages/gatsby-source-contentful/src/fetch.js index 6c8eae5d75743..f16d2f3f0d986 100644 --- a/packages/gatsby-source-contentful/src/fetch.js +++ b/packages/gatsby-source-contentful/src/fetch.js @@ -308,6 +308,18 @@ async function fetchContent({ syncToken, pluginConfig, reporter }) { }, }) } finally { + // Fix output when there was no new data in Contentful + if ( + currentSyncData.entries.length + + currentSyncData.assets.length + + currentSyncData.deletedEntries.length + + currentSyncData.deletedAssets.length === + 0 + ) { + syncProgress.tick() + syncProgress.total = 1 + } + syncProgress.done() } From 7978708c88ec4f56fb171f8091e65ecc8bc7cf53 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Sun, 10 Oct 2021 11:08:52 +0200 Subject: [PATCH 24/27] fix unit tests --- packages/gatsby-source-contentful/src/__tests__/fetch.js | 2 ++ .../gatsby-source-contentful/src/__tests__/gatsby-node.js | 2 +- packages/gatsby-source-contentful/src/fetch.js | 8 ++++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/gatsby-source-contentful/src/__tests__/fetch.js b/packages/gatsby-source-contentful/src/__tests__/fetch.js index 6bc03c7300de6..229c1facf018b 100644 --- a/packages/gatsby-source-contentful/src/__tests__/fetch.js +++ b/packages/gatsby-source-contentful/src/__tests__/fetch.js @@ -90,11 +90,13 @@ beforeAll(() => { }) const start = jest.fn() +const tick = jest.fn() const end = jest.fn() const mockActivity = { start, end, + tick, done: end, } diff --git a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js index 611bb35d76056..d01874c6d637e 100644 --- a/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/__tests__/gatsby-node.js @@ -34,7 +34,7 @@ const createMockCache = () => { describe(`gatsby-node`, () => { const actions = { createTypes: jest.fn(), setPluginStatus: jest.fn() } - const schema = { buildObjectType: jest.fn() } + const schema = { buildObjectType: jest.fn(), buildInterfaceType: jest.fn() } const store = { getState: jest.fn(() => { return { program: { directory: process.cwd() }, status: {} } diff --git a/packages/gatsby-source-contentful/src/fetch.js b/packages/gatsby-source-contentful/src/fetch.js index f16d2f3f0d986..fb37c4d5e4673 100644 --- a/packages/gatsby-source-contentful/src/fetch.js +++ b/packages/gatsby-source-contentful/src/fetch.js @@ -310,10 +310,10 @@ async function fetchContent({ syncToken, pluginConfig, reporter }) { } finally { // Fix output when there was no new data in Contentful if ( - currentSyncData.entries.length + - currentSyncData.assets.length + - currentSyncData.deletedEntries.length + - currentSyncData.deletedAssets.length === + currentSyncData?.entries.length + + currentSyncData?.assets.length + + currentSyncData?.deletedEntries.length + + currentSyncData?.deletedAssets.length === 0 ) { syncProgress.tick() From 7ea2a54d43121a1fa3c508da4a2ab3c3e572fb34 Mon Sep 17 00:00:00 2001 From: axe312ger Date: Sun, 10 Oct 2021 11:17:01 +0200 Subject: [PATCH 25/27] style: call createTypes once --- .../src/create-schema-customization.js | 60 +++++++++---------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/packages/gatsby-source-contentful/src/create-schema-customization.js b/packages/gatsby-source-contentful/src/create-schema-customization.js index d84940966ea11..54de929b5f2de 100644 --- a/packages/gatsby-source-contentful/src/create-schema-customization.js +++ b/packages/gatsby-source-contentful/src/create-schema-customization.js @@ -52,7 +52,7 @@ export async function createSchemaCustomization( const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}` await cache.set(CACHE_CONTENT_TYPES, contentTypeItems) - createTypes( + const contentfulTypes = [ schema.buildInterfaceType({ name: `ContentfulEntry`, fields: { @@ -62,10 +62,7 @@ export async function createSchemaCustomization( }, extensions: { dontInfer: {} }, interfaces: [`Node`], - }) - ) - - createTypes( + }), schema.buildInterfaceType({ name: `ContentfulReference`, fields: { @@ -73,10 +70,7 @@ export async function createSchemaCustomization( id: { type: `ID!` }, }, extensions: { dontInfer: {} }, - }) - ) - - createTypes( + }), schema.buildObjectType({ name: `ContentfulAsset`, fields: { @@ -84,34 +78,34 @@ export async function createSchemaCustomization( id: { type: `ID!` }, }, interfaces: [`ContentfulReference`, `Node`], - }) - ) + }), + ] // Create types for each content type - const gqlTypes = contentTypeItems.map(contentTypeItem => - schema.buildObjectType({ - name: _.upperFirst( - _.camelCase( - `Contentful ${ - pluginConfig.get(`useNameForId`) - ? contentTypeItem.name - : contentTypeItem.sys.id - }` - ) - ), - fields: { - contentful_id: { type: `String!` }, - id: { type: `ID!` }, - node_locale: { type: `String!` }, - }, - interfaces: [`ContentfulReference`, `ContentfulEntry`, `Node`], - }) + contentTypeItems.forEach(contentTypeItem => + contentfulTypes.push( + schema.buildObjectType({ + name: _.upperFirst( + _.camelCase( + `Contentful ${ + pluginConfig.get(`useNameForId`) + ? contentTypeItem.name + : contentTypeItem.sys.id + }` + ) + ), + fields: { + contentful_id: { type: `String!` }, + id: { type: `ID!` }, + node_locale: { type: `String!` }, + }, + interfaces: [`ContentfulReference`, `ContentfulEntry`, `Node`], + }) + ) ) - createTypes(gqlTypes) - if (pluginConfig.get(`enableTags`)) { - createTypes( + contentfulTypes.push( schema.buildObjectType({ name: `ContentfulTag`, fields: { @@ -124,4 +118,6 @@ export async function createSchemaCustomization( }) ) } + + createTypes(contentfulTypes) } From f3b91b0840e57c75b3e9ed76388415ace2e93438 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 12 Oct 2021 22:06:57 +0200 Subject: [PATCH 26/27] use cache data in PQR workers --- .../src/create-schema-customization.js | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/packages/gatsby-source-contentful/src/create-schema-customization.js b/packages/gatsby-source-contentful/src/create-schema-customization.js index 54de929b5f2de..3d47845cb9346 100644 --- a/packages/gatsby-source-contentful/src/create-schema-customization.js +++ b/packages/gatsby-source-contentful/src/create-schema-customization.js @@ -5,14 +5,11 @@ const { createPluginConfig } = require(`./plugin-options`) const { fetchContentTypes } = require(`./fetch`) const { CODES } = require(`./report`) -export async function createSchemaCustomization( - { schema, actions, reporter, cache }, - pluginOptions -) { - const { createTypes } = actions - - const pluginConfig = createPluginConfig(pluginOptions) - +async function getContentTypesFromContentFul({ + cache, + reporter, + pluginConfig, +}) { // Get content type items from Contentful const contentTypeItems = await fetchContentTypes({ pluginConfig, reporter }) @@ -52,6 +49,31 @@ export async function createSchemaCustomization( const CACHE_CONTENT_TYPES = `contentful-content-types-${sourceId}` await cache.set(CACHE_CONTENT_TYPES, contentTypeItems) + return contentTypeItems +} + +export async function createSchemaCustomization( + { schema, actions, reporter, cache }, + pluginOptions +) { + const { createTypes } = actions + + const pluginConfig = createPluginConfig(pluginOptions) + + let contentTypeItems + if (process.env.GATSBY_WORKER_ID) { + const sourceId = `${pluginConfig.get(`spaceId`)}-${pluginConfig.get( + `environment` + )}` + contentTypeItems = await cache.get(`contentful-content-types-${sourceId}`) + } else { + contentTypeItems = await getContentTypesFromContentFul({ + cache, + reporter, + pluginConfig, + }) + } + const contentfulTypes = [ schema.buildInterfaceType({ name: `ContentfulEntry`, @@ -60,7 +82,7 @@ export async function createSchemaCustomization( id: { type: `ID!` }, node_locale: { type: `String!` }, }, - extensions: { dontInfer: {} }, + extensions: { infer: false }, interfaces: [`Node`], }), schema.buildInterfaceType({ @@ -69,7 +91,7 @@ export async function createSchemaCustomization( contentful_id: { type: `String!` }, id: { type: `ID!` }, }, - extensions: { dontInfer: {} }, + extensions: { infer: false }, }), schema.buildObjectType({ name: `ContentfulAsset`, From 4f22ccf128289f8112266b5b2362f3e62bdae245 Mon Sep 17 00:00:00 2001 From: Ward Peeters Date: Tue, 12 Oct 2021 22:07:13 +0200 Subject: [PATCH 27/27] fix review exports --- packages/gatsby-source-contentful/src/fetch.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-contentful/src/fetch.js b/packages/gatsby-source-contentful/src/fetch.js index fb37c4d5e4673..b0efa6488f036 100644 --- a/packages/gatsby-source-contentful/src/fetch.js +++ b/packages/gatsby-source-contentful/src/fetch.js @@ -353,7 +353,7 @@ async function fetchContent({ syncToken, pluginConfig, reporter }) { return result } -module.exports.fetchContent = fetchContent +exports.fetchContent = fetchContent /** * Fetches: @@ -399,7 +399,7 @@ async function fetchContentTypes({ pluginConfig, reporter }) { return contentTypes } -module.exports.fetchContentTypes = fetchContentTypes +exports.fetchContentTypes = fetchContentTypes /** * Gets all the existing entities based on pagination parameters.