Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev 3.0 async #764

Merged
merged 39 commits into from
Jan 5, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
a45b27c
fix(pattern_engines): Support PatternEngine as a local symlink
bmuenzenmeyer Nov 27, 2017
bf8962e
fix(serve): Throttle liveserver startup. Safely call reload
bmuenzenmeyer Dec 5, 2017
90d424d
wip async rendering with data
bmuenzenmeyer Dec 16, 2017
fb5af7e
chore(patternlab.js): Remove debugging
bmuenzenmeyer Dec 22, 2017
6d42598
chore(object_factory.js): Remove debugging
bmuenzenmeyer Dec 22, 2017
558967e
feat(render): Add debugging
bmuenzenmeyer Dec 22, 2017
ef54007
feat(render): Alter info to debug message
bmuenzenmeyer Dec 28, 2017
f633671
feat(package): install all main engines as devDependencies so tests run
bmuenzenmeyer Dec 28, 2017
033f7de
chore(test): Move require to top of file
bmuenzenmeyer Dec 28, 2017
ecedc26
chore(liquid): Target current liquid alpha
bmuenzenmeyer Dec 28, 2017
e2fa1f5
feat(pattern_assembler): Breaking up into smaller files
bmuenzenmeyer Dec 28, 2017
f1b2d83
chore(package): Update devDependencies
bmuenzenmeyer Dec 28, 2017
0d9726a
chore(tests): WIP test files
bmuenzenmeyer Dec 28, 2017
7a37158
fix(render): Defensively set extendedTemplate as a last resort
bmuenzenmeyer Dec 28, 2017
4363f2a
chore(index): add todo
bmuenzenmeyer Dec 28, 2017
175f9c9
feat(tests): start altering tests
bmuenzenmeyer Dec 28, 2017
921b513
feat(pattern_assembler): Break out renderSync
bmuenzenmeyer Dec 28, 2017
d7087fa
feat(object_factory): Make all engines async
bmuenzenmeyer Dec 29, 2017
7d57fe7
feat(pattern_assembler): Further separate files
bmuenzenmeyer Dec 29, 2017
47c9eee
fix(get): Attempt to sanitize verbosePartial into posix format
bmuenzenmeyer Dec 29, 2017
ba1c436
feat(parameter_hunter): Async support
bmuenzenmeyer Dec 29, 2017
1dfa911
fix(test): Fix all tests except what looks like a legitimate failure
bmuenzenmeyer Dec 29, 2017
38b96ca
fix(lint): Resolve lint issues
bmuenzenmeyer Dec 29, 2017
450b586
fix(tests): Fix tests across many files
bmuenzenmeyer Dec 29, 2017
02b3598
fix(tests): fix ui_builder test mocking
bmuenzenmeyer Dec 29, 2017
a759443
fix(processRecursive): Fix async AND! recursive partial inclusion
bmuenzenmeyer Dec 29, 2017
8c43526
fix(test): Address another batch of tests
bmuenzenmeyer Dec 29, 2017
2764a75
feat(pattern_assembler): Further broke out files
bmuenzenmeyer Dec 29, 2017
f0f61cb
fix(tests): Fix broken import
bmuenzenmeyer Dec 29, 2017
e0b9143
fix(test): Fix test mock
bmuenzenmeyer Dec 29, 2017
ea69bdc
fix(list_item_hunter): Add async support and fix tests
bmuenzenmeyer Dec 29, 2017
774b827
feat(meta patterns): Async support
bmuenzenmeyer Jan 3, 2018
31d584b
fix(package): Remove two unneeded dependencies
bmuenzenmeyer Jan 3, 2018
9830da2
chore(error_handling): Remove unneeded parens
bmuenzenmeyer Jan 5, 2018
dec04fe
feat(ui_builder): Async support
bmuenzenmeyer Jan 5, 2018
7a14c09
chore(ui_builder): Fix lint
bmuenzenmeyer Jan 5, 2018
32a13e1
feat(ui_builder): Broke out buildFooter
bmuenzenmeyer Jan 5, 2018
cf0aaa5
fix(unit tests): Fix tests
bmuenzenmeyer Jan 5, 2018
3a83116
Merge remote-tracking branch 'origin/dev-3.0-async' into dev-3.0-async
bmuenzenmeyer Jan 5, 2018
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
183 changes: 103 additions & 80 deletions core/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,13 @@ const updateNotifier = require('update-notifier');
const logger = require('./lib/log');
const PatternGraph = require('./lib/pattern_graph').PatternGraph;
const CompileState = require('./lib/object_factory').CompileState;
const pa = require('./lib/pattern_assembler');
const pe = require('./lib/pattern_exporter');
const lh = require('./lib/lineage_hunter');
const markModifiedPatterns = require('./lib/markModifiedPatterns');
const parseAllLinks = require('./lib/parseAllLinks');
const processMetaPattern = require('./lib/processMetaPattern');
const render = require('./lib/render');
const Pattern = require('./lib/object_factory').Pattern;

const defaultConfig = require('../patternlab-config.json');

Expand All @@ -30,7 +34,6 @@ let assetCopier = require('./lib/asset_copy'); // eslint-disable-line
let pattern_exporter = new pe(); // eslint-disable-line
let serve = require('./lib/serve'); // eslint-disable-line

const pattern_assembler = new pa();
const lineage_hunter = new lh();

//bootstrap update notifier
Expand All @@ -39,7 +42,6 @@ updateNotifier({
updateCheckInterval: 1000 * 60 * 60 * 24 // notify at most once a day
}).notify();


/**
* Returns the standardized default config
*
Expand Down Expand Up @@ -136,7 +138,7 @@ const patternlab_module = function (config) {

function cleanBuildDirectory(incrementalBuildsEnabled) {
if (incrementalBuildsEnabled) {
logger.log.info("Incremental builds enabled.");
logger.info("Incremental builds enabled.");
} else {
// needs to be done BEFORE processing patterns
fs.removeSync(paths.public.patterns);
Expand All @@ -153,7 +155,7 @@ const patternlab_module = function (config) {
const graph = patternlab.graph = loadPatternGraph(deletePatternDir);
const graphNeedsUpgrade = !PatternGraph.checkVersion(graph);
if (graphNeedsUpgrade) {
logger.log.info("Due to an upgrade, a complete rebuild is required and the public/patterns directory was deleted. " +
logger.info("Due to an upgrade, a complete rebuild is required and the public/patterns directory was deleted. " +
"Incremental build is available again on the next successful run.");

// Ensure that the freshly built graph has the latest version again.
Expand All @@ -177,79 +179,97 @@ const patternlab_module = function (config) {

//now that all the main patterns are known, look for any links that might be within data and expand them
//we need to do this before expanding patterns & partials into extendedTemplates, otherwise we could lose the data -> partial reference
pattern_assembler.parse_data_links(patternlab);
parseAllLinks(patternlab);

//diveSync again to recursively include partials, filling out the
//extendedTemplate property of the patternlab.patterns elements
// TODO we can reduce the time needed by only processing changed patterns and their partials
patternlab.processAllPatternsRecursive(paths.source.patterns, patternlab);

//take the user defined head and foot and process any data and patterns that apply
// GTP: should these really be invoked from outside?
patternlab.processHeadPattern();
patternlab.processFootPattern();

//cascade any patternStates
lineage_hunter.cascade_pattern_states(patternlab);

//set pattern-specific header if necessary
let head;
if (patternlab.userHead) {
head = patternlab.userHead;
} else {
head = patternlab.header;
}
const headPatternPromise = processMetaPattern(`_00-head.${patternlab.config.patternExtension}`, 'userHead', patternlab);
const footPatternPromise = processMetaPattern(`_01-foot.${patternlab.config.patternExtension}`, 'userFoot', patternlab);

//set the pattern-specific header by compiling the general-header with data, and then adding it to the meta header
patternlab.data.patternLabHead = pattern_assembler.renderPattern(patternlab.header, {
cacheBuster: patternlab.cacheBuster
});
return Promise.all([headPatternPromise, footPatternPromise]).then(() => {

// If deletePatternDir == true or graph needs to be updated
// rebuild all patterns
let patternsToBuild = null;
//cascade any patternStates
lineage_hunter.cascade_pattern_states(patternlab);

// If deletePatternDir == true or graph needs to be updated
// rebuild all patterns
patternsToBuild = null;

if (patternlab.incrementalBuildsEnabled) {
// When the graph was loaded from file, some patterns might have been moved/deleted between runs
// so the graph data become out of sync
patternlab.graph.sync().forEach(n => {
logger.log.info("[Deleted/Moved] " + n);
});

// TODO Find created or deleted files
const now = new Date().getTime();
pattern_assembler.mark_modified_patterns(now, patternlab);
patternsToBuild = patternlab.graph.compileOrder();
} else {
// build all patterns, mark all to be rebuilt
patternsToBuild = patternlab.patterns;
for (const p of patternsToBuild) {
p.compileState = CompileState.NEEDS_REBUILD;
//set pattern-specific header if necessary
let head;
if (patternlab.userHead) {
head = patternlab.userHead;
} else {
head = patternlab.header;
}
}

//render all patterns last, so lineageR works
return patternsToBuild
.reduce((previousPromise, pattern) => {
return previousPromise.then(() => patternlab.renderSinglePattern(pattern, head));
}, Promise.resolve())
.then(() => {
// Saves the pattern graph when all files have been compiled
PatternGraph.storeToFile(patternlab);
if (patternlab.config.exportToGraphViz) {
PatternGraph.exportToDot(patternlab, "dependencyGraph.dot");
logger.log.info(`Exported pattern graph to ${path.join(config.paths.public.root, "dependencyGraph.dot")}`);
//set the pattern-specific header by compiling the general-header with data, and then adding it to the meta header
return render(Pattern.createEmpty({extendedTemplate: patternlab.header}), {
cacheBuster: patternlab.cacheBuster
}).then((results) => {
patternlab.data.patternLabHead = results;

// If deletePatternDir == true or graph needs to be updated
// rebuild all patterns
let patternsToBuild = null;

// If deletePatternDir == true or graph needs to be updated
// rebuild all patterns
patternsToBuild = null;

if (patternlab.incrementalBuildsEnabled) {
// When the graph was loaded from file, some patterns might have been moved/deleted between runs
// so the graph data become out of sync
patternlab.graph.sync().forEach(n => {
logger.info("[Deleted/Moved] " + n);
});

// TODO Find created or deleted files
const now = new Date().getTime();
markModifiedPatterns(now, patternlab);
patternsToBuild = patternlab.graph.compileOrder();
} else {
// build all patterns, mark all to be rebuilt
patternsToBuild = patternlab.patterns;
for (const p of patternsToBuild) {
p.compileState = CompileState.NEEDS_REBUILD;
}
}

//export patterns if necessary
pattern_exporter.export_patterns(patternlab);
//render all patterns last, so lineageR works
return patternsToBuild
.reduce((previousPromise, pattern) => {
return previousPromise.then(() => patternlab.renderSinglePattern(pattern, head));
}, Promise.resolve())
.then(() => {
// Saves the pattern graph when all files have been compiled
PatternGraph.storeToFile(patternlab);
if (patternlab.config.exportToGraphViz) {
PatternGraph.exportToDot(patternlab, "dependencyGraph.dot");
logger.info(`Exported pattern graph to ${path.join(config.paths.public.root, "dependencyGraph.dot")}`);
}

//export patterns if necessary
pattern_exporter.export_patterns(patternlab);

}).catch(reason => {
console.log(reason);
logger.error('Error rendering patterns');
});

}).catch(reason => {
console.log(reason);
logger.error('Error rendering pattern lab header');
});
}).catch((err) => {
logger.info('Error in buildPatterns():', err);

}).catch(reason => {
console.log(reason);
logger.error('Error processing meta patterns');
});

}).catch(reason => {
console.log(reason);
logger.error('Error in buildPatterns()');
});
}

Expand Down Expand Up @@ -286,26 +306,28 @@ const patternlab_module = function (config) {
patternlab.isBusy = true;
return buildPatterns(options.cleanPublic).then(() => {

new ui_builder().buildFrontend(patternlab);
assetCopier().copyAssets(patternlab.config.paths, patternlab, options);
return new ui_builder().buildFrontend(patternlab).then(() => {

this.events.on('patternlab-pattern-change', () => {
if (!patternlab.isBusy) {
options.cleanPublic = false;
return this.build(options);
}
return Promise.resolve();
});
assetCopier().copyAssets(patternlab.config.paths, patternlab, options);

this.events.on('patternlab-global-change', () => {
if (!patternlab.isBusy) {
options.cleanPublic = true; //rebuild everything
return this.build(options);
}
return Promise.resolve();
});
this.events.on('patternlab-pattern-change', () => {
if (!patternlab.isBusy) {
options.cleanPublic = false;
return this.build(options);
}
return Promise.resolve();
});

patternlab.isBusy = false;
this.events.on('patternlab-global-change', () => {
if (!patternlab.isBusy) {
options.cleanPublic = true; //rebuild everything
return this.build(options);
}
return Promise.resolve();
});

patternlab.isBusy = false;
});
});
},

Expand Down Expand Up @@ -384,6 +406,7 @@ const patternlab_module = function (config) {
options.watch = true;
return this.build(options).then(function () {
serve(patternlab);
return Promise.resolve();
});
},

Expand Down
46 changes: 46 additions & 0 deletions core/lib/addPattern.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
"use strict";

const _ = require('lodash');

const logger = require('./log');

module.exports = function (pattern, patternlab) {
//add the link to the global object
if (!patternlab.data.link) {
patternlab.data.link = {};
}
patternlab.data.link[pattern.patternPartial] = '/patterns/' + pattern.patternLink;

//only push to array if the array doesn't contain this pattern
var isNew = true;
for (var i = 0; i < patternlab.patterns.length; i++) {
//so we need the identifier to be unique, which patterns[i].relPath is
if (pattern.relPath === patternlab.patterns[i].relPath) {
//if relPath already exists, overwrite that element
patternlab.patterns[i] = pattern;
patternlab.partials[pattern.patternPartial] = pattern.extendedTemplate || pattern.template;
isNew = false;
break;
}
}

// if the pattern is new, we must register it with various data structures!
if (isNew) {

logger.debug(`found new pattern ${pattern.patternPartial}`);

// do global registration
if (pattern.isPattern) {
patternlab.partials[pattern.patternPartial] = pattern.extendedTemplate || pattern.template;

// do plugin-specific registration
pattern.registerPartial();
} else {
patternlab.partials[pattern.patternPartial] = pattern.patternDesc;
}

//patterns sorted by name so the patterntype and patternsubtype is adhered to for menu building
patternlab.patterns.splice(_.sortedIndexBy(patternlab.patterns, pattern, 'name'), 0, pattern);
patternlab.graph.add(pattern);
}
};
39 changes: 39 additions & 0 deletions core/lib/buildFooter.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
"use strict";

const jsonCopy = require('./json_copy');
const logger = require('./log');
const of = require('./object_factory');
const Pattern = of.Pattern;

let render = require('./render'); //eslint-disable-line prefer-const

/**
* Builds footer HTML from the general footer and user-defined footer
* @param patternlab - global data store
* @param patternPartial - the partial key to build this for, either viewall-patternPartial or a viewall-patternType-all
* @returns A promise which resolves with the HTML
*/
module.exports = function (patternlab, patternPartial) {
//first render the general footer
return render(Pattern.createEmpty({extendedTemplate: patternlab.footer}), {
patternData: JSON.stringify({
patternPartial: patternPartial,
}),
cacheBuster: patternlab.cacheBuster
}).then(footerPartial => {

let allFooterData;
try {
allFooterData = jsonCopy(patternlab.data, 'config.paths.source.data plus patterns data');
} catch (err) {
logger.warning('There was an error parsing JSON for patternlab.data');
logger.warning(err);
}
allFooterData.patternLabFoot = footerPartial;

return render(Pattern.createEmpty({extendedTemplate: patternlab.userFoot}), allFooterData);
}).catch(reason => {
console.log(reason);
logger.error('Error building buildFooterHTML');
});
};
27 changes: 27 additions & 0 deletions core/lib/buildListItems.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
"use strict";

const _ = require('lodash');

module.exports = function (container) {
//combine all list items into one structure
var list = [];
for (var item in container.listitems) {
if (container.listitems.hasOwnProperty(item)) {
list.push(container.listitems[item]);
}
}
container.listItemArray = _.shuffle(list);

for (var i = 1; i <= container.listItemArray.length; i++) {
var tempItems = [];
if (i === 1) {
tempItems.push(container.listItemArray[0]);
container.listitems['' + i ] = tempItems;
} else {
for (var c = 1; c <= i; c++) {
tempItems.push(container.listItemArray[c - 1]);
container.listitems['' + i ] = tempItems;
}
}
}
};
Loading