mirror of
https://github.com/GenderDysphoria/GenderDysphoria.fyi.git
synced 2025-11-26 04:52:43 +00:00
Burn it down and rise from the ashes
This commit is contained in:
Jocelyn Badgley (Twipped)
@@ -1,167 +1,69 @@
|
||||
const path = require('path');
|
||||
const glob = require('../lib/glob');
|
||||
const { chunk, uniq, difference } = require('lodash');
|
||||
const Promise = require('bluebird');
|
||||
const fs = require('fs-extra');
|
||||
const log = require('fancy-log');
|
||||
const tweetparse = require('../lib/tweetparse');
|
||||
const getEngines = require('./renderers');
|
||||
const Twitter = require('twitter-lite');
|
||||
const Page = require('./page');
|
||||
const createAssetLoader = require('./files');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
const createAssetFinder = require('./assets');
|
||||
const Cache = require('./cache');
|
||||
|
||||
exports.parse = async function parsePageContent () {
|
||||
const [ files, twitter, twitterBackup, twitterCache, Assets ] = await Promise.all([
|
||||
glob('pages/**/*.{md,hbs,html,xml}', { cwd: ROOT }),
|
||||
fs.readJson(resolve('twitter-config.json')).catch(() => null)
|
||||
.then(getTwitterClient),
|
||||
fs.readJson(resolve('twitter-backup.json')).catch(() => ({})),
|
||||
fs.readJson(resolve('twitter-cache.json')).catch(() => ({})),
|
||||
createAssetLoader(),
|
||||
]);
|
||||
const evaluate = require('./evaluate');
|
||||
|
||||
const pages = require('./pages');
|
||||
|
||||
let tweetsNeeded = [];
|
||||
const tweetsPresent = Object.keys(twitterCache);
|
||||
const twitter = require('./twitter');
|
||||
const favicon = require('./favicon');
|
||||
const assets = () => createAssetFinder().then(({ tasks }) => tasks);
|
||||
|
||||
let pages = await Promise.map(files, async (filepath) => {
|
||||
const page = new Page(filepath);
|
||||
if (!page.input) return;
|
||||
await page.load({ Assets });
|
||||
exports.everything = function (prod = false) {
|
||||
const fn = async () => {
|
||||
|
||||
if (page.tweets.length) {
|
||||
const missing = difference(page.tweets, tweetsPresent);
|
||||
tweetsNeeded.push(...missing);
|
||||
}
|
||||
const AssetFinder = await createAssetFinder();
|
||||
|
||||
return page;
|
||||
});
|
||||
await pages.parse(AssetFinder);
|
||||
|
||||
pages = pages.filter(Boolean);
|
||||
tweetsNeeded = uniq(tweetsNeeded);
|
||||
|
||||
/* Load Missing Tweets **************************************************/
|
||||
|
||||
if (tweetsNeeded.length) {
|
||||
log('Fetching tweets: ' + tweetsNeeded.join(', '));
|
||||
const arriving = await Promise.all(chunk(tweetsNeeded, 99).map(twitter));
|
||||
|
||||
const loaded = [];
|
||||
for (const tweet of arriving.flat(1)) {
|
||||
if (!twitterBackup[tweet.id_str]) twitterBackup[tweet.id_str] = tweet;
|
||||
twitterCache[tweet.id_str] = tweetparse(tweet);
|
||||
loaded.push(tweet.id_str);
|
||||
}
|
||||
|
||||
const absent = difference(tweetsNeeded, loaded);
|
||||
for (const id of absent) {
|
||||
if (twitterBackup[id]) {
|
||||
log('Pulled tweet from backup ' + id);
|
||||
twitterCache[id] = tweetparse(twitterBackup[id]);
|
||||
continue;
|
||||
}
|
||||
log.error('Could not find tweet ' + id);
|
||||
}
|
||||
}
|
||||
|
||||
/* Apply Tweets to Pages **************************************************/
|
||||
|
||||
const twitterMedia = [];
|
||||
|
||||
// now loop through pages and substitute the tweet data for the ids
|
||||
for (const page of pages) {
|
||||
if (!page.tweets || !page.tweets.length) continue;
|
||||
|
||||
page.tweets = page.tweets.reduce((dict, tweetid) => {
|
||||
const tweet = twitterCache[tweetid];
|
||||
if (!tweet) {
|
||||
log.error(`Tweet ${tweetid} is missing from the cache.`);
|
||||
return dict;
|
||||
}
|
||||
dict[tweetid] = tweet;
|
||||
twitterMedia.push( ...tweet.media );
|
||||
return dict;
|
||||
}, {});
|
||||
|
||||
}
|
||||
|
||||
await Promise.all([
|
||||
fs.writeFile(path.join(ROOT, 'pages.json'), JSON.stringify(pages.map((p) => p.toJson()), null, 2)),
|
||||
fs.writeFile(path.join(ROOT, 'twitter-media.json'), JSON.stringify(twitterMedia, null, 2)),
|
||||
fs.writeFile(path.join(ROOT, 'twitter-cache.json'), JSON.stringify(twitterCache, null, 2)),
|
||||
fs.writeFile(path.join(ROOT, 'twitter-backup.json'), JSON.stringify(twitterBackup, null, 2)),
|
||||
]);
|
||||
|
||||
return pages;
|
||||
};
|
||||
|
||||
exports.write = async function writePageContent ({ prod }) {
|
||||
const [ pages, { siteInfo }, engines ] = await Promise.all([
|
||||
fs.readJson(resolve('pages.json')),
|
||||
fs.readJson(resolve('package.json')),
|
||||
getEngines(prod),
|
||||
]);
|
||||
|
||||
await Promise.map(pages, async (page) => {
|
||||
// page = new Page(page);
|
||||
|
||||
var data = {
|
||||
...page,
|
||||
meta: { ...page.meta, ...page },
|
||||
page: {
|
||||
domain: siteInfo.domain,
|
||||
title: page.meta.title
|
||||
? (page.meta.title + (page.meta.subtitle ? ', ' + page.meta.subtitle : '') + ' :: ' + siteInfo.title)
|
||||
: siteInfo.title,
|
||||
description: page.meta.description || siteInfo.description,
|
||||
},
|
||||
site: siteInfo,
|
||||
local: {
|
||||
cwd: page.cwd,
|
||||
root: ROOT,
|
||||
basename: page.basename,
|
||||
},
|
||||
pages,
|
||||
};
|
||||
|
||||
const html = String(engines[page.engine](data.source, data));
|
||||
const json = page.json && {
|
||||
url: page.fullurl,
|
||||
title: page.meta.title,
|
||||
subtitle: page.meta.subtitle,
|
||||
description: page.meta.description,
|
||||
tweets: page.tweets,
|
||||
images: page.images,
|
||||
dateCreated: page.dateCreated,
|
||||
dateModified: page.dateModified,
|
||||
titlecard: page.titlecard,
|
||||
preview: page.engine === 'md' && String(engines.preview(data.source, data)),
|
||||
};
|
||||
|
||||
await fs.ensureDir(path.dirname(page.output));
|
||||
await Promise.all([
|
||||
fs.writeFile(page.output, Buffer.from(html)),
|
||||
json && fs.writeFile(page.json, Buffer.from(prod ? JSON.stringify(json) : JSON.stringify(json, null, 2))),
|
||||
const tasks = await Promise.all([
|
||||
AssetFinder.tasks,
|
||||
twitter(prod),
|
||||
favicon(prod),
|
||||
]);
|
||||
});
|
||||
|
||||
if (!tasks.length) return;
|
||||
|
||||
const cache = new Cache({ prod });
|
||||
await cache.load();
|
||||
await evaluate(tasks.flat(), cache);
|
||||
await cache.save();
|
||||
|
||||
await pages.write(prod);
|
||||
};
|
||||
|
||||
const ret = () => fn().catch((err) => { console.log(err.trace || err); throw err; });
|
||||
ret.displayName = prod ? 'generateEverythingForProd' : 'generateEverything';
|
||||
return ret;
|
||||
};
|
||||
|
||||
exports.write.prod = function writePageContentForProduction () { return exports.write({ prod: true }); };
|
||||
exports.task = function (action, prod = false) {
|
||||
let fn;
|
||||
|
||||
if (action === 'parse') {
|
||||
fn = () => pages.parse();
|
||||
} else if (action === 'pages') {
|
||||
fn = () => pages.write(prod);
|
||||
} else {
|
||||
fn = async () => {
|
||||
const tasks = await {
|
||||
twitter,
|
||||
favicon,
|
||||
assets,
|
||||
}[action](prod);
|
||||
|
||||
/* Utility Functions **************************************************/
|
||||
if (!tasks.length) return;
|
||||
|
||||
function resolve (fpath, ...args) {
|
||||
if (fpath[0] === '/') fpath = fpath.slice(1);
|
||||
return path.resolve(ROOT, fpath, ...args);
|
||||
}
|
||||
const cache = new Cache({ prod });
|
||||
await cache.load();
|
||||
await evaluate(tasks, cache);
|
||||
await cache.save();
|
||||
};
|
||||
}
|
||||
|
||||
function getTwitterClient (config) {
|
||||
if (!config) return () => [];
|
||||
const client = new Twitter(config);
|
||||
return (tweetids) => client
|
||||
.get('statuses/lookup', { id: tweetids.join(','), tweet_mode: 'extended' })
|
||||
.catch((e) => { log.error(e); return []; });
|
||||
}
|
||||
const ret = () => fn().catch((err) => { console.log(err.trace || err); throw err; });
|
||||
ret.displayName = prod ? action + 'ForProd' : action;
|
||||
|
||||
return ret;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user