github.com/GenderDysphoria.fyi
1
0
Fork 1
mirror of https://github.com/GenderDysphoria/GenderDysphoria.fyi.git synced 2025-11-25 20:42:40 +00:00
Code Activity

More overhaul, image tasks and content assets now come from the same code.

Browse Source
This commit is contained in:
Jocelyn Badgley (Twipped)
2020-02-23 18:36:33 -08:00
parent 24dab66898
commit 9c483c9dd7
7 changed files with 509 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files

310
gulp/imgflow/index.js
View File

@@ -1,130 +1,47 @@
const path = require('path');
const glob = require('../lib/glob');
const { sortBy, uniqBy } = require('lodash');
const { uniqBy } = require('lodash');
const Promise = require('bluebird');
const fs = require('fs-extra');
const log = require('fancy-log');
const actions = require('./actions');
const getDimensions = require('../lib/dimensions');
const createAssetLoader = require('../content/files');
const CWD = path.resolve(__dirname, '../..');
const PAGES = path.join(CWD, 'pages');
const SOURCE = path.resolve(CWD, 'pages/**/*.{jpeg,jpg,png,gif,mp4}');
const MANIFEST_PATH = path.resolve(CWD, 'if-manifest.json');
const REV_MANIFEST_PATH = path.resolve(CWD, 'rev-manifest.json');
const MEDIA_INDEX = path.resolve(CWD, 'twitter-media.json');
const ROOT = path.resolve(__dirname, '../..');
const CACHE = 'if-cache';
const revHash = require('rev-hash');
const revPath = require('rev-path');
const { changed, execute } = require('./pipeline');
const LOG = {
new: true,
update: true,
skip: true,
rebuild: true,
cached: false,
copy: false,
};
function resolve (...args) {
args = args.filter(Boolean);
let fpath = args.shift();
if (!fpath) return ROOT;
if (fpath[0] === '/') fpath = fpath.slice(1);
return path.resolve(ROOT, fpath, ...args);
}
module.exports = exports = async function postImages ({ rev = false }) {
var manifest;
try {
manifest = JSON.parse(await fs.readFile(MANIFEST_PATH));
} catch (e) {
manifest = {};
}
const [ manifest, { tasks } ] = await Promise.all([
fs.readJson(resolve('if-manifest.json')).catch(() => ({})),
createAssetLoader(),
fs.ensureDir(resolve(CACHE)),
]);
await fs.ensureDir(path.resolve(CWD, CACHE));
const allfiles = (await glob(SOURCE));
const tasks = [];
for (const filepath of allfiles) {
const input = path.relative(CWD, filepath);
const output = path.relative(PAGES, filepath).replace('/_images', '');
const file = path.parse(output);
// console.log(input, output);
// is a titlecard image or a video
if (file.name === 'titlecard' || file.ext === '.mp4') {
tasks.push({
input,
output,
action: actions.copy,
});
continue;
}
// is a file we've pre-sized and do not want processed
if (file.name[0] === '_') {
tasks.push({
input,
output: path.format({ ...file, base: file.base.substring(1) }),
action: actions.copy,
});
continue;
}
const format = {
'.jpeg': 'jpeg',
'.jpg': 'jpeg',
'.png': 'png',
'.gif': 'gif',
}[file.ext];
if (!format) throw new Error('Got an unexpected format: ' + file.ext);
const dimensions = await getDimensions(filepath);
tasks.push({
input: filepath,
output: `${file.dir}/${file.name}.${format}`,
format,
action: actions.image,
});
for (const w of [ 2048, 1024, 768, 576, 300, 100 ]) {
if (w > dimensions.width) continue;
tasks.push({
input: filepath,
output: `${file.dir}/${file.name}.${w}w.${format}`,
format,
width: w,
action: actions.image,
});
}
}
const filtered = await filter(manifest, tasks);
const filtered = await changed(manifest, tasks);
await execute(manifest, filtered, rev);
};
exports.prod = function imagesProd () { return exports({ rev: true }); };
exports.twitter = async function twitterImages ({ rev = false }) {
await fs.ensureDir(path.resolve(CWD, CACHE));
const [ manifest, media ] = await Promise.all([
fs.readJson(resolve('if-manifest.json')).catch(() => ({})),
fs.readJson(resolve('twitter-media.json')).catch(() => ([])),
fs.ensureDir(resolve(CACHE)),
]);
var manifest;
try {
manifest = JSON.parse(await fs.readFile(MANIFEST_PATH));
} catch (e) {
manifest = {};
}
var media;
try {
media = JSON.parse(await fs.readFile(MEDIA_INDEX));
} catch (e) {
media = [];
}
media = uniqBy(media, 'output');
const tasks = media.map((m) => ({ ...m, action: actions.fetch }));
const filtered = await filter(manifest, tasks);
const tasks = uniqBy(media, 'output').map((m) => ({ ...m, action: actions.fetch }));
const filtered = await changed(manifest, tasks);
await execute(manifest, filtered, rev);
};
@@ -132,16 +49,11 @@ exports.twitter.prod = function imagesProd () { return exports.twitter({ rev: tr
exports.favicon = async function favicon ({ rev = false }) {
await fs.ensureDir(path.resolve(CWD, CACHE));
const input = path.resolve(CWD, 'favicon.png');
var manifest;
try {
manifest = JSON.parse(await fs.readFile(MANIFEST_PATH));
} catch (e) {
manifest = {};
}
const input = resolve('favicon.png');
const [ manifest ] = await Promise.all([
fs.readJson(resolve('if-manifest.json')).catch(() => ({})),
fs.ensureDir(resolve(CACHE)),
]);
const tasks = [ 32, 57, 64, 76, 96, 114, 120, 128, 144, 152, 180, 192, 196, 228 ].map((width) => ({
input,
@@ -158,169 +70,9 @@ exports.favicon = async function favicon ({ rev = false }) {
action: actions.image,
});
const filtered = await filter(manifest, tasks);
const filtered = await changed(manifest, tasks);
await execute(manifest, filtered, rev);
};
exports.favicon.prod = function imagesProd () { return exports.favicon({ rev: true }); };
async function filter (manifest, tasks) {
const statMap = new Map();
async function stat (f) {
if (statMap.has(f)) return statMap.get(f);
const p = fs.stat(path.resolve(CWD, f))
.catch(() => null)
.then((stats) => (stats && Math.floor(stats.mtimeMs / 1000)));
statMap.set(f, p);
return p;
}
return Promise.filter(tasks, async (task) => {
const local = task.input.slice(0, 4) !== 'http';
const hash = task.action.name + '.' + revHash(task.input) + '|' + revHash(task.output);
const cachePath = path.join(CACHE, `${hash}${path.extname(task.output)}`);
const [ inTime, outTime, cachedTime ] = await Promise.all([
local && stat(path.resolve(CWD, task.input)),
stat(path.resolve(CWD, 'dist', task.output)),
stat(path.resolve(CWD, cachePath)),
]);
task.manifest = manifest[hash];
task.hash = hash;
task.cache = cachePath;
// how did this happen?
if (local && !inTime) {
log.error('Input file could not be found?', task.input);
return false;
}
// never seen this file before
if (!task.manifest) {
task.apply = {
hash,
input: task.input,
output: task.output,
mtime: inTime,
};
task.log = [ 'new', task.input, task.output, hash ];
return true;
}
// file modification time does not match last read, rebuild
if (local && inTime > task.manifest.mtime) {
task.log = [ 'update', task.input, task.output ];
task.apply = {
mtime: inTime,
};
return true;
}
task.apply = {
mtime: local ? inTime : Math.floor(Date.now() / 1000),
};
// target file exists, nothing to do
if (outTime) {
return false;
// task.log = [ 'skip', task.input, task.output, inTime, task.manifest.mtime ];
// task.action = null;
// return true;
}
// file exists in the cache, change the task to a copy action
if (cachedTime) {
task.log = [ 'cached', task.input, task.output ];
task.action = actions.copy;
task.input = cachePath;
return true;
}
// task is a file copy
if (task.action === actions.copy) {
task.log = [ 'copy', task.input, task.output ];
return true;
}
// file does not exist in cache, build it
task.log = [ 'rebuild', task.input, task.output ];
return true;
});
}
async function execute (manifest, tasks, rev) {
const lastSeen = Math.floor(Date.now() / 1000);
const revManifest = {};
let writeCounter = 0;
let lastWriteTime = 0;
async function writeManifest (force) {
if (!force && rev) return; // disable interim writes during prod builds.
if (!force && ++writeCounter % 100) return;
const now = Date.now();
if (!force && now - lastWriteTime < 10000) return;
lastWriteTime = now;
await fs.writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2));
}
await Promise.map(sortBy(tasks, [ 'input', 'output' ]), async (task) => {
const output = path.resolve(CWD, 'dist', task.output);
const result = task.action && await task.action({ ...task, output });
const apply = task.apply || {};
if (task.log && LOG[task.log[0]]) log.info(...task.log);
apply.lastSeen = lastSeen;
apply.lastSeenHuman = new Date();
if (!result) log('Nothing happened?', task);
const rhash = result && revHash(result);
const hashedPath = revPath(task.output, rhash);
apply.revHash = rhash;
apply.revPath = hashedPath;
if (rev && rhash) {
const rOutPath = task.output;
const rNewPath = hashedPath;
revManifest[rOutPath] = rNewPath;
await fs.copy(output, path.resolve(CWD, 'dist', hashedPath));
}
manifest[task.hash] = { ...manifest[task.hash], ...apply };
await writeManifest();
}, { concurrency: rev ? 20 : 10 });
// filter unseen files from history
// manifest = omitBy(manifest, (m) => m.lastSeen !== lastSeen);
await writeManifest(true);
if (rev) {
let originalManifest = {};
try {
if (await fs.exists(REV_MANIFEST_PATH)) {
originalManifest = JSON.parse(await fs.readFile(REV_MANIFEST_PATH));
}
} catch (e) {
// do nothing
}
Object.assign(originalManifest, revManifest);
await fs.writeFile(REV_MANIFEST_PATH, JSON.stringify(originalManifest, null, 2));
}
}
if (require.main === module) {
exports().catch(console.error).then(() => process.exit()); // eslint-disable-line
}

179
gulp/imgflow/pipeline.js Normal file
View File

@@ -0,0 +1,179 @@
const path = require('path');
const { sortBy } = require('lodash');
const Promise = require('bluebird');
const fs = require('fs-extra');
const log = require('fancy-log');
const actions = require('./actions');
const revHash = require('rev-hash');
const revPath = require('rev-path');
const CWD = path.resolve(__dirname, '../..');
const PAGES = path.join(CWD, 'pages');
const SOURCE = path.resolve(PAGES, '**/*.{jpeg,jpg,png,gif,mp4}');
const MANIFEST_PATH = path.resolve(CWD, 'if-manifest.json');
const REV_MANIFEST_PATH = path.resolve(CWD, 'rev-manifest.json');
const MEDIA_INDEX = path.resolve(CWD, 'twitter-media.json');
const CACHE = 'if-cache';
const LOG = {
new: true,
update: true,
skip: true,
rebuild: true,
cached: false,
copy: false,
};
exports.changed = async function changed (manifest, tasks) {
const statMap = new Map();
async function stat (f) {
if (statMap.has(f)) return statMap.get(f);
const p = fs.stat(path.resolve(CWD, f))
.catch(() => null)
.then((stats) => (stats && Math.floor(stats.mtimeMs / 1000)));
statMap.set(f, p);
return p;
}
return Promise.filter(tasks, async (task) => {
const local = task.input.slice(0, 4) !== 'http';
const hash = task.action.name + '.' + revHash(task.input) + '|' + revHash(task.output);
const cachePath = path.join(CACHE, `${hash}${path.extname(task.output)}`);
const [ inTime, outTime, cachedTime ] = await Promise.all([
local && stat(path.resolve(CWD, task.input)),
stat(path.resolve(CWD, 'dist', task.output)),
stat(path.resolve(CWD, cachePath)),
]);
task.manifest = manifest[hash];
task.hash = hash;
task.cache = cachePath;
// how did this happen?
if (local && !inTime) {
log.error('Input file could not be found?', task.input);
return false;
}
// never seen this file before
if (!task.manifest) {
task.apply = {
hash,
input: task.input,
output: task.output,
mtime: inTime,
};
task.log = [ 'new', task.input, task.output, hash ];
return true;
}
// file modification time does not match last read, rebuild
if (local && inTime > task.manifest.mtime) {
task.log = [ 'update', task.input, task.output ];
task.apply = {
mtime: inTime,
};
return true;
}
task.apply = {
mtime: local ? inTime : Math.floor(Date.now() / 1000),
};
// target file exists, nothing to do
if (outTime) {
return false;
// task.log = [ 'skip', task.input, task.output, inTime, task.manifest.mtime ];
// task.action = null;
// return true;
}
// file exists in the cache, change the task to a copy action
if (cachedTime) {
task.log = [ 'cached', task.input, task.output ];
task.action = actions.copy;
task.input = cachePath;
return true;
}
// task is a file copy
if (task.action === actions.copy) {
task.log = [ 'copy', task.input, task.output ];
return true;
}
// file does not exist in cache, build it
task.log = [ 'rebuild', task.input, task.output ];
return true;
});
};
exports.execute = async function execute (manifest, tasks, rev) {
const lastSeen = Math.floor(Date.now() / 1000);
const revManifest = {};
let writeCounter = 0;
let lastWriteTime = 0;
async function writeManifest (force) {
if (!force && rev) return; // disable interim writes during prod builds.
if (!force && ++writeCounter % 100) return;
const now = Date.now();
if (!force && now - lastWriteTime < 10000) return;
lastWriteTime = now;
await fs.writeFile(MANIFEST_PATH, JSON.stringify(manifest, null, 2));
}
await Promise.map(sortBy(tasks, [ 'input', 'output' ]), async (task) => {
const output = path.resolve(CWD, 'dist', task.output);
const result = task.action && await task.action({ ...task, output });
const apply = task.apply || {};
if (task.log && LOG[task.log[0]]) log.info(...task.log);
apply.lastSeen = lastSeen;
apply.lastSeenHuman = new Date();
if (!result) log('Nothing happened?', task);
const rhash = result && revHash(result);
const hashedPath = revPath(task.output, rhash);
apply.revHash = rhash;
apply.revPath = hashedPath;
if (rev && rhash) {
const rOutPath = task.output;
const rNewPath = hashedPath;
revManifest[rOutPath] = rNewPath;
await fs.copy(output, path.resolve(CWD, 'dist', hashedPath));
}
manifest[task.hash] = { ...manifest[task.hash], ...apply };
await writeManifest();
}, { concurrency: rev ? 20 : 10 });
// filter unseen files from history
// manifest = omitBy(manifest, (m) => m.lastSeen !== lastSeen);
await writeManifest(true);
if (rev) {
let originalManifest = {};
try {
if (await fs.exists(REV_MANIFEST_PATH)) {
originalManifest = JSON.parse(await fs.readFile(REV_MANIFEST_PATH));
}
} catch (e) {
// do nothing
}
Object.assign(originalManifest, revManifest);
await fs.writeFile(REV_MANIFEST_PATH, JSON.stringify(originalManifest, null, 2));
}
};