Initial site commit

gulp/lib/crass.js

@@ -0,0 +1,31 @@
+
+const through = require('./through');
+const crass = require('crass');
+const PluginError = require('plugin-error');
+
+module.exports = exports = function (options) {
+  options = {
+    pretty: false,
+    o1: true,
+    ...options,
+  };
+
+  return through(async (stream, file) => {
+    if (file.isNull()) {
+      stream.push(file);
+      return;
+    }
+
+    try {
+      var parsed = crass.parse(file.contents.toString());
+      parsed = parsed.optimize({ O1: !!options.o1 });
+      if (options.pretty) parsed = parsed.pretty();
+
+      file.contents = Buffer.from(parsed.toString());
+    } catch (err) {
+      this.emit('error', new PluginError('gulp-crass', err));
+    }
+
+    stream.push(file);
+  });
+};

gulp/lib/debug.js

@@ -0,0 +1,32 @@
+
+const through = require('./through');
+const log = require('fancy-log');
+const { get } = require('lodash');
+
+module.exports = exports = function debug (...targets) {
+  return through(async (stream, file) => {
+    var data;
+    const { path, relative, base, basename, extname } = file;
+
+    if (targets.length === 1 && Array.isArray(targets[0])) {
+      targets = targets[0];
+    }
+
+    if (targets.length) {
+      data = targets.reduce((result, target) => {
+        if (target === 'contents') {
+          result.contents = file.contents.toString();
+          return result;
+        }
+
+        result[target] = get(file, target);
+        return result;
+      }, {});
+    } else {
+      data = { ...file, path, relative, base, basename, extname };
+    }
+    log(data);
+    stream.push(file);
+  });
+};
+

gulp/lib/dimensions.js

@@ -0,0 +1,3 @@
+
+var { promisify } = require('util');
+module.exports = exports = promisify(require('image-size'));

gulp/lib/filter.js

@@ -0,0 +1,18 @@
+
+const filter = require('gulp-filter');
+
+module.exports = exports = function filter2 (pattern, options) {
+  if (pattern instanceof RegExp) {
+    return filter((file) => pattern.test(file.path), options);
+  }
+
+  return filter(pattern, options);
+};
+
+exports.not = function notfilter2 (pattern, options) {
+  if (pattern instanceof RegExp) {
+    return filter((file) => !pattern.test(file.path), options);
+  }
+
+  throw new Error('filter.not only takes regular expressions');
+};

gulp/lib/glob.js

@@ -0,0 +1,4 @@
+
+var { promisify } = require('util');
+const glob = require('glob');
+module.exports = exports = promisify(glob);

gulp/lib/load.js

@@ -0,0 +1,26 @@
+
+const through = require('./through');
+const fs = require('fs-extra');
+const log = require('fancy-log');
+const parallelize = require('concurrent-transform');
+
+module.exports = exports = function load () {
+  return parallelize(through(async (stream, file) => {
+
+    if (file.contents) {
+      // file already has contents, ignore
+      stream.push(file);
+      return;
+    }
+
+    const exists = await fs.pathExists(file.path);
+    // if (!exists) return;
+
+    log('[loading]', file.path, exists);
+
+    file.contents = await fs.readFile(file.path);
+
+    stream.push(file);
+  }), 20);
+};
+

gulp/lib/markdown-raw-html.js

@@ -0,0 +1,181 @@
+
+module.exports = exports = function (md, options) {
+
+  options = {
+    fence: '|||',
+    ...options,
+  };
+
+  function debug (...args) {
+    if (options.debug) console.log(...args); // eslint-disable-line
+  }
+
+  const fenceLen = options.fence.length;
+  // const fenceFirst = options.fence.charCodeAt(0);
+
+  function scanAhead (state, line, pos) {
+    const position = state.src.indexOf(options.fence, pos);
+    if (position === -1) {
+      // there are no html blocks in this entire file
+      state.discreteHtmlScan = {
+        present: false,
+      };
+      return false;
+    }
+
+    while (position > state.eMarks[line]) {
+      line++;
+    }
+
+    state.discreteHtmlScan = {
+      present: true,
+      position,
+      line,
+    };
+
+    return true;
+  }
+
+  md.block.ruler.before('fence', 'raw', (state, startLine, lastLine) => {
+    let pos = state.bMarks[startLine] + state.tShift[startLine];
+    let endOfLine = state.eMarks[startLine];
+
+    // if we have yet to do a scan of this file, perform one.
+    if (!state.discreteHtmlScan && !scanAhead(state, startLine, pos)) {
+      debug('First scan, nothing found');
+      return false;
+    }
+
+    if (!state.discreteHtmlScan.present) {
+      debug('Have scanned, did not find');
+      return false;
+    }
+
+    // add one to the line here in case there is a line break in a paragraph.
+    if (state.discreteHtmlScan.line > startLine + 1) {
+      debug('Have scanned, found, but after this line', { startLine, targetLine: state.discreteHtmlScan.line });
+      return false;
+    }
+
+    if (startLine > state.discreteHtmlScan.line) {
+      // we dun fucked up
+      debug('We somehow got ahead of ourselves', { startLine, line: state.discreteHtmlScan.line, lastLine, pos, endOfLine, tokens: state.tokens });
+      throw new Error('markdown-it-discrete-html encountered a parsing error.');
+    }
+
+    // at this point we should be on a line that contains a fence mark
+    debug({ l: 67, startLine, scan: state.discreteHtmlScan });
+
+    let openIndex, closer, nextLine;
+    openIndex = state.discreteHtmlScan.position;
+    do {
+      let token, closeIndex;
+      const tokens = [];
+      const preBlock = openIndex > pos && state.src.slice(pos, openIndex);
+      debug({ l: 75, preBlock, startLine, lastLine });
+      openIndex += fenceLen;
+      pos = openIndex;
+
+      if (preBlock && !!preBlock.trim()) {
+        md.block.parse(preBlock, md, state.env, tokens);
+
+        switch (tokens[tokens.length - 1].type) {
+        case 'heading_close':
+        case 'paragraph_close':
+          closer = tokens.pop();
+          // fallthrough
+        default:
+          state.tokens.push(...tokens);
+        }
+      }
+
+      debug({ l: 92, tokens });
+
+      // find terminating fence
+      if (!scanAhead(state, startLine, pos)) {
+        debug({ l: 96, remaining: state.src.slice(pos) });
+        // console.error(state.src)
+        throw new Error(`Could not find terminating "${options.fence}" for a raw html block.`);
+      }
+
+      closeIndex = state.discreteHtmlScan.position;
+      nextLine = state.discreteHtmlScan.line;
+
+      if (nextLine === startLine) nextLine++;
+      endOfLine = state.eMarks[nextLine];
+
+      const content = state.src.substring(openIndex, closeIndex);
+      closeIndex += fenceLen;
+      pos = closeIndex;
+
+      if (content.trim()) {
+        token = state.push(closer ? 'html_inline' : 'html_block', '', 0);
+        token.map     = [ startLine, nextLine ];
+        token.content = content;
+        token.block = true;
+        debug({ l: 115, tokens: [ token ], nextLine, pos, endOfLine: state.eMarks[nextLine], len: state.src.length, remaining: state.src.slice(pos) }); // eslint-disable-line
+      }
+
+      if (pos === endOfLine) {
+        // we have ended this line, nothing more to do here.
+        if (closer) {
+          state.tokens.push(closer);
+          debug({ l: 122, tokens: [ closer ] });
+        }
+        state.discreteHtmlScan = null;
+        state.line = nextLine + 1;
+        return true;
+      }
+
+      // still more left in this line, see if there is another block
+      if (scanAhead(state, nextLine, pos)) {
+        // we found another block, but it isn't on this line, so break out.
+        if (state.discreteHtmlScan.line > nextLine) {
+          if (closer) {
+            state.tokens.push(closer);
+            debug({ l: 135, tokens: [ closer ] });
+          }
+          state.line = nextLine + 1;
+          return true;
+        }
+
+        // next block is on this line, grab everything between here and there
+        openIndex = state.discreteHtmlScan.position;
+      } else {
+        // no more blocks on this line, grab everything between here and the end of the line
+        openIndex = endOfLine;
+      }
+      debug({ l: 147, pos, openIndex, remaining: state.src.slice(pos) });
+
+      const postBlock = state.src.slice(pos, openIndex);
+      token = null;
+      if (postBlock.trim()) {
+        token          = state.push('inline', '', 0);
+        token.content  = postBlock;
+        token.map      = [ nextLine, nextLine ];
+        token.children = [];
+        tokens.push(token);
+      }
+      debug({ l: 158, tokens: [ token ], postBlock, pos, openIndex, closeIndex, endOfLine });
+
+      pos = openIndex;
+      startLine = nextLine + 1;
+      endOfLine = state.eMarks[startLine];
+
+      debug({ l: 164, pos, startLine, endOfLine, remaining: state.src.slice(pos) });
+    } while (pos + fenceLen < endOfLine);
+
+    if (closer) {
+      state.tokens.push(closer);
+      debug({ l: 169, tokens: [ closer ] });
+    }
+
+    openIndex += fenceLen;
+    pos = openIndex;
+
+    state.line = startLine;
+    return true;
+  });
+
+};
+

gulp/lib/markdown-token-filter.js

@@ -0,0 +1,46 @@
+
+const { flatten } = require('lodash');
+
+module.exports = exports = function (md) {
+  md.core.ruler.push(
+    'modify-token',
+    (state) => {
+      state.tokens = flatten(state.tokens.map(descend).filter(Boolean));
+      return false;
+    },
+  );
+};
+
+function descend (token) {
+
+  switch (token.type) {
+  case 'link_open':
+  case 'link_close':
+  case 'html_block':
+    return false;
+
+  case 'heading_open':
+    token.type = 'paragraph_open';
+    token.tag = 'p';
+    token.markup = '';
+    return token;
+
+  case 'heading_close':
+    token.type = 'paragraph_close';
+    token.tag = 'p';
+    token.markup = '';
+    return token;
+
+  case 'image':
+  case 'container':
+    return token.children;
+
+  default:
+
+    if (token.children && token.children.length) {
+      token.children = flatten(token.children.map(descend).filter(Boolean));
+    }
+
+    return token;
+  }
+}

gulp/lib/random.js

@@ -0,0 +1,47 @@
+'use strict';
+
+var crypto = require('crypto');
+var uuid = require('uuid').v4;
+
+// based on code from http://stackoverflow.com/a/25690754/110189
+function randomString (length, chars) {
+  if (!chars) {
+    throw new Error('Argument \'chars\' is undefined');
+  }
+
+  var charsLength = chars.length;
+  if (charsLength > 256) {
+    throw new Error('Length must be less than 256 characters');
+  }
+
+  var randomBytes = crypto.randomBytes(length);
+
+  var result = new Array(length);
+
+  var cursor = 0;
+  for (var i = 0; i < length; i++) {
+    cursor += randomBytes[i];
+    result[i] = chars[cursor % charsLength];
+  }
+
+  return result.join('');
+}
+
+module.exports = exports = function (min, max) {
+  if (Array.isArray(min)) return exports.from(min);
+  if (typeof max === 'undefined') {
+    if (min > 0) {
+      max = min;
+      min = 0;
+    } else {
+      max = 0;
+    }
+  }
+  return Math.floor((Math.random() * (max - min + 1)) + min);
+};
+
+exports.alphanumeric = (length) => randomString(length, 'ABCDEFGHIJKLMNOPQRSTUWXYZ0123456789');
+exports.alpha = (length) => randomString(length, 'ABCDEFGHIJKLMNOPQRSTUWXYZ');
+exports.fromCharSet = randomString;
+exports.from = (array) => array[exports(array.length - 1)];
+exports.id = (length) => uuid().replace(/-/g, '').substr(0, length);

gulp/lib/sort.js

@@ -0,0 +1,27 @@
+const through = require('./through');
+const sortBy = require('lodash/sortBy');
+
+function sleep (ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+module.exports = exports = function (iteratees) {
+  var files = [];
+
+  return through(
+    async (stream, file) => {
+      if (file.isNull()) return;
+
+      files.push(file);
+    },
+    async (stream) => {
+      const queue = sortBy(files, iteratees);
+      files = null;
+
+      for (const file of queue) {
+        stream.push(file);
+        await sleep(100);
+      }
+    }
+  );
+};

gulp/lib/through.js

@@ -0,0 +1,19 @@
+
+const log = require('fancy-log');
+var through = require('through2');
+
+module.exports = exports = function asyncthrough (...args) {
+  const [ fn, donefn ] = args;
+
+  args[0] = function (file, enc, next) {
+    fn(this, file, enc).then(() => next(), (err) => { log.error(err, 'Error thrown'); next(err); });
+  };
+
+  if (donefn) {
+    args[1] = function (next) {
+      donefn(this).then(() => next(), (err) => { log.error(err, 'Error thrown'); next(err); });
+    };
+  }
+
+  return through.obj(...args);
+};

gulp/lib/tweetparse.js

@@ -0,0 +1,140 @@
+var twemoji = require('twemoji' );
+const { deepPick, has } = require('./util');
+
+const schema = {
+  id_str: true,
+  created_at: true,
+  user: {
+    screen_name: true,
+    avatar: true,
+    name_html: true,
+    verified: true,
+    protected: true,
+  },
+  html: true,
+  quoted_status: {
+    user: {
+      screen_name: true,
+      avatar: true,
+      name_html: true,
+      verified: true,
+      protected: true,
+    },
+  },
+  entities: { media: [ {
+    type: true,
+    media_url_https: true,
+    video_info: { variants: [ {
+      url: true,
+      content_type: true,
+    } ] },
+  } ] },
+  media: true,
+};
+
+var entityProcessors = {
+  hashtags (tags, tweet) {
+    tags.forEach((tagObj) => {
+      tweet.html = tweet.html.replace('#' + tagObj.text, `<a href="https://twitter.com/hashtag/{tagObj.text}" class="hashtag">#${tagObj.text}</a>`);
+    });
+  },
+
+  symbols (/* symbols, tweet */) {
+
+  },
+
+  user_mentions (users, tweet) {
+    users.forEach((userObj) => {
+      var regex = new RegExp('@' + userObj.screen_name, 'gi' );
+      tweet.html = tweet.html.replace(regex, `<a href="https://twitter.com/${userObj.screen_name}" class="mention">@${userObj.screen_name}</a>`);
+    });
+  },
+
+  urls (urls, tweet) {
+    urls.forEach((urlObj) => {
+      var quotedTweetHtml = '';
+      var indices = urlObj.indices;
+      var urlToReplace = (tweet.full_text || tweet.text).substring(indices[0], indices[1]);
+
+      var finalText = quotedTweetHtml || urlObj.display_url.link(urlObj.expanded_url);
+      tweet.html = tweet.html.replace(urlToReplace, finalText);
+    });
+  },
+
+  media (media, tweet) {
+    media.forEach((mediaObj) => {
+      tweet.html = tweet.html.replace(mediaObj.url, '');
+      return;
+
+      // if (mediaObj.type === 'photo') {
+      //   // Use HTTPS if available
+      //   var src = mediaObj.media_url_https ? mediaObj.media_url_https : mediaObj.media_url;
+
+      //   if (options &&
+      //     options.photoSize &&
+      //     mediaObj.sizes &&
+      //     mediaObj.sizes[options.photoSize]) {
+      //     // If specified size is available, patch image src to use it
+      //     src = src + ':' + options.photoSize;
+      //   }
+
+      //   tweet.html = tweet.html.replace(mediaObj.url, `<img src="${src}" alt=""/>`);
+      // } else if (mediaObj.type === 'video') {
+      //   var source = '';
+      //   mediaObj.video_info.variants.forEach((info) => {
+      //     source += `<source src="${info.url}" type="${info.content_type}">`;
+      //   });
+      //   var video = `<video controls poster="${mediaObj.media_url}">${source}</video>`;
+      //   tweet.html = tweet.html.replace(mediaObj.url, video);
+      // }
+    });
+  },
+};
+
+module.exports = exports = function (tweets) {
+  return Array.isArray(tweets) ? tweets.map(parseTweet) : parseTweet(tweets);
+
+  function parseTweet (tweet) {
+    // clone the tweet so we're not altering the original
+    tweet = JSON.parse(JSON.stringify(tweet));
+
+    tweet.user.avatar = {
+      input: tweet.user.profile_image_url_https,
+      output: 'tweets/' + tweet.user.screen_name + '.jpg',
+    };
+
+    tweet.media = [
+      tweet.user.avatar,
+    ];
+
+    // Copying text value to a new property html. The final output will be set to this property
+    tweet.html = (tweet.full_text || tweet.text)
+      .split(/(\r\n|\n\r|\r|\n)+/)
+      .map((s) => s.trim() && '<p>' + s + '</p>')
+      .filter(Boolean)
+      .join('');
+
+    if (tweet.quoted_status) {
+      tweet.quoted_status = parseTweet(tweet.quoted_status);
+    }
+
+    if (has(tweet, 'entities.media') && has(tweet, 'extended_entities.media')) {
+      tweet.entities.media = tweet.extended_entities.media;
+      delete tweet.extended_entities;
+    }
+
+    // Process entities
+    if (Object.getOwnPropertyNames(tweet.entities).length) {
+      for (let [ entityType, entity ] of Object.entries(tweet.entities)) { // eslint-disable-line prefer-const
+        entityProcessors[entityType](entity, tweet);
+      }
+    }
+
+    // Process Emoji's
+    tweet.html = twemoji.parse(tweet.html);
+    tweet.user.name_html = twemoji.parse(tweet.user.name);
+
+    return deepPick(tweet, schema);
+  }
+
+};