Merge branch '26371-native-emojis-v3-code' into '26371-native-emojis-v3'

Native Unicode Emojis

See merge request !9570

app/assets/javascripts/behaviors/gl_emoji.js

+const installCustomElements = require('document-register-element');
+const emojiMap = require('emoji-map');
+const emojiAliases = require('emoji-aliases');
+const generatedUnicodeSupportMap = require('./gl_emoji/unicode_support_map');
+const spreadString = require('./gl_emoji/spread_string');
+
+installCustomElements(window);
+
+function emojiImageTag(name, src) {
+  return `<img class="emoji" title=":${name}:" alt=":${name}:" src="${src}" width="20" height="20" align="absmiddle" />`;
+}
+
+function assembleFallbackImageSrc(inputName) {
+  const name = Object.prototype.hasOwnProperty.call(emojiAliases, inputName) ?
+    emojiAliases[inputName] : inputName;
+  const emojiInfo = emojiMap[name];
+  const fallbackImageSrc = `${gon.asset_host || ''}${gon.relative_url_root || ''}/assets/emoji/${name}-${emojiInfo.digest}.png`;
+
+  return fallbackImageSrc;
+}
+const glEmojiTagDefaults = {
+  sprite: false,
+  forceFallback: false,
+};
+function glEmojiTag(inputName, options) {
+  const opts = Object.assign({}, glEmojiTagDefaults, options);
+  const name = Object.prototype.hasOwnProperty.call(emojiAliases, inputName) ?
+    emojiAliases[inputName] : inputName;
+  const emojiInfo = emojiMap[name];
+  const fallbackImageSrc = assembleFallbackImageSrc(name);
+  const fallbackSpriteClass = `emoji-${name}`;
+
+  const classList = [];
+  if (opts.forceFallback && opts.sprite) {
+    classList.push('emoji-icon');
+    classList.push(fallbackSpriteClass);
+  }
+  const classAttribute = classList.length > 0 ? `class="${classList.join(' ')}"` : '';
+  const fallbackSpriteAttribute = opts.sprite ? `data-fallback-sprite-class="${fallbackSpriteClass}"` : '';
+  let contents = emojiInfo.moji;
+  if (opts.forceFallback && !opts.sprite) {
+    contents = emojiImageTag(name, fallbackImageSrc);
+  }
+
+  return `
+  <gl-emoji
+    ${classAttribute}
+    data-name="${name}"
+    data-fallback-src="${fallbackImageSrc}"
+    ${fallbackSpriteAttribute}
+    data-unicode-version="${emojiInfo.unicodeVersion}"
+  >
+    ${contents}
+  </gl-emoji>
+  `;
+}
+
+// On Windows, flags render as two-letter country codes, see http://emojipedia.org/flags/
+const flagACodePoint = 127462; // parseInt('1F1E6', 16)
+const flagZCodePoint = 127487; // parseInt('1F1FF', 16)
+function isFlagEmoji(emojiUnicode) {
+  const cp = emojiUnicode.codePointAt(0);
+  // Length 4 because flags are made of 2 characters which are surrogate pairs
+  return emojiUnicode.length === 4 && cp >= flagACodePoint && cp <= flagZCodePoint;
+}
+
+// Chrome <57 renders keycaps oddly
+// See https://bugs.chromium.org/p/chromium/issues/detail?id=632294
+// Same issue on Windows also fixed in Chrome 57, http://i.imgur.com/rQF7woO.png
+function isKeycapEmoji(emojiUnicode) {
+  return emojiUnicode.length === 3 && emojiUnicode[2] === '\u20E3';
+}
+
+// Check for a skin tone variation emoji which aren't always supported
+const tone1 = 127995;// parseInt('1F3FB', 16)
+const tone5 = 127999;// parseInt('1F3FF', 16)
+function isSkinToneComboEmoji(emojiUnicode) {
+  return emojiUnicode.length > 2 && spreadString(emojiUnicode).some((char) => {
+    const cp = char.codePointAt(0);
+    return cp >= tone1 && cp <= tone5;
+  });
+}
+
+// macOS supports most skin tone emoji's but
+// doesn't support the skin tone versions of horse racing
+const horseRacingCodePoint = 127943;// parseInt('1F3C7', 16)
+function isHorceRacingSkinToneComboEmoji(emojiUnicode) {
+  return spreadString(emojiUnicode)[0].codePointAt(0) === horseRacingCodePoint &&
+    isSkinToneComboEmoji(emojiUnicode);
+}
+
+// Check for `family_*`, `kiss_*`, `couple_*`
+// For ex. Windows 8.1 Firefox 51.0.1, doesn't support these
+const zwj = 8205; // parseInt('200D', 16)
+const personStartCodePoint = 128102; // parseInt('1F466', 16)
+const personEndCodePoint = 128105; // parseInt('1F469', 16)
+function isPersonZwjEmoji(emojiUnicode) {
+  let hasPersonEmoji = false;
+  let hasZwj = false;
+  spreadString(emojiUnicode).forEach((character) => {
+    const cp = character.codePointAt(0);
+    if (cp === zwj) {
+      hasZwj = true;
+    } else if (cp >= personStartCodePoint && cp <= personEndCodePoint) {
+      hasPersonEmoji = true;
+    }
+  });
+
+  return hasPersonEmoji && hasZwj;
+}
+
+// Helper so we don't have to run `isFlagEmoji` twice
+// in `isEmojiUnicodeSupported` logic
+function checkFlagEmojiSupport(unicodeSupportMap, emojiUnicode) {
+  const isFlagResult = isFlagEmoji(emojiUnicode);
+  return (
+    (unicodeSupportMap.flag && isFlagResult) ||
+    !isFlagResult
+  );
+}
+
+// Helper so we don't have to run `isSkinToneComboEmoji` twice
+// in `isEmojiUnicodeSupported` logic
+function checkSkinToneModifierSupport(unicodeSupportMap, emojiUnicode) {
+  const isSkinToneResult = isSkinToneComboEmoji(emojiUnicode);
+  return (
+    (unicodeSupportMap.skinToneModifier && isSkinToneResult) ||
+    !isSkinToneResult
+  );
+}
+
+// Helper func so we don't have to run `isHorceRacingSkinToneComboEmoji` twice
+// in `isEmojiUnicodeSupported` logic
+function checkHorseRacingSkinToneComboEmojiSupport(unicodeSupportMap, emojiUnicode) {
+  const isHorseRacingSkinToneResult = isHorceRacingSkinToneComboEmoji(emojiUnicode);
+  return (
+    (unicodeSupportMap.horseRacing && isHorseRacingSkinToneResult) ||
+    !isHorseRacingSkinToneResult
+  );
+}
+
+// Helper so we don't have to run `isPersonZwjEmoji` twice
+// in `isEmojiUnicodeSupported` logic
+function checkPersonEmojiSupport(unicodeSupportMap, emojiUnicode) {
+  const isPersonZwjResult = isPersonZwjEmoji(emojiUnicode);
+  return (
+    (unicodeSupportMap.personZwj && isPersonZwjResult) ||
+    !isPersonZwjResult
+  );
+}
+
+// Takes in a support map and determines whether
+// the given unicode emoji is supported on the platform.
+//
+// Combines all the edge case tests into a one-stop shop method
+function isEmojiUnicodeSupported(unicodeSupportMap = {}, emojiUnicode, unicodeVersion) {
+  const isOlderThanChrome57 = unicodeSupportMap.meta && unicodeSupportMap.meta.isChrome &&
+    unicodeSupportMap.meta.chromeVersion < 57;
+
+  // For comments about each scenario, see the comments above each individual respective function
+  return unicodeSupportMap[unicodeVersion] &&
+    !(isOlderThanChrome57 && isKeycapEmoji(emojiUnicode)) &&
+    checkFlagEmojiSupport(unicodeSupportMap, emojiUnicode) &&
+    checkSkinToneModifierSupport(unicodeSupportMap, emojiUnicode) &&
+    checkHorseRacingSkinToneComboEmojiSupport(unicodeSupportMap, emojiUnicode) &&
+    checkPersonEmojiSupport(unicodeSupportMap, emojiUnicode);
+}
+
+const GlEmojiElementProto = Object.create(HTMLElement.prototype);
+GlEmojiElementProto.createdCallback = function createdCallback() {
+  const emojiUnicode = this.textContent.trim();
+  const {
+    name,
+    unicodeVersion,
+    fallbackSrc,
+    fallbackSpriteClass,
+  } = this.dataset;
+
+  const isEmojiUnicode = this.childNodes && Array.prototype.every.call(
+    this.childNodes,
+    childNode => childNode.nodeType === 3,
+  );
+  const hasImageFallback = fallbackSrc && fallbackSrc.length > 0;
+  const hasCssSpriteFalback = fallbackSpriteClass && fallbackSpriteClass.length > 0;
+
+  if (
+    isEmojiUnicode &&
+    !isEmojiUnicodeSupported(generatedUnicodeSupportMap, emojiUnicode, unicodeVersion)
+  ) {
+    // CSS sprite fallback takes precedence over image fallback
+    if (hasCssSpriteFalback) {
+      // IE 11 doesn't like adding multiple at once :(
+      this.classList.add('emoji-icon');
+      this.classList.add(fallbackSpriteClass);
+    } else if (hasImageFallback) {
+      this.innerHTML = emojiImageTag(name, fallbackSrc);
+    } else {
+      const src = assembleFallbackImageSrc(name);
+      this.innerHTML = emojiImageTag(name, src);
+    }
+  }
+};
+
+document.registerElement('gl-emoji', {
+  prototype: GlEmojiElementProto,
+});
+
+module.exports = {
+  emojiImageTag,
+  glEmojiTag,
+  isEmojiUnicodeSupported,
+  isFlagEmoji,
+  isKeycapEmoji,
+  isSkinToneComboEmoji,
+  isHorceRacingSkinToneComboEmoji,
+  isPersonZwjEmoji,
+};

app/assets/javascripts/behaviors/gl_emoji/spread_string.js

+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/charCodeAt#Fixing_charCodeAt()_to_handle_non-Basic-Multilingual-Plane_characters_if_their_presence_earlier_in_the_string_is_known
+function knownCharCodeAt(givenString, index) {
+  const str = `${givenString}`;
+  const end = str.length;
+
+  const surrogatePairs = /[\uD800-\uDBFF][\uDC00-\uDFFF]/g;
+  let idx = index;
+  while ((surrogatePairs.exec(str)) != null) {
+    const li = surrogatePairs.lastIndex;
+    if (li - 2 < idx) {
+      idx += 1;
+    } else {
+      break;
+    }
+  }
+
+  if (idx >= end || idx < 0) {
+    return NaN;
+  }
+
+  const code = str.charCodeAt(idx);
+
+  let high;
+  let low;
+  if (code >= 0xD800 && code <= 0xDBFF) {
+    high = code;
+    low = str.charCodeAt(idx + 1);
+    // Go one further, since one of the "characters" is part of a surrogate pair
+    return ((high - 0xD800) * 0x400) + (low - 0xDC00) + 0x10000;
+  }
+  return code;
+}
+
+// See http://stackoverflow.com/a/38901550/796832
+// ES5/PhantomJS compatible version of spreading a string
+//
+// [...'foo'] -> ['f', 'o', 'o']
+// [...'🖐🏿'] -> ['🖐', '🏿']
+function spreadString(str) {
+  const arr = [];
+  let i = 0;
+  while (!isNaN(knownCharCodeAt(str, i))) {
+    const codePoint = knownCharCodeAt(str, i);
+    arr.push(String.fromCodePoint(codePoint));
+    i += 1;
+  }
+  return arr;
+}
+
+module.exports = spreadString;

app/assets/javascripts/behaviors/gl_emoji/unicode_support_map.js

+const unicodeSupportTestMap = {
+  // man, student (emojione does not have any of these yet), http://emojipedia.org/emoji-zwj-sequences/
+  // occupationZwj: '\u{1F468}\u{200D}\u{1F393}',
+  // woman, biking (emojione does not have any of these yet), http://emojipedia.org/emoji-zwj-sequences/
+  // sexZwj: '\u{1F6B4}\u{200D}\u{2640}',
+  // family_mwgb
+  // Windows 8.1, Firefox 51.0.1 does not support `family_`, `kiss_`, `couple_`
+  personZwj: '\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}\u{200D}\u{1F466}',
+  // horse_racing_tone5
+  // Special case that is not supported on macOS 10.12 even though `skinToneModifier` succeeds
+  horseRacing: '\u{1F3C7}\u{1F3FF}',
+  // US flag, http://emojipedia.org/flags/
+  flag: '\u{1F1FA}\u{1F1F8}',
+  // http://emojipedia.org/modifiers/
+  skinToneModifier: [
+    // spy_tone5
+    '\u{1F575}\u{1F3FF}',
+    // person_with_ball_tone5
+    '\u{26F9}\u{1F3FF}',
+    // angel_tone5
+    '\u{1F47C}\u{1F3FF}',
+  ],
+  // rofl, http://emojipedia.org/unicode-9.0/
+  '9.0': '\u{1F923}',
+  // metal, http://emojipedia.org/unicode-8.0/
+  '8.0': '\u{1F918}',
+  // spy, http://emojipedia.org/unicode-7.0/
+  '7.0': '\u{1F575}',
+  // expressionless, http://emojipedia.org/unicode-6.1/
+  6.1: '\u{1F611}',
+  // japanese_goblin, http://emojipedia.org/unicode-6.0/
+  '6.0': '\u{1F47A}',
+  // sailboat, http://emojipedia.org/unicode-5.2/
+  5.2: '\u{26F5}',
+  // mahjong, http://emojipedia.org/unicode-5.1/
+  5.1: '\u{1F004}',
+  // gear, http://emojipedia.org/unicode-4.1/
+  4.1: '\u{2699}',
+  // zap, http://emojipedia.org/unicode-4.0/
+  '4.0': '\u{26A1}',
+  // recycle, http://emojipedia.org/unicode-3.2/
+  3.2: '\u{267B}',
+  // information_source, http://emojipedia.org/unicode-3.0/
+  '3.0': '\u{2139}',
+  // heart, http://emojipedia.org/unicode-1.1/
+  1.1: '\u{2764}',
+};
+
+function checkPixelInImageDataArray(pixelOffset, imageDataArray) {
+  // `4 *` because RGBA
+  const indexOffset = 4 * pixelOffset;
+  const hasColor = imageDataArray[indexOffset + 0] ||
+    imageDataArray[indexOffset + 1] ||
+    imageDataArray[indexOffset + 2];
+  const isVisible = imageDataArray[indexOffset + 3];
+  // Check for some sort of color other than black
+  if (hasColor && isVisible) {
+    return true;
+  }
+  return false;
+}
+
+const chromeMatches = navigator.userAgent.match(/Chrom(?:e|ium)\/([0-9]+)\./);
+const isChrome = chromeMatches && chromeMatches.length > 0;
+const chromeVersion = chromeMatches && chromeMatches[1] && parseInt(chromeMatches[1], 10);
+
+// We use 16px because mobile Safari (iOS 9.3) doesn't properly scale emojis :/
+// See 32px, https://i.imgur.com/htY6Zym.png
+// See 16px, https://i.imgur.com/FPPsIF8.png
+const fontSize = 16;
+function testUnicodeSupportMap(testMap) {
+  const testMapKeys = Object.keys(testMap);
+  const numTestEntries = testMapKeys
+    .reduce((list, testKey) => list.concat(testMap[testKey]), []).length;
+
+  const canvas = document.createElement('canvas');
+  (window.gl || window).testEmojiUnicodeSupportMapCanvas = canvas;
+  const ctx = canvas.getContext('2d');
+  canvas.width = (2 * fontSize);
+  canvas.height = (numTestEntries * fontSize);
+  ctx.fillStyle = '#000000';
+  ctx.textBaseline = 'middle';
+  ctx.font = `${fontSize}px "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"`;
+  // Write each emoji to the canvas vertically
+  let writeIndex = 0;
+  testMapKeys.forEach((testKey) => {
+    const testEntry = testMap[testKey];
+    [].concat(testEntry).forEach((emojiUnicode) => {
+      ctx.fillText(emojiUnicode, 0, (writeIndex * fontSize) + (fontSize / 2));
+      writeIndex += 1;
+    });
+  });
+
+  // Read from the canvas
+  const resultMap = {};
+  let readIndex = 0;
+  testMapKeys.forEach((testKey) => {
+    const testEntry = testMap[testKey];
+    // This needs to be a `reduce` instead of `every` because we need to
+    // keep the `readIndex` in sync from the writes by running all entries
+    const isTestSatisfied = [].concat(testEntry).reduce((isSatisfied) => {
+      // Sample along the vertical-middle for a couple of characters
+      const imageData = ctx.getImageData(
+          0,
+          (readIndex * fontSize) + (fontSize / 2),
+          2 * fontSize,
+          1,
+        ).data;
+
+      let isValidEmoji = false;
+      for (let currentPixel = 0; currentPixel < 64; currentPixel += 1) {
+        const isLookingAtFirstChar = currentPixel < fontSize;
+        const isLookingAtSecondChar = currentPixel >= (fontSize + (fontSize / 2));
+        // Check for the emoji somewhere along the row
+        if (isLookingAtFirstChar && checkPixelInImageDataArray(currentPixel, imageData)) {
+          isValidEmoji = true;
+
+        // Check to see that nothing is rendered next to the first character
+        // to ensure that the ZWJ sequence rendered as one piece
+        } else if (isLookingAtSecondChar && checkPixelInImageDataArray(currentPixel, imageData)) {
+          isValidEmoji = false;
+          break;
+        }
+      }
+
+      readIndex += 1;
+      return isSatisfied && isValidEmoji;
+    }, true);
+
+    resultMap[testKey] = isTestSatisfied;
+  });
+
+  resultMap.meta = {
+    isChrome,
+    chromeVersion,
+  };
+
+  return resultMap;
+}
+
+let unicodeSupportMap;
+const userAgentFromCache = window.localStorage.getItem('gl-emoji-user-agent');
+try {
+  unicodeSupportMap = JSON.parse(window.localStorage.getItem('gl-emoji-unicode-support-map'));
+} catch (err) {
+  // swallow
+}
+if (!unicodeSupportMap || userAgentFromCache !== navigator.userAgent) {
+  unicodeSupportMap = testUnicodeSupportMap(unicodeSupportTestMap);
+  window.localStorage.setItem('gl-emoji-user-agent', navigator.userAgent);
+  window.localStorage.setItem('gl-emoji-unicode-support-map', JSON.stringify(unicodeSupportMap));
+}
+
+module.exports = unicodeSupportMap;

app/assets/javascripts/copy_as_gfm.js

@@ -49,6 +49,9 @@ require('./lib/utils/common_utils');
       'img.emoji'(el, text) {
         return el.getAttribute('alt');
       },
+      'gl-emoji'(el, text) {
+        return `:${el.getAttribute('data-name')}:`;
+      },
     },
     ImageLinkFilter: {
       'a.no-attachment-icon'(el, text) {

app/assets/javascripts/extensions/string.js

+require('string.prototype.codepointat');
+require('string.fromcodepoint');

app/assets/javascripts/gfm_auto_complete.js

 /* eslint-disable func-names, space-before-function-paren, no-template-curly-in-string, comma-dangle, object-shorthand, quotes, dot-notation, no-else-return, one-var, no-var, no-underscore-dangle, one-var-declaration-per-line, no-param-reassign, no-useless-escape, prefer-template, consistent-return, wrap-iife, prefer-arrow-callback, camelcase, no-unused-vars, no-useless-return, vars-on-top, max-len */
 
+const emojiMap = require('emoji-map');
+const emojiAliases = require('emoji-aliases');
+const glEmoji = require('./behaviors/gl_emoji');
+
+const glEmojiTag = glEmoji.glEmojiTag;
+
 // Creates the variables for setting up GFM auto-completion
 (function() {
   if (window.gl == null) {
@@ -26,7 +32,12 @@
     },
     // Emoji
     Emoji: {
-      template: '<li>${name} <img alt="${name}" height="20" src="${path}" width="20" /></li>'
+      templateFunction: function(name) {
+        return `<li>
+          ${name} ${glEmojiTag(name)}
+        </li>
+        `;
+      }
     },
     // Team Members
     Members: {
@@ -113,7 +124,7 @@
       $input.atwho({
         at: ':',
         displayTpl: function(value) {
-          return value.path != null ? this.Emoji.template : this.Loading.template;
+          return value && value.name ? this.Emoji.templateFunction(value.name) : this.Loading.template;
         }.bind(this),
         insertTpl: ':${name}:',
         skipSpecialCharacterTest: true,
@@ -355,6 +366,8 @@
       this.isLoadingData[at] = true;
       if (this.cachedData[at]) {
         this.loadData($input, at, this.cachedData[at]);
+      } else if (this.atTypeMap[at] === 'emojis') {
+        this.loadData($input, at, Object.keys(emojiMap).concat(Object.keys(emojiAliases)));
       } else {
         $.getJSON(this.dataSources[this.atTypeMap[at]], (data) => {
           this.loadData($input, at, data);

app/assets/javascripts/main.js

@@ -3,7 +3,6 @@
 /* global Cookies */
 /* global Flash */
 /* global ConfirmDangerModal */
-/* global AwardsHandler */
 /* global Aside */
 
 import jQuery from 'jquery';
@@ -19,6 +18,15 @@ require('mousetrap/plugins/pause/mousetrap-pause');
 require('vendor/fuzzaldrin-plus');
 require('es6-promise').polyfill();
 
+// extensions
+require('./extensions/string');
+require('./extensions/array');
+require('./extensions/custom_event');
+require('./extensions/element');
+require('./extensions/jquery');
+require('./extensions/object');
+require('es6-promise').polyfill();
+
 // expose common libraries as globals (TODO: remove these)
 window.jQuery = jQuery;
 window.$ = jQuery;
@@ -61,13 +69,6 @@ require('./templates/issuable_template_selectors');
 require('./commit/file.js');
 require('./commit/image_file.js');
 
-// extensions
-require('./extensions/array');
-require('./extensions/custom_event');
-require('./extensions/element');
-require('./extensions/jquery');
-require('./extensions/object');
-
 // lib/utils
 require('./lib/utils/animate');
 require('./lib/utils/bootstrap_linked_tabs');
@@ -99,7 +100,7 @@ require('./ajax_loading_spinner');
 require('./api');
 require('./aside');
 require('./autosave');
-require('./awards_handler');
+const AwardsHandler = require('./awards_handler');
 require('./breakpoints');
 require('./broadcast_message');
 require('./build');

app/assets/stylesheets/framework.scss

@@ -44,5 +44,6 @@
 @import "framework/images.scss";
 @import "framework/broadcast-messages";
 @import "framework/emojis.scss";
+@import "framework/emoji-sprites.scss";
 @import "framework/icons.scss";
 @import "framework/snippets.scss";

app/assets/stylesheets/framework/awards.scss

@@ -7,6 +7,7 @@
 
 .emoji-menu {
   position: absolute;
+  top: 0;
   margin-top: 3px;
   padding: $gl-padding;
   z-index: 9;
@@ -20,7 +21,7 @@
   opacity: 0;
   transform: scale(.2);
   transform-origin: 0 -45px;
-  transition: .3s cubic-bezier(.87,-.41,.19,1.44);
+  transition: .3s cubic-bezier(.67,.06,.19,1.44);
   transition-property: transform, opacity;
 
   &.is-aligned-right {
@@ -47,12 +48,13 @@
 }
 
 .emoji-menu-list {
-  list-style: none;
-  padding-left: 0;
   margin-bottom: 0;
+  padding-left: 0;
+  list-style: none;
 }
 
 .emoji-menu-list-item {
+  float: left;
   padding: 3px;
   margin-left: 1px;
   margin-right: 1px;
@@ -97,6 +99,8 @@
   padding: 5px 6px;
   outline: 0;
 
+  line-height: 1;
+
   &.disabled {
     cursor: default;
 

app/assets/stylesheets/framework/markdown_area.scss

@@ -147,6 +147,9 @@
 }
 
 .atwho-view {
+  overflow-y: auto;
+  overflow-x: hidden;
+