Merge branch 'upgrade-to-webpack-v4' into 'master'

Upgrade to Webpack 4

Closes #43400

See merge request gitlab-org/gitlab-ce!17218

app/assets/javascripts/ide/index.js

@@ -4,7 +4,9 @@ import ide from './components/ide.vue';
 import store from './stores';
 import router from './ide_router';
 
-function initIde(el) {
+Vue.use(Translate);
+
+export function initIde(el) {
   if (!el) return null;
 
   return new Vue({
@@ -27,8 +29,12 @@ function initIde(el) {
   });
 }
 
-const ideElement = document.getElementById('ide');
-
-Vue.use(Translate);
-
-initIde(ideElement);
+// tell webpack to load assets from origin so that web workers don't break
+export function resetServiceWorkersPublicPath() {
+  // __webpack_public_path__ is a global variable that can be used to adjust
+  // the webpack publicPath setting at runtime.
+  // see: https://webpack.js.org/guides/public-path/
+  const relativeRootPath = (gon && gon.relative_url_root) || '';
+  const webpackAssetPath = `${relativeRootPath}/assets/webpack/`;
+  __webpack_public_path__ = webpackAssetPath; // eslint-disable-line camelcase
+}

app/assets/javascripts/main.js

-/* eslint-disable import/first */
 /* global $ */
 
 import jQuery from 'jquery';
 import Cookies from 'js-cookie';
 import svg4everybody from 'svg4everybody';
 
-// expose common libraries as globals (TODO: remove these)
-window.jQuery = jQuery;
-window.$ = jQuery;
+// bootstrap webpack, common libs, polyfills, and behaviors
+import './webpack';
+import './commons';
+import './behaviors';
 
 // lib/utils
 import { handleLocationHash, addSelectOnFocusBehaviour } from './lib/utils/common_utils';
 import { localTimeAgo } from './lib/utils/datetime_utility';
 import { getLocationHash, visitUrl } from './lib/utils/url_utility';
 
-// behaviors
-import './behaviors/';
-
 // everything else
 import loadAwardsHandler from './awards_handler';
 import bp from './breakpoints';
@@ -31,9 +28,12 @@ import initLogoAnimation from './logo';
 import './milestone_select';
 import './projects_dropdown';
 import initBreadcrumbs from './breadcrumb';
-
 import initDispatcher from './dispatcher';
 
+// expose jQuery as global (TODO: remove these)
+window.jQuery = jQuery;
+window.$ = jQuery;
+
 // inject test utilities if necessary
 if (process.env.NODE_ENV !== 'production' && gon && gon.test_env) {
   $.fx.off = true;
@@ -52,10 +52,14 @@ document.addEventListener('beforeunload', () => {
 });
 
 window.addEventListener('hashchange', handleLocationHash);
-window.addEventListener('load', function onLoad() {
-  window.removeEventListener('load', onLoad, false);
-  handleLocationHash();
-}, false);
+window.addEventListener(
+  'load',
+  function onLoad() {
+    window.removeEventListener('load', onLoad, false);
+    handleLocationHash();
+  },
+  false,
+);
 
 gl.lazyLoader = new LazyLoader({
   scrollContainer: window,
@@ -89,9 +93,7 @@ document.addEventListener('DOMContentLoaded', () => {
   if (bootstrapBreakpoint === 'xs') {
     const $rightSidebar = $('aside.right-sidebar, .layout-page');
 
-    $rightSidebar
-      .removeClass('right-sidebar-expanded')
-      .addClass('right-sidebar-collapsed');
+    $rightSidebar.removeClass('right-sidebar-expanded').addClass('right-sidebar-collapsed');
   }
 
   // prevent default action for disabled buttons
@@ -108,7 +110,8 @@ document.addEventListener('DOMContentLoaded', () => {
   addSelectOnFocusBehaviour('.js-select-on-focus');
 
   $('.remove-row').on('ajax:success', function removeRowAjaxSuccessCallback() {
-    $(this).tooltip('destroy')
+    $(this)
+      .tooltip('destroy')
       .closest('li')
       .fadeOut();
   });
@@ -118,7 +121,9 @@ document.addEventListener('DOMContentLoaded', () => {
   });
 
   $('.js-remove-tr').on('ajax:success', function removeTRAjaxSuccessCallback() {
-    $(this).closest('tr').fadeOut();
+    $(this)
+      .closest('tr')
+      .fadeOut();
   });
 
   // Initialize select2 selects
@@ -155,7 +160,9 @@ document.addEventListener('DOMContentLoaded', () => {
 
   // Form submitter
   $('.trigger-submit').on('change', function triggerSubmitCallback() {
-    $(this).parents('form').submit();
+    $(this)
+      .parents('form')
+      .submit();
   });
 
   localTimeAgo($('abbr.timeago, .js-timeago'), true);
@@ -204,9 +211,15 @@ document.addEventListener('DOMContentLoaded', () => {
     $this.toggleClass('active');
 
     if ($this.hasClass('active')) {
-      notesHolders.show().find('.hide, .content').show();
+      notesHolders
+        .show()
+        .find('.hide, .content')
+        .show();
     } else {
-      notesHolders.hide().find('.content').hide();
+      notesHolders
+        .hide()
+        .find('.content')
+        .hide();
     }
 
     $(document).trigger('toggle.comments');
@@ -247,9 +260,11 @@ document.addEventListener('DOMContentLoaded', () => {
   const flashContainer = document.querySelector('.flash-container');
 
   if (flashContainer && flashContainer.children.length) {
-    flashContainer.querySelectorAll('.flash-alert, .flash-notice, .flash-success').forEach((flashEl) => {
-      removeFlashClickListener(flashEl);
-    });
+    flashContainer
+      .querySelectorAll('.flash-alert, .flash-notice, .flash-success')
+      .forEach(flashEl => {
+        removeFlashClickListener(flashEl);
+      });
   }
 
   initDispatcher();

app/assets/javascripts/pages/ide/index.js

+import { initIde, resetServiceWorkersPublicPath } from '~/ide/index';
+
+document.addEventListener('DOMContentLoaded', () => {
+  const ideElement = document.getElementById('ide');
+  if (ideElement) {
+    resetServiceWorkersPublicPath();
+    initIde(ideElement);
+  }
+});

app/helpers/webpack_helper.rb

-require 'webpack/rails/manifest'
+require 'gitlab/webpack/manifest'
 
 module WebpackHelper
-  def webpack_bundle_tag(bundle, force_same_domain: false)
-    javascript_include_tag(*gitlab_webpack_asset_paths(bundle, force_same_domain: force_same_domain))
+  def webpack_bundle_tag(bundle)
+    javascript_include_tag(*webpack_entrypoint_paths(bundle))
   end
 
   def webpack_controller_bundle_tags
-    bundles = []
+    chunks = []
 
     action = case controller.action_name
              when 'create' then 'new'
@@ -16,37 +16,44 @@ def webpack_controller_bundle_tags
 
     route = [*controller.controller_path.split('/'), action].compact
 
-    until route.empty?
+    until chunks.any? || route.empty?
+      entrypoint = "pages.#{route.join('.')}"
       begin
-        asset_paths = gitlab_webpack_asset_paths("pages.#{route.join('.')}", extension: 'js')
-        bundles.unshift(*asset_paths)
-      rescue Webpack::Rails::Manifest::EntryPointMissingError
+        chunks = webpack_entrypoint_paths(entrypoint, extension: 'js')
+      rescue Gitlab::Webpack::Manifest::AssetMissingError
         # no bundle exists for this path
       end
-
       route.pop
     end
 
-    javascript_include_tag(*bundles)
+    if chunks.empty?
+      chunks = webpack_entrypoint_paths("default", extension: 'js')
+    end
+
+    javascript_include_tag(*chunks)
   end
 
-  # override webpack-rails gem helper until changes can make it upstream
-  def gitlab_webpack_asset_paths(source, extension: nil, force_same_domain: false)
+  def webpack_entrypoint_paths(source, extension: nil, exclude_duplicates: true)
     return "" unless source.present?
 
-    paths = Webpack::Rails::Manifest.asset_paths(source)
+    paths = Gitlab::Webpack::Manifest.entrypoint_paths(source)
     if extension
       paths.select! { |p| p.ends_with? ".#{extension}" }
     end
 
-    unless force_same_domain
-      force_host = webpack_public_host
-      if force_host
-        paths.map! { |p| "#{force_host}#{p}" }
-      end
+    force_host = webpack_public_host
+    if force_host
+      paths.map! { |p| "#{force_host}#{p}" }
     end
 
-    paths
+    if exclude_duplicates
+      @used_paths ||= []
+      new_paths = paths - @used_paths
+      @used_paths += new_paths
+      new_paths
+    else
+      paths
+    end
   end
 
   def webpack_public_host

app/views/ide/index.html.haml

 - @body_class = 'ide'
 - page_title 'IDE'
 
-- content_for :page_specific_javascripts do
-  = webpack_bundle_tag 'ide', force_same_domain: true
-
 #ide.ide-loading{ data: {"empty-state-svg-path" => image_path('illustrations/multi_file_editor_empty.svg'),
   "no-changes-state-svg-path" => image_path('illustrations/multi-editor_no_changes_empty.svg'),
   "committed-state-svg-path" => image_path('illustrations/multi-editor_all_changes_committed_empty.svg') } }

app/views/layouts/_head.html.haml

@@ -38,9 +38,6 @@
     = yield :library_javascripts
 
   = javascript_include_tag locale_path unless I18n.locale == :en
-  = webpack_bundle_tag "webpack_runtime"
-  = webpack_bundle_tag "common"
-  = webpack_bundle_tag "main"
   = webpack_bundle_tag "raven" if Gitlab::CurrentSettings.clientside_sentry_enabled
 
   - if content_for?(:page_specific_javascripts)

config/initializers/static_files.rb

@@ -34,7 +34,7 @@
       )
       app.config.middleware.insert_before(
         Gitlab::Middleware::Static,
-        Gitlab::Middleware::WebpackProxy,
+        Gitlab::Webpack::DevServerMiddleware,
         proxy_path: app.config.webpack.public_path,
         proxy_host: dev_server.host,
         proxy_port: dev_server.port

config/karma.config.js

@@ -12,16 +12,14 @@ function fatalError(message) {
   process.exit(1);
 }
 
-// remove problematic plugins
-if (webpackConfig.plugins) {
-  webpackConfig.plugins = webpackConfig.plugins.filter(function(plugin) {
-    return !(
-      plugin instanceof webpack.optimize.CommonsChunkPlugin ||
-      plugin instanceof webpack.optimize.ModuleConcatenationPlugin ||
-      plugin instanceof webpack.DefinePlugin
-    );
-  });
-}
+// disable problematic options
+webpackConfig.entry = undefined;
+webpackConfig.mode = 'development';
+webpackConfig.optimization.runtimeChunk = false;
+webpackConfig.optimization.splitChunks = false;
+
+// use quicker sourcemap option
+webpackConfig.devtool = 'cheap-inline-source-map';
 
 const specFilters = argumentsParser
   .option(
@@ -77,9 +75,6 @@ if (specFilters.length) {
   );
 }
 
-webpackConfig.entry = undefined;
-webpackConfig.devtool = 'cheap-inline-source-map';
-
 // Karma configuration
 module.exports = function(config) {
   process.env.TZ = 'Etc/UTC';

config/webpack.config.js

-const crypto = require('crypto');
 const fs = require('fs');
 const path = require('path');
 const glob = require('glob');
@@ -6,9 +5,7 @@ const webpack = require('webpack');
 const StatsWriterPlugin = require('webpack-stats-plugin').StatsWriterPlugin;
 const CopyWebpackPlugin = require('copy-webpack-plugin');
 const CompressionPlugin = require('compression-webpack-plugin');
-const NameAllModulesPlugin = require('name-all-modules-plugin');
 const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
-const WatchMissingNodeModulesPlugin = require('react-dev-utils/WatchMissingNodeModulesPlugin');
 
 const ROOT_PATH = path.resolve(__dirname, '..');
 const IS_PRODUCTION = process.env.NODE_ENV === 'production';
@@ -21,10 +18,12 @@ const NO_COMPRESSION = process.env.NO_COMPRESSION;
 
 let autoEntriesCount = 0;
 let watchAutoEntries = [];
+const defaultEntries = ['./main'];
 
 function generateEntries() {
   // generate automatic entry points
   const autoEntries = {};
+  const autoEntriesMap = {};
   const pageEntries = glob.sync('pages/**/index.js', {
     cwd: path.join(ROOT_PATH, 'app/assets/javascripts'),
   });
@@ -33,25 +32,38 @@ function generateEntries() {
   function generateAutoEntries(path, prefix = '.') {
     const chunkPath = path.replace(/\/index\.js$/, '');
     const chunkName = chunkPath.replace(/\//g, '.');
-    autoEntries[chunkName] = `${prefix}/${path}`;
+    autoEntriesMap[chunkName] = `${prefix}/${path}`;
   }
 
   pageEntries.forEach(path => generateAutoEntries(path));
 
-  autoEntriesCount = Object.keys(autoEntries).length;
+  const autoEntryKeys = Object.keys(autoEntriesMap);
+  autoEntriesCount = autoEntryKeys.length;
+
+  // import ancestor entrypoints within their children
+  autoEntryKeys.forEach(entry => {
+    const entryPaths = [autoEntriesMap[entry]];
+    const segments = entry.split('.');
+    while (segments.pop()) {
+      const ancestor = segments.join('.');
+      if (autoEntryKeys.includes(ancestor)) {
+        entryPaths.unshift(autoEntriesMap[ancestor]);
+      }
+    }
+    autoEntries[entry] = defaultEntries.concat(entryPaths);
+  });
 
   const manualEntries = {
-    common: './commons/index.js',
-    main: './main.js',
+    default: defaultEntries,
     raven: './raven/index.js',
-    webpack_runtime: './webpack.js',
-    ide: './ide/index.js',
   };
 
   return Object.assign(manualEntries, autoEntries);
 }
 
 const config = {
+  mode: IS_PRODUCTION ? 'production' : 'development',
+
   context: path.join(ROOT_PATH, 'app/assets/javascripts'),
 
   entry: generateEntries,
@@ -59,8 +71,36 @@ const config = {
   output: {
     path: path.join(ROOT_PATH, 'public/assets/webpack'),
     publicPath: '/assets/webpack/',
-    filename: IS_PRODUCTION ? '[name].[chunkhash].bundle.js' : '[name].bundle.js',
-    chunkFilename: IS_PRODUCTION ? '[name].[chunkhash].chunk.js' : '[name].chunk.js',
+    filename: IS_PRODUCTION ? '[name].[chunkhash:8].bundle.js' : '[name].bundle.js',
+    chunkFilename: IS_PRODUCTION ? '[name].[chunkhash:8].chunk.js' : '[name].chunk.js',
+    globalObject: 'this', // allow HMR and web workers to play nice
+  },
+
+  optimization: {
+    nodeEnv: false,
+    runtimeChunk: 'single',
+    splitChunks: {
+      maxInitialRequests: 4,
+      cacheGroups: {
+        default: false,
+        common: () => ({
+          priority: 20,
+          name: 'main',
+          chunks: 'initial',
+          minChunks: autoEntriesCount * 0.9,
+        }),
+        vendors: {
+          priority: 10,
+          chunks: 'async',
+          test: /[\\/](node_modules|vendor[\\/]assets[\\/]javascripts)[\\/]/,
+        },
+        commons: {
+          chunks: 'all',
+          minChunks: 2,
+          reuseExistingChunk: true,
+        },
+      },
+    },
   },
 
   module: {
@@ -92,10 +132,10 @@ const config = {
           {
             loader: 'worker-loader',
             options: {
-              inline: true,
+              name: '[name].[hash:8].worker.js',
             },
           },
-          { loader: 'babel-loader' },
+          'babel-loader',
         ],
       },
       {
@@ -103,7 +143,7 @@ const config = {
         exclude: /node_modules/,
         loader: 'file-loader',
         options: {
-          name: '[name].[hash].[ext]',
+          name: '[name].[hash:8].[ext]',
         },
       },
       {
@@ -114,7 +154,7 @@ const config = {
           {
             loader: 'css-loader',
             options: {
-              name: '[name].[hash].[ext]',
+              name: '[name].[hash:8].[ext]',
             },
           },
         ],
@@ -124,7 +164,7 @@ const config = {
         include: /node_modules\/katex\/dist\/fonts/,
         loader: 'file-loader',
         options: {
-          name: '[name].[hash].[ext]',
+          name: '[name].[hash:8].[ext]',
         },
       },
       {
@@ -166,54 +206,6 @@ const config = {
       jQuery: 'jquery',
     }),
 
-    // assign deterministic module ids
-    new webpack.NamedModulesPlugin(),
-    new NameAllModulesPlugin(),
-
-    // assign deterministic chunk ids
-    new webpack.NamedChunksPlugin(chunk => {
-      if (chunk.name) {
-        return chunk.name;
-      }
-
-      const moduleNames = [];
-
-      function collectModuleNames(m) {
-        // handle ConcatenatedModule which does not have resource nor context set
-        if (m.modules) {
-          m.modules.forEach(collectModuleNames);
-          return;
-        }
-
-        const pagesBase = path.join(ROOT_PATH, 'app/assets/javascripts/pages');
-
-        if (m.resource.indexOf(pagesBase) === 0) {
-          moduleNames.push(
-            path
-              .relative(pagesBase, m.resource)
-              .replace(/\/index\.[a-z]+$/, '')
-              .replace(/\//g, '__')
-          );
-        } else {
-          moduleNames.push(path.relative(m.context, m.resource));
-        }
-      }
-
-      chunk.forEachModule(collectModuleNames);
-
-      const hash = crypto
-        .createHash('sha256')
-        .update(moduleNames.join('_'))
-        .digest('hex');
-
-      return `${moduleNames[0]}-${hash.substr(0, 6)}`;
-    }),
-
-    // create cacheable common library bundles
-    new webpack.optimize.CommonsChunkPlugin({
-      names: ['main', 'common', 'webpack_runtime'],
-    }),
-
     // copy pre-compiled vendor libraries verbatim
     new CopyWebpackPlugin([
       {
@@ -260,20 +252,6 @@ const config = {
 
 if (IS_PRODUCTION) {
   config.devtool = 'source-map';
-  config.plugins.push(
-    new webpack.NoEmitOnErrorsPlugin(),
-    new webpack.LoaderOptionsPlugin({
-      minimize: true,
-      debug: false,
-    }),
-    new webpack.optimize.ModuleConcatenationPlugin(),
-    new webpack.optimize.UglifyJsPlugin({
-      sourceMap: true,
-    }),
-    new webpack.DefinePlugin({
-      'process.env': { NODE_ENV: JSON.stringify('production') },
-    })
-  );
 
   // compression can require a lot of compute time and is disabled in CI
   if (!NO_COMPRESSION) {
@@ -292,29 +270,30 @@ if (IS_DEV_SERVER) {
     hot: DEV_SERVER_LIVERELOAD,
     inline: DEV_SERVER_LIVERELOAD,
   };
-  config.plugins.push(
-    // watch node_modules for changes if we encounter a missing module compile error
-    new WatchMissingNodeModulesPlugin(path.join(ROOT_PATH, 'node_modules')),
-
-    // watch for changes to our automatic entry point modules
-    {
-      apply(compiler) {
-        compiler.plugin('emit', (compilation, callback) => {
-          compilation.contextDependencies = [
-            ...compilation.contextDependencies,
-            ...watchAutoEntries,
-          ];
-
-          // report our auto-generated bundle count
-          console.log(
-            `${autoEntriesCount} entries from '/pages' automatically added to webpack output.`
-          );
-
-          callback();
-        });
-      },
-    }
-  );
+  config.plugins.push({
+    apply(compiler) {
+      compiler.hooks.emit.tapAsync('WatchForChangesPlugin', (compilation, callback) => {
+        const missingDeps = Array.from(compilation.missingDependencies);
+        const nodeModulesPath = path.join(ROOT_PATH, 'node_modules');
+        const hasMissingNodeModules = missingDeps.some(
+          file => file.indexOf(nodeModulesPath) !== -1
+        );
+
+        // watch for changes to missing node_modules
+        if (hasMissingNodeModules) compilation.contextDependencies.add(nodeModulesPath);
+
+        // watch for changes to automatic entrypoints
+        watchAutoEntries.forEach(watchPath => compilation.contextDependencies.add(watchPath));
+
+        // report our auto-generated bundle count
+        console.log(
+          `${autoEntriesCount} entries from '/pages' automatically added to webpack output.`
+        );
+
+        callback();
+      });
+    },
+  });
   if (DEV_SERVER_LIVERELOAD) {
     config.plugins.push(new webpack.HotModuleReplacementPlugin());
   }

lib/gitlab/middleware/webpack_proxy.rb → lib/gitlab/webpack/dev_server_middleware.rb

@@ -3,8 +3,8 @@
 
 # :nocov:
 module Gitlab
-  module Middleware
-    class WebpackProxy < Rack::Proxy
+  module Webpack
+    class DevServerMiddleware < Rack::Proxy
       def initialize(app = nil, opts = {})
         @proxy_host = opts.fetch(:proxy_host, 'localhost')
         @proxy_port = opts.fetch(:proxy_port, 3808)

lib/gitlab/webpack/manifest.rb

+require 'webpack/rails/manifest'
+
+module Gitlab
+  module Webpack
+    class Manifest < ::Webpack::Rails::Manifest
+      # Raised if a supplied asset does not exist in the webpack manifest
+      AssetMissingError = Class.new(StandardError)
+
+      class << self
+        def entrypoint_paths(source)
+          raise ::Webpack::Rails::Manifest::WebpackError, manifest["errors"] unless manifest_bundled?
+
+          entrypoint = manifest["entrypoints"][source]