Merge branch '11-9-stable-patch-8' into '11-9-stable'

Prepare 11.9.8 release

See merge request gitlab-org/gitlab-ce!27253

.rubocop_todo.yml

@@ -268,6 +268,7 @@ Rails/Presence:
     - 'app/models/clusters/platforms/kubernetes.rb'
     - 'app/models/concerns/mentionable.rb'
     - 'app/models/concerns/token_authenticatable.rb'
+    - 'app/models/project_services/hipchat_service.rb'
     - 'app/models/project_services/irker_service.rb'
     - 'app/models/project_services/jira_service.rb'
     - 'app/models/project_services/kubernetes_service.rb'

Gemfile

@@ -204,6 +204,9 @@ gem 'connection_pool', '~> 2.0'
 # Discord integration
 gem 'discordrb-webhooks-blackst0ne', '~> 3.3', require: false
 
+# HipChat integration
+gem 'hipchat', '~> 1.5.0'
+
 # JIRA integration
 gem 'jira-ruby', '~> 1.4'
 

Gemfile.lock

@@ -364,6 +364,9 @@ GEM
       hashie (>= 3.0)
     health_check (2.6.0)
       rails (>= 4.0)
+    hipchat (1.5.2)
+      httparty
+      mimemagic
     html-pipeline (2.8.4)
       activesupport (>= 2)
       nokogiri (>= 1.4)
@@ -1041,6 +1044,7 @@ DEPENDENCIES
   hangouts-chat (~> 0.0.5)
   hashie-forbidden_attributes
   health_check (~> 2.6.0)
+  hipchat (~> 1.5.0)
   html-pipeline (~> 2.8)
   html2text
   httparty (~> 0.13.3)

app/controllers/dashboard_controller.rb

@@ -46,7 +46,10 @@ class DashboardController < Dashboard::ApplicationController
   end
 
   def check_filters_presence!
-    @no_filters_set = finder_type.scalar_params.none? { |k| params.key?(k) }
+    no_scalar_filters_set = finder_type.scalar_params.none? { |k| params.key?(k) }
+    no_array_filters_set = finder_type.array_params.none? { |k, _| params.key?(k) }
+
+    @no_filters_set = no_scalar_filters_set && no_array_filters_set
 
     return unless @no_filters_set
 

app/finders/issuable_finder.rb

@@ -53,7 +53,6 @@ class IssuableFinder
       assignee_username
       author_id
       author_username
-      label_name
       milestone_title
       my_reaction_emoji
       search

app/models/merge_request.rb

@@ -1301,7 +1301,7 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def has_commits?
-    merge_request_diff && commits_count > 0
+    merge_request_diff && commits_count.to_i > 0
   end
 
   def has_no_commits?

app/models/project.rb

@@ -147,6 +147,7 @@ class Project < ActiveRecord::Base
   has_one :pipelines_email_service
   has_one :irker_service
   has_one :pivotaltracker_service
+  has_one :hipchat_service
   has_one :flowdock_service
   has_one :assembla_service
   has_one :asana_service

app/models/project_services/hipchat_service.rb

+# frozen_string_literal: true
+
+class HipchatService < Service
+  include ActionView::Helpers::SanitizeHelper
+
+  MAX_COMMITS = 3
+  HIPCHAT_ALLOWED_TAGS = %w[
+    a b i strong em br img pre code
+    table th tr td caption colgroup col thead tbody tfoot
+    ul ol li dl dt dd
+  ].freeze
+
+  prop_accessor :token, :room, :server, :color, :api_version
+  boolean_accessor :notify_only_broken_pipelines, :notify
+  validates :token, presence: true, if: :activated?
+
+  def initialize_properties
+    if properties.nil?
+      self.properties = {}
+      self.notify_only_broken_pipelines = true
+    end
+  end
+
+  def title
+    'HipChat'
+  end
+
+  def description
+    'Private group chat and IM'
+  end
+
+  def self.to_param
+    'hipchat'
+  end
+
+  def fields
+    [
+      { type: 'text', name: 'token',     placeholder: 'Room token', required: true },
+      { type: 'text', name: 'room',      placeholder: 'Room name or ID' },
+      { type: 'checkbox', name: 'notify' },
+      { type: 'select', name: 'color', choices: %w(yellow red green purple gray random) },
+      { type: 'text', name: 'api_version',
+        placeholder: 'Leave blank for default (v2)' },
+      { type: 'text', name: 'server',
+        placeholder: 'Leave blank for default. https://hipchat.example.com' },
+      { type: 'checkbox', name: 'notify_only_broken_pipelines' }
+    ]
+  end
+
+  def self.supported_events
+    %w(push issue confidential_issue merge_request note confidential_note tag_push pipeline)
+  end
+
+  def execute(data)
+    return unless supported_events.include?(data[:object_kind])
+
+    message = create_message(data)
+    return unless message.present?
+
+    gate[room].send('GitLab', message, message_options(data)) # rubocop:disable GitlabSecurity/PublicSend
+  end
+
+  def test(data)
+    begin
+      result = execute(data)
+    rescue StandardError => error
+      return { success: false, result: error }
+    end
+
+    { success: true, result: result }
+  end
+
+  private
+
+  def gate
+    options = { api_version: api_version.present? ? api_version : 'v2' }
+    options[:server_url] = server unless server.blank?
+    @gate ||= HipChat::Client.new(token, options)
+  end
+
+  def message_options(data = nil)
+    { notify: notify.present? && Gitlab::Utils.to_boolean(notify), color: message_color(data) }
+  end
+
+  def create_message(data)
+    object_kind = data[:object_kind]
+
+    case object_kind
+    when "push", "tag_push"
+      create_push_message(data)
+    when "issue"
+      create_issue_message(data) unless update?(data)
+    when "merge_request"
+      create_merge_request_message(data) unless update?(data)
+    when "note"
+      create_note_message(data)
+    when "pipeline"
+      create_pipeline_message(data) if should_pipeline_be_notified?(data)
+    end
+  end
+
+  def render_line(text)
+    markdown(text.lines.first.chomp, pipeline: :single_line) if text
+  end
+
+  def create_push_message(push)
+    ref_type = Gitlab::Git.tag_ref?(push[:ref]) ? 'tag' : 'branch'
+    ref = Gitlab::Git.ref_name(push[:ref])
+
+    before = push[:before]
+    after = push[:after]
+
+    message = []
+    message << "#{push[:user_name]} "
+
+    if Gitlab::Git.blank_ref?(before)
+      message << "pushed new #{ref_type} <a href=\""\
+                 "#{project_url}/commits/#{CGI.escape(ref)}\">#{ref}</a>"\
+                 " to #{project_link}\n"
+    elsif Gitlab::Git.blank_ref?(after)
+      message << "removed #{ref_type} <b>#{ref}</b> from <a href=\"#{project.web_url}\">#{project_name}</a> \n"
+    else
+      message << "pushed to #{ref_type} <a href=\""\
+                  "#{project.web_url}/commits/#{CGI.escape(ref)}\">#{ref}</a> "
+      message << "of <a href=\"#{project.web_url}\">#{project.full_name.gsub!(/\s/, '')}</a> "
+      message << "(<a href=\"#{project.web_url}/compare/#{before}...#{after}\">Compare changes</a>)"
+
+      push[:commits].take(MAX_COMMITS).each do |commit|
+        message << "<br /> - #{render_line(commit[:message])} (<a href=\"#{commit[:url]}\">#{commit[:id][0..5]}</a>)"
+      end
+
+      if push[:commits].count > MAX_COMMITS
+        message << "<br />... #{push[:commits].count - MAX_COMMITS} more commits"
+      end
+    end
+
+    message.join
+  end
+
+  def markdown(text, options = {})
+    return "" unless text
+
+    context = {
+      project: project,
+      pipeline: :email
+    }
+
+    Banzai.render(text, context)
+
+    context.merge!(options)
+
+    html = Banzai.render_and_post_process(text, context)
+    sanitized_html = sanitize(html, tags: HIPCHAT_ALLOWED_TAGS, attributes: %w[href title alt])
+
+    sanitized_html.truncate(200, separator: ' ', omission: '...')
+  end
+
+  def create_issue_message(data)
+    user_name = data[:user][:name]
+
+    obj_attr = data[:object_attributes]
+    obj_attr = HashWithIndifferentAccess.new(obj_attr)
+    title = render_line(obj_attr[:title])
+    state = obj_attr[:state]
+    issue_iid = obj_attr[:iid]
+    issue_url = obj_attr[:url]
+    description = obj_attr[:description]
+
+    issue_link = "<a href=\"#{issue_url}\">issue ##{issue_iid}</a>"
+
+    message = ["#{user_name} #{state} #{issue_link} in #{project_link}: <b>#{title}</b>"]
+    message << "<pre>#{markdown(description)}</pre>"
+
+    message.join
+  end
+
+  def create_merge_request_message(data)
+    user_name = data[:user][:name]
+
+    obj_attr = data[:object_attributes]
+    obj_attr = HashWithIndifferentAccess.new(obj_attr)
+    merge_request_id = obj_attr[:iid]
+    state = obj_attr[:state]
+    description = obj_attr[:description]
+    title = render_line(obj_attr[:title])
+
+    merge_request_url = "#{project_url}/merge_requests/#{merge_request_id}"
+    merge_request_link = "<a href=\"#{merge_request_url}\">merge request !#{merge_request_id}</a>"
+    message = ["#{user_name} #{state} #{merge_request_link} in " \
+      "#{project_link}: <b>#{title}</b>"]
+
+    message << "<pre>#{markdown(description)}</pre>"
+    message.join
+  end
+
+  def format_title(title)
+    "<b>#{render_line(title)}</b>"
+  end
+
+  def create_note_message(data)
+    data = HashWithIndifferentAccess.new(data)
+    user_name = data[:user][:name]
+
+    obj_attr = HashWithIndifferentAccess.new(data[:object_attributes])
+    note = obj_attr[:note]
+    note_url = obj_attr[:url]
+    noteable_type = obj_attr[:noteable_type]
+    commit_id = nil
+
+    case noteable_type
+    when "Commit"
+      commit_attr = HashWithIndifferentAccess.new(data[:commit])
+      commit_id = commit_attr[:id]
+      subject_desc = commit_id
+      subject_desc = Commit.truncate_sha(subject_desc)
+      subject_type = "commit"
+      title = format_title(commit_attr[:message])
+    when "Issue"
+      subj_attr = HashWithIndifferentAccess.new(data[:issue])
+      subject_id = subj_attr[:iid]
+      subject_desc = "##{subject_id}"
+      subject_type = "issue"
+      title = format_title(subj_attr[:title])
+    when "MergeRequest"
+      subj_attr = HashWithIndifferentAccess.new(data[:merge_request])
+      subject_id = subj_attr[:iid]
+      subject_desc = "!#{subject_id}"
+      subject_type = "merge request"
+      title = format_title(subj_attr[:title])
+    when "Snippet"
+      subj_attr = HashWithIndifferentAccess.new(data[:snippet])
+      subject_id = subj_attr[:id]
+      subject_desc = "##{subject_id}"
+      subject_type = "snippet"
+      title = format_title(subj_attr[:title])
+    end
+
+    subject_html = "<a href=\"#{note_url}\">#{subject_type} #{subject_desc}</a>"
+    message = ["#{user_name} commented on #{subject_html} in #{project_link}: "]
+    message << title
+
+    message << "<pre>#{markdown(note, ref: commit_id)}</pre>"
+    message.join
+  end
+
+  def create_pipeline_message(data)
+    pipeline_attributes = data[:object_attributes]
+    pipeline_id = pipeline_attributes[:id]
+    ref_type = pipeline_attributes[:tag] ? 'tag' : 'branch'
+    ref = pipeline_attributes[:ref]
+    user_name = (data[:user] && data[:user][:name]) || 'API'
+    status = pipeline_attributes[:status]
+    duration = pipeline_attributes[:duration]
+
+    branch_link = "<a href=\"#{project_url}/commits/#{CGI.escape(ref)}\">#{ref}</a>"
+    pipeline_url = "<a href=\"#{project_url}/pipelines/#{pipeline_id}\">##{pipeline_id}</a>"
+
+    "#{project_link}: Pipeline #{pipeline_url} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status(status)} in #{duration} second(s)"
+  end
+
+  def message_color(data)
+    pipeline_status_color(data) || color || 'yellow'
+  end
+
+  def pipeline_status_color(data)
+    return unless data && data[:object_kind] == 'pipeline'
+
+    case data[:object_attributes][:status]
+    when 'success'
+      'green'
+    else
+      'red'
+    end
+  end
+
+  def project_name
+    project.full_name.gsub(/\s/, '')
+  end
+
+  def project_url
+    project.web_url
+  end
+
+  def project_link
+    "<a href=\"#{project_url}\">#{project_name}</a>"
+  end
+
+  def update?(data)
+    data[:object_attributes][:action] == 'update'
+  end
+
+  def humanized_status(status)
+    case status
+    when 'success'
+      'passed'
+    else
+      status
+    end
+  end
+
+  def should_pipeline_be_notified?(data)
+    case data[:object_attributes][:status]
+    when 'success'
+      !notify_only_broken_pipelines?
+    when 'failed'
+      true
+    else
+      false
+    end
+  end
+end

app/models/service.rb

@@ -255,6 +255,7 @@ class Service < ActiveRecord::Base
       external_wiki
       flowdock
       hangouts_chat
+      hipchat
       irker
       jira
       kubernetes

changelogs/unreleased/allow-to-use-untrusted-ruby-syntax.yml

+---
+title: Allow to use untrusted Regexp via feature flag
+merge_request: 26905
+author:
+type: deprecated

changelogs/unreleased/fix-pull-request-importer.yml

+---
+title: Improve performance of PR import
+merge_request: 27121
+author:
+type: performance

changelogs/unreleased/restore-hipchat.yml

+---
+title: Restore HipChat project service
+merge_request: 27172
+author:
+type: change

changelogs/unreleased/sh-disable-diff-instrumentation.yml

+---
+title: Disable method instrumentation for diffs
+merge_request: 27235
+author:
+type: performance

config/initializers/hipchat_client_patch.rb

+# frozen_string_literal: true
+# This monkey patches the HTTParty used in https://github.com/hipchat/hipchat-rb.
+module HipChat
+  class Client
+    connection_adapter ::Gitlab::ProxyHTTPConnectionAdapter
+  end
+
+  class Room
+    connection_adapter ::Gitlab::ProxyHTTPConnectionAdapter
+  end
+
+  class User
+    connection_adapter ::Gitlab::ProxyHTTPConnectionAdapter
+  end
+end

config/initializers/zz_metrics.rb

@@ -30,7 +30,6 @@ def instrument_classes(instrumentation)
     # are included.
     %w(app services [^concerns]**) => %w(app services),
     %w(lib gitlab conflicts)       => ['lib'],
-    %w(lib gitlab diff)            => ['lib'],
     %w(lib gitlab email message)   => ['lib'],
     %w(lib gitlab checks)          => ['lib']
   }

db/migrate/20190107151029_remove_hipchat_services.rb

-# frozen_string_literal: true
-
-# See http://doc.gitlab.com/ce/development/migration_style_guide.html
-# for more information on how to write migrations for GitLab.
-
-class RemoveHipchatServices < ActiveRecord::Migration[5.0]
-  DOWNTIME = false
-
-  def up
-    execute "DELETE FROM services WHERE type = 'HipchatService'"
-  end
-
-  def down
-    # no-op
-  end
-end

doc/administration/auth/ldap.md

@@ -48,6 +48,14 @@ LDAP-enabled users can always authenticate with Git using their GitLab username
 or email and LDAP password, even if password authentication for Git is disabled
 in the application settings.
 
+## Google Secure LDAP **[CORE ONLY]**
+
+> Introduced in GitLab 11.9.
+
+[Google Cloud Identity](https://cloud.google.com/identity/) provides a Secure
+LDAP service that can be configured with GitLab for authentication and group sync.
+See [Google Secure LDAP](google_secure_ldap.md) for detailed configuration instructions.
+
 ## Configuration
 
 NOTE: **Note**:

doc/api/services.md

@@ -449,6 +449,45 @@ Get Hangouts Chat service settings for a project.
 GET /projects/:id/services/hangouts-chat
 ```
 
+## HipChat
+
+Private group chat and IM
+
+### Create/Edit HipChat service
+
+Set HipChat service for a project.
+
+```
+PUT /projects/:id/services/hipchat
+```
+
+Parameters:
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `token` | string | true | Room token |
+| `color` | string | false | The room color |
+| `notify` | boolean | false | Enable notifications |
+| `room` | string | false |Room name or ID |
+| `api_version` | string | false | Leave blank for default (v2) |
+| `server` | string | false | Leave blank for default. For example, `https://hipchat.example.com`. |
+
+### Delete HipChat service
+
+Delete HipChat service for a project.
+
+```
+DELETE /projects/:id/services/hipchat
+```
+
+### Get HipChat service settings
+
+Get HipChat service settings for a project.
+
+```
+GET /projects/:id/services/hipchat
+```
+
 ## Irker (IRC gateway)
 
 Send IRC messages, on update, to a list of recipients through an Irker gateway.

doc/ci/yaml/README.md

@@ -397,6 +397,27 @@ job:
   only: ['branches', 'tags']
 ```
 
+### Supported `only`/`except` regexp syntax
+
+CAUTION: **Warning:**
+This is a breaking change that was introduced with GitLab 11.9.4.
+
+In GitLab 11.9.4, GitLab begun internally converting regexp used
+in `only` and `except` parameters to [RE2](https://github.com/google/re2/wiki/Syntax).
+
+This means that only subset of features provided by [Ruby Regexp](https://ruby-doc.org/core/Regexp.html)
+is supported. [RE2](https://github.com/google/re2/wiki/Syntax) limits the set of features
+provided due to computational complexity, which means some features became unavailable in GitLab 11.9.4.
+For example, negative lookaheads.
+
+For GitLab versions from 11.9.7 and up to GitLab 12.0, GitLab provides a feature flag that can be
+enabled by administrators that allows users to use unsafe regexp syntax. This brings compatibility
+with previously allowed syntax version and allows users to gracefully migrate to the new syntax.
+
+```ruby
+Feature.enable(:allow_unsafe_ruby_regexp)
+```
+
 ### `only`/`except` (advanced)
 
 > - `refs` and `kubernetes` policies introduced in GitLab 10.0.

doc/integration/README.md

@@ -29,8 +29,8 @@ See the documentation below for details on how to configure these services.
 
 ## Project services
 
-Integration with services such as Campfire, Flowdock, Pivotal Tracker, and Slack
-are available in the form of a [Project Service][].
+Integration with services such as Campfire, Flowdock, HipChat,
+Pivotal Tracker, and Slack are available in the form of a [Project Service][].
 
 [Project Service]: ../user/project/integrations/project_services.md
 

doc/project_services/hipchat.md

+This document was moved to [user/project/integrations/hipchat.md](../user/project/integrations/hipchat.md).

doc/university/glossary/README.md

@@ -41,7 +41,7 @@ Objects (usually binary and large) created by a build process. These can include
 
 ### Atlassian
 
-A [company](https://www.atlassian.com) that develops software products for developers and project managers including Bitbucket, Jira, Confluence, Bamboo.
+A [company](https://www.atlassian.com) that develops software products for developers and project managers including Bitbucket, Jira, Hipchat, Confluence, Bamboo.
 
 ### Audit Log
 

doc/user/index.md

@@ -65,9 +65,7 @@ With GitLab Enterprise Edition, you can also:
 - View the current health and status of each CI environment running on Kubernetes with [Deploy Boards](https://docs.gitlab.com/ee/user/project/deploy_boards.html)
 - Leverage continuous delivery method with [Canary Deployments](https://docs.gitlab.com/ee/user/project/canary_deployments.html)
 
-You can also [integrate](project/integrations/project_services.md) GitLab with
-numerous third-party applications, such as Mattermost, Microsoft Teams, Trello,
-Slack, Bamboo CI, JIRA, and a lot more.
+You can also [integrate](project/integrations/project_services.md) GitLab with numerous third-party applications, such as Mattermost, Microsoft Teams, HipChat, Trello, Slack, Bamboo CI, JIRA, and a lot more.
 
 ## Projects
 

doc/user/project/integrations/hipchat.md

+# Atlassian HipChat
+
+GitLab provides a way to send HipChat notifications upon a number of events,
+such as when a user pushes code, creates a branch or tag, adds a comment, and
+creates a merge request.
+
+## Setup
+
+GitLab requires the use of a HipChat v2 API token to work. v1 tokens are
+not supported at this time. Note the differences between v1 and v2 tokens:
+
+HipChat v1 API (legacy) supports "API Auth Tokens" in the Group API menu. A v1
+token is allowed to send messages to *any* room.
+
+HipChat v2 API has tokens that are can be created using the Integrations tab
+in the Group or Room admin page. By design, these are lightweight tokens that
+allow GitLab to send messages only to *one* room.
+
+### Complete these steps in HipChat
+
+1. Go to: <https://admin.hipchat.com/admin>
+1. Click on "Group Admin" -> "Integrations".
+1. Find "Build Your Own!" and click "Create".
+1. Select the desired room, name the integration "GitLab", and click "Create".
+1. In the "Send messages to this room by posting this URL" column, you should
+see a URL in the format:
+
+```
+https://api.hipchat.com/v2/room/<room>/notification?auth_token=<token>
+```
+
+HipChat is now ready to accept messages from GitLab. Next, set up the HipChat
+service in GitLab.
+
+### Complete these steps in GitLab
+
+1. Navigate to the project you want to configure for notifications.
+1. Navigate to the [Integrations page](project_services.md#accessing-the-project-services)
+1. Click "HipChat".
+1. Select the "Active" checkbox.
+1. Insert the `token` field from the URL into the `Token` field on the Web page.
+1. Insert the `room` field from the URL into the `Room` field on the Web page.
+1. Save or optionally click "Test Settings".
+
+## Troubleshooting
+
+If you do not see notifications, make sure you are using a HipChat v2 API
+token, not a v1 token.
+
+Note that the v2 token is tied to a specific room. If you want to be able to
+specify arbitrary rooms, you can create an API token for a specific user in
+HipChat under "Account settings" and "API access". Use the `XXX` value under
+`auth_token=XXX`.

doc/user/project/integrations/project_services.md

@@ -36,6 +36,7 @@ Click on the service links to see further configuration instructions and details
 | External Wiki | Replaces the link to the internal wiki with a link to an external wiki |
 | Flowdock | Flowdock is a collaboration web app for technical teams |
 | [Hangouts Chat](hangouts_chat.md) | Receive events notifications in Google Hangouts Chat |
+| [HipChat](hipchat.md) | Private group chat and IM |
 | [Irker (IRC gateway)](irker.md) | Send IRC messages, on update, to a list of recipients through an Irker gateway |
 | [JIRA](jira.md) | JIRA issue tracker |
 | JetBrains TeamCity CI | A continuous integration and build server |

lib/api/services.rb

+# coding: utf-8
 # frozen_string_literal: true
 module API
   class Services < Grape::API
@@ -371,6 +372,44 @@ module API
         },
         CHAT_NOTIFICATION_EVENTS
       ].flatten,
+      'hipchat' => [
+        {
+          required: true,
+          name: :token,
+          type: String,
+          desc: 'The room token'
+        },
+        {
+          required: false,
+          name: :room,
+          type: String,
+          desc: 'The room name or ID'
+        },
+        {
+          required: false,
+          name: :color,
+          type: String,
+          desc: 'The room color'
+        },
+        {
+          required: false,
+          name: :notify,
+          type: Boolean,
+          desc: 'Enable notifications'
+        },
+        {