Merge branch 'issue_1008_1' into 'master'

Jira refactoring to jira gem

part of gitlab-org/gitlab-ee#1008

Original MR: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/2572/commits

See merge request !6598

CHANGELOG.md

@@ -9,6 +9,7 @@ Please view this file on the master branch, on stable branches it's out of date.
   - Fix extra space on Build sidebar on Firefox !7060
   - Fix mobile layout issues in admin user overview page !7087
   - Fix HipChat notifications rendering (airatshigapov, eisnerd)
+  - Refactor Jira service to use jira-ruby gem
   - Add hover to trash icon in notes !7008 (blackst0ne)
   - Fix sidekiq stats in admin area (blackst0ne)
   - Removed delete branch tooltip !6954

Gemfile

@@ -161,6 +161,9 @@ gem 'connection_pool', '~> 2.0'
 # HipChat integration
 gem 'hipchat', '~> 1.5.0'
 
+# JIRA integration
+gem 'jira-ruby', '~> 1.1.2'
+
 # Flowdock integration
 gem 'gitlab-flowdock-git-hook', '~> 1.0.1'
 

Gemfile.lock

@@ -356,6 +356,9 @@ GEM
       cause
       json
     ipaddress (0.8.3)
+    jira-ruby (1.1.2)
+      activesupport
+      oauth (~> 0.5, >= 0.5.0)
     jquery-atwho-rails (1.3.2)
     jquery-rails (4.1.1)
       rails-dom-testing (>= 1, < 3)
@@ -421,7 +424,7 @@ GEM
       mini_portile2 (~> 2.1.0)
       pkg-config (~> 1.1.7)
     numerizer (0.1.1)
-    oauth (0.4.7)
+    oauth (0.5.1)
     oauth2 (1.2.0)
       faraday (>= 0.8, < 0.10)
       jwt (~> 1.0)
@@ -881,6 +884,7 @@ DEPENDENCIES
   html-pipeline (~> 1.11.0)
   httparty (~> 0.13.3)
   influxdb (~> 0.2)
+  jira-ruby (~> 1.1.2)
   jquery-atwho-rails (~> 1.3.2)
   jquery-rails (~> 4.1.0)
   jquery-turbolinks (~> 2.1.0)

app/controllers/concerns/service_params.rb

@@ -18,7 +18,7 @@ module ServiceParams
                     :add_pusher, :send_from_committer_email, :disable_diffs,
                     :external_wiki_url, :notify, :color,
                     :server_host, :server_port, :default_irc_uri, :enable_ssl_verification,
-                    :jira_issue_transition_id]
+                    :jira_issue_transition_id, :url, :project_key]
 
   # Parameters to ignore if no value is specified
   FILTER_BLANK_PARAMS = [:password]

app/models/project_services/bugzilla_service.rb

 class BugzillaService < IssueTrackerService
+  validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated?
+
   prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url
 
   def title

app/models/project_services/custom_issue_tracker_service.rb

 class CustomIssueTrackerService < IssueTrackerService
+  validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated?
+
   prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url
 
   def title

app/models/project_services/gitlab_issue_tracker_service.rb

 class GitlabIssueTrackerService < IssueTrackerService
   include Gitlab::Routing.url_helpers
 
+  validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated?
+
   prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url
 
   default_value_for :default, true

app/models/project_services/issue_tracker_service.rb

 class IssueTrackerService < Service
-  validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated?
-
   default_value_for :category, 'issue_tracker'
 
   # Pattern used to extract links from comments
@@ -38,18 +36,24 @@ def fields
     ]
   end
 
-  def initialize_properties
-    if properties.nil?
-      if enabled_in_gitlab_config
+  # Initialize with default properties values
+  # or receive a block with custom properties
+  def initialize_properties(&block)
+    return unless properties.nil?
+
+    if enabled_in_gitlab_config
+      if block_given?
+        yield
+      else
         self.properties = {
           title: issues_tracker['title'],
           project_url: issues_tracker['project_url'],
           issues_url: issues_tracker['issues_url'],
           new_issue_url: issues_tracker['new_issue_url']
         }
-      else
-        self.properties = {}
       end
+    else
+      self.properties = {}
     end
   end
 

app/models/project_services/jira_service.rb

+# == Schema Information
+#
+# Table name: services
+#
+#  id                    :integer          not null, primary key
+#  type                  :string(255)
+#  title                 :string(255)
+#  project_id            :integer
+#  created_at            :datetime
+#  updated_at            :datetime
+#  active                :boolean          default(FALSE), not null
+#  properties            :text
+#  template              :boolean          default(FALSE)
+#  push_events           :boolean          default(TRUE)
+#  issues_events         :boolean          default(TRUE)
+#  merge_requests_events :boolean          default(TRUE)
+#  tag_push_events       :boolean          default(TRUE)
+#  note_events           :boolean          default(TRUE), not null
+#  build_events          :boolean          default(FALSE), not null
+#
+
 class JiraService < IssueTrackerService
-  include HTTParty
   include Gitlab::Routing.url_helpers
 
-  DEFAULT_API_VERSION = 2
-
-  prop_accessor :username, :password, :api_url, :jira_issue_transition_id,
-                :title, :description, :project_url, :issues_url, :new_issue_url
+  validates :url, url: true, presence: true, if: :activated?
+  validates :project_key, presence: true, if: :activated?
 
-  validates :api_url, presence: true, url: true, if: :activated?
-
-  before_validation :set_api_url, :set_jira_issue_transition_id
+  prop_accessor :username, :password, :url, :project_key,
+                :jira_issue_transition_id, :title, :description
 
   before_update :reset_password
 
@@ -18,16 +35,46 @@ def reference_pattern
     @reference_pattern ||= %r{(?<issue>\b([A-Z][A-Z0-9_]+-)\d+)}
   end
 
+  def initialize_properties
+    super do
+      self.properties = {
+        title: issues_tracker['title'],
+        url: issues_tracker['url']
+      }
+    end
+  end
+
   def reset_password
     # don't reset the password if a new one is provided
-    if api_url_changed? && !password_touched?
+    if url_changed? && !password_touched?
       self.password = nil
     end
   end
 
+  def options
+    url = URI.parse(self.url)
+
+    {
+      username: self.username,
+      password: self.password,
+      site: URI.join(url, '/').to_s,
+      context_path: url.path,
+      auth_type: :basic,
+      read_timeout: 120,
+      use_ssl: url.scheme == 'https'
+    }
+  end
+
+  def client
+    @client ||= JIRA::Client.new(options)
+  end
+
+  def jira_project
+    @jira_project ||= client.Project.find(project_key)
+  end
+
   def help
-    'Setting `project_url`, `issues_url` and `new_issue_url` will '\
-    'allow a user to easily navigate to the Jira issue tracker. See the '\
+    'See the ' \
     '[integration doc](http://doc.gitlab.com/ce/integration/external-issue-tracker.html) '\
     'for details.'
   end
@@ -53,12 +100,26 @@ def to_param
   end
 
   def fields
-    super.push(
-      { type: 'text', name: 'api_url', placeholder: 'https://jira.example.com/rest/api/2' },
+    [
+      { type: 'text', name: 'url', title: 'URL', placeholder: 'https://jira.example.com' },
+      { type: 'text', name: 'project_key', placeholder: 'Project Key' },
       { type: 'text', name: 'username', placeholder: '' },
       { type: 'password', name: 'password', placeholder: '' },
       { type: 'text', name: 'jira_issue_transition_id', placeholder: '2' }
-    )
+    ]
+  end
+
+  # URLs to redirect from Gitlab issues pages to jira issue tracker
+  def project_url
+    "#{url}/issues/?jql=project=#{project_key}"
+  end
+
+  def issues_url
+    "#{url}/browse/:id"
+  end
+
+  def new_issue_url
+    "#{url}/secure/CreateIssue.jspa"
   end
 
   def execute(push, issue = nil)
@@ -72,7 +133,7 @@ def execute(push, issue = nil)
   end
 
   def create_cross_reference_note(mentioned, noteable, author)
-    issue_name = mentioned.id
+    issue_key = mentioned.id
     project = self.project
     noteable_name = noteable.class.name.underscore.downcase
     noteable_id = if noteable.is_a?(Commit)
@@ -99,58 +160,28 @@ def create_cross_reference_note(mentioned, noteable, author)
       }
     }
 
-    add_comment(data, issue_name)
+    add_comment(data, issue_key)
   end
 
   def test_settings
-    return unless api_url.present?
-    result = JiraService.get(
-      jira_api_test_url,
-      headers: {
-        'Content-Type' => 'application/json',
-        'Authorization' => "Basic #{auth}"
-      }
-    )
+    return unless url.present?
+    # Test settings by getting the project
+    jira_project
 
-    case result.code
-    when 201, 200
-      Rails.logger.info("#{self.class.name} SUCCESS #{result.code}: Successfully connected to #{api_url}.")
-      true
-    else
-      Rails.logger.info("#{self.class.name} ERROR #{result.code}: #{result.parsed_response}")
-      false
-    end
-  rescue Errno::ECONNREFUSED => e
-    Rails.logger.info "#{self.class.name} ERROR: #{e.message}. API URL: #{api_url}."
+  rescue Errno::ECONNREFUSED, JIRA::HTTPError => e
+    Rails.logger.info "#{self.class.name} ERROR: #{e.message}. API URL: #{url}."
     false
   end
 
   private
 
-  def build_api_url_from_project_url
-    server = URI(project_url)
-    default_ports = [["http", 80], ["https", 443]].include?([server.scheme, server.port])
-    server_url = "#{server.scheme}://#{server.host}"
-    server_url.concat(":#{server.port}") unless default_ports
-    "#{server_url}/rest/api/#{DEFAULT_API_VERSION}"
-  rescue
-    "" # looks like project URL was not valid
-  end
-
-  def set_api_url
-    self.api_url = build_api_url_from_project_url if self.api_url.blank?
-  end
-
-  def set_jira_issue_transition_id
-    self.jira_issue_transition_id ||= "2"
-  end
-
   def close_issue(entity, issue)
     commit_id = if entity.is_a?(Commit)
                   entity.id
                 elsif entity.is_a?(MergeRequest)
                   entity.diff_head_sha
                 end
+
     commit_url = build_entity_url(:commit, commit_id)
 
     # Depending on the JIRA project's workflow, a comment during transition
@@ -161,24 +192,16 @@ def close_issue(entity, issue)
   end
 
   def transition_issue(issue)
-    message = {
-      transition: {
-        id: jira_issue_transition_id
-      }
-    }
-    send_message(close_issue_url(issue.iid), message.to_json)
+    issue = client.Issue.find(issue.iid)
+    issue.transitions.build.save(transition: { id: jira_issue_transition_id })
   end
 
   def add_issue_solved_comment(issue, commit_id, commit_url)
-    comment = {
-      body: "Issue solved with [#{commit_id}|#{commit_url}]."
-    }
-
-    send_message(comment_url(issue.iid), comment.to_json)
+    comment = "Issue solved with [#{commit_id}|#{commit_url}]."
+    send_message(issue.iid, comment)
   end
 
-  def add_comment(data, issue_name)
-    url = comment_url(issue_name)
+  def add_comment(data, issue_key)
     user_name = data[:user][:name]
     user_url = data[:user][:url]
     entity_name = data[:entity][:name]
@@ -186,68 +209,31 @@ def add_comment(data, issue_name)
     entity_title = data[:entity][:title]
     project_name = data[:project][:name]
 
-    message = {
-      body: %Q{[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]:\n'#{entity_title}'}
-    }
+    message = "[#{user_name}|#{user_url}] mentioned this issue in [a #{entity_name} of #{project_name}|#{entity_url}]:\n'#{entity_title}'"
 
-    unless existing_comment?(issue_name, message[:body])
-      send_message(url, message.to_json)
+    unless comment_exists?(issue_key, message)
+      send_message(issue_key, message)
     end
   end
 
-  def auth
-    require 'base64'
-    Base64.urlsafe_encode64("#{self.username}:#{self.password}")
-  end
-
-  def send_message(url, message)
-    return unless api_url.present?
-    result = JiraService.post(
-      url,
-      body: message,
-      headers: {
-        'Content-Type' => 'application/json',
-        'Authorization' => "Basic #{auth}"
-      }
-    )
-
-    message = case result.code
-              when 201, 200, 204
-                "#{self.class.name} SUCCESS #{result.code}: Successfully posted to #{url}."
-              when 401
-                "#{self.class.name} ERROR 401: Unauthorized. Check the #{self.username} credentials and JIRA access permissions and try again."
-              else
-                "#{self.class.name} ERROR #{result.code}: #{result.parsed_response}"
-              end
-
-    Rails.logger.info(message)
-    message
-  rescue URI::InvalidURIError, Errno::ECONNREFUSED => e
-    Rails.logger.info "#{self.class.name} ERROR: #{e.message}. Hostname: #{url}."
+  def comment_exists?(issue_key, message)
+    comments = client.Issue.find(issue_key).comments
+    comments.map { |comment| comment.body.include?(message) }.any?
   end
 
-  def existing_comment?(issue_name, new_comment)
-    return unless api_url.present?
-    result = JiraService.get(
-      comment_url(issue_name),
-      headers: {
-        'Content-Type' => 'application/json',
-        'Authorization' => "Basic #{auth}"
-      }
-    )
+  def send_message(issue_key, message)
+    return unless url.present?
 
-    case result.code
-    when 201, 200
-      existing_comments = JSON.parse(result.body)['comments']
+    issue = client.Issue.find(issue_key)
 
-      if existing_comments.present?
-        return existing_comments.map { |comment| comment['body'].include?(new_comment) }.any?
-      end
+    if issue.comments.build.save!(body: message)
+      result_message = "#{self.class.name} SUCCESS: Successfully posted to #{url}."
     end
 
-    false
-  rescue JSON::ParserError
-    false
+    Rails.logger.info(result_message)
+    result_message
+  rescue URI::InvalidURIError, Errno::ECONNREFUSED, JIRA::HTTPError => e
+    Rails.logger.info "#{self.class.name} Send message ERROR: #{url} - #{e.message}"
   end
 
   def resource_url(resource)
@@ -267,16 +253,4 @@ def build_entity_url(entity_name, entity_id)
       )
     )
   end
-
-  def close_issue_url(issue_name)
-    "#{self.api_url}/issue/#{issue_name}/transitions"
-  end
-
-  def comment_url(issue_name)
-    "#{self.api_url}/issue/#{issue_name}/comment"
-  end
-
-  def jira_api_test_url
-    "#{self.api_url}/myself"
-  end
 end

app/models/project_services/redmine_service.rb

 class RedmineService < IssueTrackerService
+  validates :project_url, :issues_url, :new_issue_url, presence: true, url: true, if: :activated?
+
   prop_accessor :title, :description, :project_url, :issues_url, :new_issue_url
 
   def title

config/gitlab.yml.example

@@ -547,6 +547,10 @@ test:
       project_url: "http://redmine/projects/:issues_tracker_id"
       issues_url: "http://redmine/:project_id/:issues_tracker_id/:id"
       new_issue_url: "http://redmine/projects/:issues_tracker_id/issues/new"
+    jira:
+      title: "JIRA"
+      url: https://sample_company.atlasian.net
+      project_key: PROJECT
   ldap:
     enabled: false
     servers:

db/migrate/20161011222551_remove_inactive_jira_service_properties.rb

+class RemoveInactiveJiraServiceProperties < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = true
+  DOWNTIME_REASON = "Removes all inactive jira_service properties"
+
+  def up
+    execute("UPDATE services SET properties = '{}' WHERE services.type = 'JiraService' and services.active = false")
+  end
+end

db/migrate/20161025231710_migrate_jira_to_gem.rb

+class MigrateJiraToGem < ActiveRecord::Migration
+  DOWNTIME = true
+
+  DOWNTIME_REASON = <<-HEREDOC
+    Refactor all Jira services properties(serialized field) to use new jira-ruby gem.
+    There were properties on old Jira service that are not needed anymore after the
+    service refactoring: api_url, project_url, new_issue_url, issues_url.
+    We extract the new necessary some properties from old keys and delete them:
+    taking project_key from project_url and url from api_url
+  HEREDOC
+
+  def up
+    active_services_query = "SELECT id, properties FROM services WHERE services.type IN ('JiraService') AND services.active = true"
+
+    select_all(active_services_query).each do |service|
+      id = service['id']
+      properties = JSON.parse(service['properties'])
+      properties_was = properties.clone
+
+      # Migrate `project_url` to `project_key`
+      # Ignore if `project_url` doesn't have jql project query with project key
+      if properties['project_url'].present?
+        jql = properties['project_url'].match('project=([A-Za-z]*)')
+        properties['project_key'] = jql.captures.first if jql
+      end
+
+      # Migrate `api_url` to `url`
+      if properties['api_url'].present?
+        url = properties['api_url'].match('(.*)\/rest\/api')
+        properties['url'] = url.captures.first if url
+      end
+
+      # Delete now unnecessary properties
+      properties.delete('api_url')
+      properties.delete('project_url')
+      properties.delete('new_issue_url')
+      properties.delete('issues_url')
+
+      # Update changes properties
+      if properties != properties_was
+        execute("UPDATE services SET properties = '#{quote_string(properties.to_json)}' WHERE id = #{id}")
+      end
+    end
+  end
+
+  def down
+    active_services_query = "SELECT id, properties FROM services WHERE services.type IN ('JiraService') AND services.active = true"
+
+    select_all(active_services_query).each do |service|
+      id = service['id']
+      properties = JSON.parse(service['properties'])
+      properties_was = properties.clone
+
+      # Rebuild old properties based on sane defaults
+      if properties['url'].present?
+        properties['api_url'] = "#{properties['url']}/rest/api/2"
+        properties['project_url'] =
+          "#{properties['url']}/issues/?jql=project=#{properties['project_key']}"
+        properties['issues_url'] = "#{properties['url']}/browse/:id"
+        properties['new_issue_url'] = "#{properties['url']}/secure/CreateIssue.jspa"
+      end
+
+      # Delete the new properties
+      properties.delete('url')
+      properties.delete('project_key')
+
+      # Update changes properties
+      if properties != properties_was
+        execute("UPDATE services SET properties = '#{quote_string(properties.to_json)}' WHERE id = #{id}")
+      end
+    end
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20161024042317) do
+ActiveRecord::Schema.define(version: 20161025231710) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"

doc/integration/README.md

@@ -5,7 +5,7 @@ trackers and external authentication.
 
 See the documentation below for details on how to configure these services.
 
-- [Jira](../project_services/jira.md) Integrate with the JIRA issue tracker
+- [JIRA](jira.md) Integrate with the JIRA issue tracker
 - [External issue tracker](external-issue-tracker.md) Redmine, JIRA, etc.
 - [LDAP](ldap.md) Set up sign in via LDAP
 - [OmniAuth](omniauth.md) Sign in via Twitter, GitHub, GitLab.com, Google, Bitbucket, Facebook, Shibboleth, SAML, Crowd and Azure

doc/integration/jira.md

 # GitLab JIRA integration
 
-This document was moved under [project_services/jira](../project_services/jira.md).
+GitLab can be configured to interact with JIRA. Configuration happens via
+user name and password. Connecting to a JIRA server via CAS is not possible.
+
+Each project can be configured to connect to a different JIRA instance, see the
+[configuration](#configuration) section. If you have one JIRA instance you can
+pre-fill the settings page with a default template. To configure the template
+see the [Services Templates][services-templates] document.
+
+Once the project is connected to JIRA, you can reference and close the issues
+in JIRA directly from GitLab.
+
+## Table of Contents
+* [Referencing JIRA Issues from GitLab](#referencing-JIRA-issues)
+* [Closing JIRA Issues from GitLab](#closing-JIRA-issues)
+* [Configuration](#configuration)
+
+### Referencing JIRA Issues
+
+When GitLab project has JIRA issue tracker configured and enabled, mentioning
+JIRA issue in GitLab will automatically add a comment in JIRA issue with the
+link back to GitLab. This means that in comments in merge requests and commits
+referencing an issue, eg. `PROJECT-7`, will add a comment in JIRA issue in the
+format:
+
+```
+ USER mentioned this issue in RESOURCE_NAME of [PROJECT_NAME|LINK_TO_COMMENT]: