Migrate CI::Services and CI::WebHooks to Services and WebHooks

CHANGELOG

@@ -16,6 +16,7 @@ v 8.3.0 (unreleased)
   - Fire update hook from GitLab
   - Style warning about mentioning many people in a comment
   - Fix: sort milestones by due date once again (Greg Smethells)
+  - Migrate all CI::Services and CI::WebHooks to Services and WebHooks
   - Don't show project fork event as "imported"
   - Add API endpoint to fetch merge request commits list
   - Expose events API with comment information and author info

app/controllers/projects/ci_services_controller.rb

-class Projects::CiServicesController < Projects::ApplicationController
-  before_action :ci_project
-  before_action :authorize_admin_project!
-
-  layout "project_settings"
-
-  def index
-    @ci_project.build_missing_services
-    @services = @ci_project.services.reload
-  end
-
-  def edit
-    service
-  end
-
-  def update
-    if service.update_attributes(service_params)
-      redirect_to edit_namespace_project_ci_service_path(@project.namespace, @project, service.to_param)
-    else
-      render 'edit'
-    end
-  end
-
-  def test
-    last_build = @project.ci_builds.last
-
-    if service.execute(last_build)
-      message = { notice: 'We successfully tested the service' }
-    else
-      message = { alert: 'We tried to test the service but error occurred' }
-    end
-
-    redirect_back_or_default(options: message)
-  end
-
-  private
-
-  def service
-    @service ||= @ci_project.services.find { |service| service.to_param == params[:id] }
-  end
-
-  def service_params
-    params.require(:service).permit(
-      :type, :active, :webhook, :notify_only_broken_builds,
-      :email_recipients, :email_only_broken_builds, :email_add_pusher,
-      :hipchat_token, :hipchat_room, :hipchat_server
-    )
-  end
-end

app/controllers/projects/ci_web_hooks_controller.rb

-class Projects::CiWebHooksController < Projects::ApplicationController
-  before_action :ci_project
-  before_action :authorize_admin_project!
-
-  layout "project_settings"
-
-  def index
-    @web_hooks = @ci_project.web_hooks
-    @web_hook = Ci::WebHook.new
-  end
-
-  def create
-    @web_hook = @ci_project.web_hooks.new(web_hook_params)
-    @web_hook.save
-
-    if @web_hook.valid?
-      redirect_to namespace_project_ci_web_hooks_path(@project.namespace, @project)
-    else
-      @web_hooks = @ci_project.web_hooks.select(&:persisted?)
-      render :index
-    end
-  end
-
-  def test
-    Ci::TestHookService.new.execute(hook, current_user)
-
-    redirect_back_or_default(default: { action: 'index' })
-  end
-
-  def destroy
-    hook.destroy
-
-    redirect_to namespace_project_ci_web_hooks_path(@project.namespace, @project)
-  end
-
-  private
-
-  def hook
-    @web_hook ||= @ci_project.web_hooks.find(params[:id])
-  end
-
-  def web_hook_params
-    params.require(:web_hook).permit(:url)
-  end
-end

app/controllers/projects/hooks_controller.rb

@@ -53,6 +53,7 @@ def hook
 
   def hook_params
     params.require(:hook).permit(:url, :push_events, :issues_events,
-      :merge_requests_events, :tag_push_events, :note_events, :enable_ssl_verification)
+      :merge_requests_events, :tag_push_events, :note_events,
+      :build_events, :enable_ssl_verification)
   end
 end

app/controllers/projects/services_controller.rb

@@ -6,7 +6,9 @@ class Projects::ServicesController < Projects::ApplicationController
                     :description, :issues_url, :new_issue_url, :restrict_to_branch, :channel,
                     :colorize_messages, :channels,
                     :push_events, :issues_events, :merge_requests_events, :tag_push_events,
-                    :note_events, :send_from_committer_email, :disable_diffs, :external_wiki_url,
+                    :note_events, :build_events,
+                    :notify_only_broken_builds, :add_pusher,
+                    :send_from_committer_email, :disable_diffs, :external_wiki_url,
                     :notify, :color,
                     :server_host, :server_port, :default_irc_uri, :enable_ssl_verification]
 

app/mailers/ci/emails/builds.rb

-module Ci
-  module Emails
-    module Builds
-      def build_fail_email(build_id, to)
-        @build = Ci::Build.find(build_id)
-        @project = @build.project
-        mail(to: to, subject: subject("Build failed for #{@project.name}", @build.short_sha))
-      end
-
-      def build_success_email(build_id, to)
-        @build = Ci::Build.find(build_id)
-        @project = @build.project
-        mail(to: to, subject: subject("Build success for #{@project.name}", @build.short_sha))
-      end
-    end
-  end
-end

app/mailers/ci/notify.rb

-module Ci
-  class Notify < ActionMailer::Base
-    include Ci::Emails::Builds
-
-    add_template_helper Ci::GitlabHelper
-
-    default_url_options[:host]     = Gitlab.config.gitlab.host
-    default_url_options[:protocol] = Gitlab.config.gitlab.protocol
-    default_url_options[:port]     = Gitlab.config.gitlab.port unless Gitlab.config.gitlab_on_standard_port?
-    default_url_options[:script_name] = Gitlab.config.gitlab.relative_url_root
-
-    default from: Gitlab.config.gitlab.email_from
-
-    # Just send email with 3 seconds delay
-    def self.delay
-      delay_for(2.seconds)
-    end
-
-    private
-
-    # Formats arguments into a String suitable for use as an email subject
-    #
-    # extra - Extra Strings to be inserted into the subject
-    #
-    # Examples
-    #
-    #   >> subject('Lorem ipsum')
-    #   => "GitLab-CI | Lorem ipsum"
-    #
-    #   # Automatically inserts Project name when @project is set
-    #   >> @project = Project.last
-    #   => #<Project id: 1, name: "Ruby on Rails", path: "ruby_on_rails", ...>
-    #   >> subject('Lorem ipsum')
-    #   => "GitLab-CI | Ruby on Rails | Lorem ipsum "
-    #
-    #   # Accepts multiple arguments
-    #   >> subject('Lorem ipsum', 'Dolor sit amet')
-    #   => "GitLab-CI | Lorem ipsum | Dolor sit amet"
-    def subject(*extra)
-      subject = "GitLab-CI"
-      subject << (@project ? " | #{@project.name}" : "")
-      subject << " | " + extra.join(' | ') if extra.present?
-      subject
-    end
-  end
-end

app/mailers/emails/builds.rb

+module Emails
+  module Builds
+    def build_fail_email(build_id, to)
+      @build = Ci::Build.find(build_id)
+      @project = @build.project
+      mail(to: to, subject: subject("Build failed for #{@project.name}", @build.short_sha))
+    end
+
+    def build_success_email(build_id, to)
+      @build = Ci::Build.find(build_id)
+      @project = @build.project
+      mail(to: to, subject: subject("Build success for #{@project.name}", @build.short_sha))
+    end
+  end
+end

app/mailers/notify.rb

@@ -7,6 +7,7 @@ class Notify < BaseMailer
   include Emails::Projects
   include Emails::Profile
   include Emails::Groups
+  include Emails::Builds
 
   add_template_helper MergeRequestsHelper
   add_template_helper EmailsHelper

app/models/ci/build.rb

@@ -96,21 +96,21 @@ def retry(build)
     end
 
     state_machine :status, initial: :pending do
+      after_transition pending: :running do |build, transition|
+        build.execute_hooks
+      end
+
       after_transition any => [:success, :failed, :canceled] do |build, transition|
         return unless build.gl_project
 
         project = build.project
 
-        if project.web_hooks?
-          Ci::WebHookService.new.build_end(build)
-        end
-
-        build.commit.create_next_builds(build)
-        project.execute_services(build)
-
         if project.coverage_enabled?
           build.update_coverage
         end
+
+        build.commit.create_next_builds(build)
+        build.execute_hooks
       end
     end
 
@@ -275,6 +275,12 @@ def download_url
       end
     end
 
+    def execute_hooks
+      build_data = Gitlab::BuildDataBuilder.build(self)
+      gl_project.execute_hooks(build_data.dup, :build_hooks)
+      gl_project.execute_services(build_data.dup, :build_hooks)
+    end
+
     private
 
     def yaml_variables

app/models/ci/commit.rb

@@ -178,6 +178,10 @@ def duration
       duration_array.reduce(:+).to_i
     end
 
+    def started_at
+      @started_at ||= statuses.order('started_at ASC').first.try(:started_at)
+    end
+
     def finished_at
       @finished_at ||= statuses.order('finished_at DESC').first.try(:finished_at)
     end

app/models/ci/project.rb

@@ -35,17 +35,10 @@ class Project < ActiveRecord::Base
 
     has_many :runner_projects, dependent: :destroy, class_name: 'Ci::RunnerProject'
     has_many :runners, through: :runner_projects, class_name: 'Ci::Runner'
-    has_many :web_hooks, dependent: :destroy, class_name: 'Ci::WebHook'
     has_many :events, dependent: :destroy, class_name: 'Ci::Event'
     has_many :variables, dependent: :destroy, class_name: 'Ci::Variable'
     has_many :triggers, dependent: :destroy, class_name: 'Ci::Trigger'
 
-    # Project services
-    has_many :services, dependent: :destroy, class_name: 'Ci::Service'
-    has_one :hip_chat_service, dependent: :destroy, class_name: 'Ci::HipChatService'
-    has_one :slack_service, dependent: :destroy, class_name: 'Ci::SlackService'
-    has_one :mail_service, dependent: :destroy, class_name: 'Ci::MailService'
-
     accepts_nested_attributes_for :variables, allow_destroy: true
 
     delegate :name_with_namespace, :path_with_namespace, :web_url, :http_url_to_repo, :ssh_url_to_repo, to: :gl_project
@@ -122,14 +115,6 @@ def email_notification?
       email_add_pusher || email_recipients.present?
     end
 
-    def web_hooks?
-      web_hooks.any?
-    end
-
-    def services?
-      services.any?
-    end
-
     def timeout_in_minutes
       timeout / 60
     end
@@ -151,32 +136,6 @@ def repo_url_with_auth
       end
     end
 
-    def available_services_names
-      %w(slack mail hip_chat)
-    end
-
-    def build_missing_services
-      available_services_names.each do |service_name|
-        service = services.find { |service| service.to_param == service_name }
-
-        # If service is available but missing in db
-        # we should create an instance. Ex `create_gitlab_ci_service`
-        self.send :"create_#{service_name}_service" if service.nil?
-      end
-    end
-
-    def execute_services(data)
-      services.each do |service|
-
-        # Call service hook only if it is active
-        begin
-          service.execute(data) if service.active && service.can_execute?(data)
-        rescue => e
-          logger.error(e)
-        end
-      end
-    end
-
     def setup_finished?
       commits.any?
     end

app/models/ci/service.rb

-# == Schema Information
-#
-# Table name: ci_services
-#
-#  id         :integer          not null, primary key
-#  type       :string(255)
-#  title      :string(255)
-#  project_id :integer          not null
-#  created_at :datetime
-#  updated_at :datetime
-#  active     :boolean          default(FALSE), not null
-#  properties :text
-#
-
-# To add new service you should build a class inherited from Service
-# and implement a set of methods
-module Ci
-  class Service < ActiveRecord::Base
-    extend Ci::Model
-    
-    serialize :properties, JSON
-
-    default_value_for :active, false
-
-    after_initialize :initialize_properties
-
-    belongs_to :project, class_name: 'Ci::Project'
-
-    validates :project_id, presence: true
-
-    def activated?
-      active
-    end
-
-    def category
-      :common
-    end
-
-    def initialize_properties
-      self.properties = {} if properties.nil?
-    end
-
-    def title
-      # implement inside child
-    end
-
-    def description
-      # implement inside child
-    end
-
-    def help
-      # implement inside child
-    end
-
-    def to_param
-      # implement inside child
-    end
-
-    def fields
-      # implement inside child
-      []
-    end
-
-    def can_test?
-      project.builds.any?
-    end
-
-    def can_execute?(build)
-      true
-    end
-
-    def execute(build)
-      # implement inside child
-    end
-
-    # Provide convenient accessor methods
-    # for each serialized property.
-    def self.prop_accessor(*args)
-      args.each do |arg|
-        class_eval %{
-          def #{arg}
-            (properties || {})['#{arg}']
-          end
-
-          def #{arg}=(value)
-            self.properties ||= {}
-            self.properties['#{arg}'] = value
-          end
-        }
-      end
-    end
-
-    def self.boolean_accessor(*args)
-      self.prop_accessor(*args)
-
-      args.each do |arg|
-        class_eval %{
-          def #{arg}?
-            ActiveRecord::ConnectionAdapters::Column::TRUE_VALUES.include?(#{arg})
-          end
-        }
-      end
-    end
-  end
-end

app/models/ci/web_hook.rb

-# == Schema Information
-#
-# Table name: ci_web_hooks
-#
-#  id         :integer          not null, primary key
-#  url        :string(255)      not null
-#  project_id :integer          not null
-#  created_at :datetime
-#  updated_at :datetime
-#
-
-module Ci
-  class WebHook < ActiveRecord::Base
-    extend Ci::Model
-
-    include HTTParty
-
-    belongs_to :project, class_name: 'Ci::Project'
-
-    # HTTParty timeout
-    default_timeout 10
-
-    validates :url, presence: true, url: true
-
-    def execute(data)
-      parsed_url = URI.parse(url)
-      if parsed_url.userinfo.blank?
-        Ci::WebHook.post(url, body: data.to_json, headers: { "Content-Type" => "application/json" }, verify: false)
-      else
-        post_url = url.gsub("#{parsed_url.userinfo}@", "")
-        auth = {
-          username: URI.decode(parsed_url.user),
-          password: URI.decode(parsed_url.password),
-        }
-        Ci::WebHook.post(post_url,
-                     body: data.to_json,
-                     headers: { "Content-Type" => "application/json" },
-                     verify: false,
-                     basic_auth: auth)
-      end
-    end
-  end
-end

app/models/hooks/project_hook.rb

@@ -25,4 +25,5 @@ class ProjectHook < WebHook
   scope :issue_hooks, -> { where(issues_events: true) }
   scope :note_hooks, -> { where(note_events: true) }
   scope :merge_request_hooks, -> { where(merge_requests_events: true) }
+  scope :build_hooks, -> { where(build_events: true) }
 end

app/models/hooks/web_hook.rb

@@ -26,6 +26,7 @@ class WebHook < ActiveRecord::Base
   default_value_for :note_events, false
   default_value_for :merge_requests_events, false
   default_value_for :tag_push_events, false
+  default_value_for :build_events, false
   default_value_for :enable_ssl_verification, true
 
   # HTTParty timeout

app/models/project.rb

@@ -81,6 +81,7 @@ def set_last_activity_at
   has_one :campfire_service, dependent: :destroy
   has_one :drone_ci_service, dependent: :destroy
   has_one :emails_on_push_service, dependent: :destroy
+  has_one :builds_email_service, dependent: :destroy
   has_one :irker_service, dependent: :destroy
   has_one :pivotaltracker_service, dependent: :destroy
   has_one :hipchat_service, dependent: :destroy

app/models/project_services/builds_email_service.rb

+# == Schema Information
+#
+# Table name: services
+#
+#  id                    :integer          not null, primary key
+#  type                  :string(255)
+#  title                 :string(255)
+#  project_id            :integer
+#  created_at            :datetime
+#  updated_at            :datetime
+#  active                :boolean          default(FALSE), not null
+#  properties            :text
+#  template              :boolean          default(FALSE)
+#  push_events           :boolean          default(TRUE)
+#  issues_events         :boolean          default(TRUE)
+#  merge_requests_events :boolean          default(TRUE)
+#  tag_push_events       :boolean          default(TRUE)
+#  note_events           :boolean          default(TRUE), not null
+#
+
+class BuildsEmailService < Service
+  prop_accessor :recipients
+  boolean_accessor :add_pusher
+  boolean_accessor :notify_only_broken_builds
+  validates :recipients, presence: true, if: :activated?
+
+  def title
+    'Builds emails'
+  end
+
+  def description
+    'Email the builds status to a list of recipients.'
+  end
+
+  def to_param
+    'builds_email'
+  end
+
+  def supported_events
+    %w(build)
+  end
+
+  def execute(push_data)
+    return unless supported_events.include?(push_data[:object_kind])
+
+    if should_build_be_notified?(push_data)
+      BuildEmailWorker.perform_async(
+        push_data[:build_id],
+        all_recipients(push_data),
+        push_data,
+      )
+    end
+  end
+
+  def fields
+    [
+      { type: 'textarea', name: 'recipients', placeholder: 'Emails separated by whitespace' },
+      { type: 'checkbox', name: 'add_pusher', label: 'Add pusher to recipients list' },
+      { type: 'checkbox', name: 'notify_only_broken_builds' },
+    ]
+  end
+
+  def should_build_be_notified?(data)
+    case data[:build_status]
+    when 'success'
+      !notify_only_broken_builds?
+    when 'failed'
+      true
+    else
+      false
+    end
+  end
+
+  def all_recipients(data)
+    if add_pusher? && data[:user][:email]
+      recipients + " #{data[:user][:email]}"
+    else
+      recipients
+    end
+  end
+end

app/models/project_services/ci/hip_chat_message.rb

-module Ci
-  class HipChatMessage
-    include Gitlab::Application.routes.url_helpers
-
-    attr_reader :build
-
-    def initialize(build)
-      @build = build
-    end
-
-    def to_s
-      lines = Array.new
-      lines.push("<a href=\"#{ci_project_url(project)}\">#{project.name}</a> - ")
-      lines.push("<a href=\"#{builds_namespace_project_commit_url(commit.gl_project.namespace, commit.gl_project, commit.sha)}\">Commit ##{commit.id}</a></br>")
-      lines.push("#{commit.short_sha} #{commit.git_author_name} - #{commit.git_commit_message}</br>")
-      lines.push("#{humanized_status(commit_status)} in #{commit.duration} second(s).")
-      lines.join('')
-    end
-
-    def status_color(build_or_commit=nil)
-      build_or_commit ||= commit_status