Commit 9a130593 authored by Tiago Botelho's avatar Tiago Botelho

Backports every CE related change from ee-5484 to CE

parent 2d84de9e
class Projects::MirrorsController < Projects::ApplicationController
include RepositorySettingsRedirect
# Authorize
before_action :authorize_admin_mirror!
before_action :remote_mirror, only: [:update]
layout "project_settings"
def show
redirect_to_repository_settings(project)
end
def update
if project.update_attributes(mirror_params)
flash[:notice] = 'Mirroring settings were successfully updated.'
else
flash[:alert] = project.errors.full_messages.join(', ').html_safe
end
respond_to do |format|
format.html { redirect_to_repository_settings(project) }
format.json do
if project.errors.present?
render json: project.errors, status: :unprocessable_entity
else
render json: ProjectMirrorSerializer.new.represent(project)
end
end
end
end
def update_now
if params[:sync_remote]
project.update_remote_mirrors
flash[:notice] = "The remote repository is being updated..."
end
redirect_to_repository_settings(project)
end
private
def remote_mirror
@remote_mirror = project.remote_mirrors.first_or_initialize
end
def mirror_params_attributes
[
remote_mirrors_attributes: %i[
url
id
enabled
only_protected_branches
]
]
end
def mirror_params
params.require(:project).permit(mirror_params_attributes)
end
end
......@@ -2,6 +2,7 @@ module Projects
module Settings
class RepositoryController < Projects::ApplicationController
before_action :authorize_admin_project!
before_action :remote_mirror, only: [:show]
def show
render_show
......@@ -25,6 +26,7 @@ module Projects
define_deploy_token
define_protected_refs
remote_mirror
render 'show'
end
......@@ -41,6 +43,12 @@ module Projects
load_gon_index
end
def remote_mirror
return unless project.feature_available?(:repository_mirrors)
@remote_mirror = project.remote_mirrors.first_or_initialize
end
def access_levels_options
{
create_access_levels: levels_for_dropdown,
......
......@@ -64,6 +64,9 @@ class Project < ActiveRecord::Base
default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
add_authentication_token_field :runners_token
before_validation :mark_remote_mirrors_for_removal
before_save :ensure_runners_token
after_save :update_project_statistics, if: :namespace_id_changed?
......@@ -241,11 +244,17 @@ class Project < ActiveRecord::Base
has_many :project_badges, class_name: 'ProjectBadge'
has_one :ci_cd_settings, class_name: 'ProjectCiCdSetting', inverse_of: :project, autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
has_many :remote_mirrors, inverse_of: :project
accepts_nested_attributes_for :variables, allow_destroy: true
accepts_nested_attributes_for :project_feature, update_only: true
accepts_nested_attributes_for :import_data
accepts_nested_attributes_for :auto_devops, update_only: true
accepts_nested_attributes_for :remote_mirrors,
allow_destroy: true,
reject_if: ->(attrs) { attrs[:id].blank? && attrs[:url].blank? }
delegate :name, to: :owner, allow_nil: true, prefix: true
delegate :members, to: :team, prefix: true
delegate :add_user, :add_users, to: :team
......@@ -335,6 +344,7 @@ class Project < ActiveRecord::Base
scope :with_issues_enabled, -> { with_feature_enabled(:issues) }
scope :with_issues_available_for_user, ->(current_user) { with_feature_available_for_user(:issues, current_user) }
scope :with_merge_requests_enabled, -> { with_feature_enabled(:merge_requests) }
scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
scope :with_group_runners_enabled, -> do
joins(:ci_cd_settings)
......@@ -754,6 +764,37 @@ class Project < ActiveRecord::Base
import_type == 'gitea'
end
def has_remote_mirror?
remote_mirror_available? && remote_mirrors.enabled.exists?
end
def updating_remote_mirror?
remote_mirrors.enabled.started.exists?
end
def update_remote_mirrors
return unless remote_mirror_available?
remote_mirrors.enabled.each(&:sync)
end
def mark_stuck_remote_mirrors_as_failed!
remote_mirrors.stuck.update_all(
update_status: :failed,
last_error: 'The remote mirror took to long to complete.',
last_update_at: Time.now
)
end
def mark_remote_mirrors_for_removal
remote_mirrors.each(&:mark_for_delete_if_blank_url)
end
def remote_mirror_available?
remote_mirror_available_overridden ||
::Gitlab::CurrentSettings.mirror_available
end
def check_limit
unless creator.can_create_project? || namespace.kind == 'group'
projects_limit = creator.projects_limit
......
class RemoteMirror < ActiveRecord::Base
include AfterCommitQueue
PROTECTED_BACKOFF_DELAY = 1.minute
UNPROTECTED_BACKOFF_DELAY = 5.minutes
attr_encrypted :credentials,
key: Gitlab::Application.secrets.db_key_base,
marshal: true,
encode: true,
mode: :per_attribute_iv_and_salt,
insecure_mode: true,
algorithm: 'aes-256-cbc'
default_value_for :only_protected_branches, true
belongs_to :project, inverse_of: :remote_mirrors
validates :url, presence: true, url: { protocols: %w(ssh git http https), allow_blank: true }
validates :url, addressable_url: true, if: :url_changed?
before_save :set_new_remote_name, if: :mirror_url_changed?
after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available }
after_save :refresh_remote, if: :mirror_url_changed?
after_update :reset_fields, if: :mirror_url_changed?
after_commit :remove_remote, on: :destroy
scope :enabled, -> { where(enabled: true) }
scope :started, -> { with_update_status(:started) }
scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }
state_machine :update_status, initial: :none do
event :update_start do
transition [:none, :finished, :failed] => :started
end
event :update_finish do
transition started: :finished
end
event :update_fail do
transition started: :failed
end
state :started
state :finished
state :failed
after_transition any => :started do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_running, path: remote_mirror.project.full_path)
remote_mirror.update(last_update_started_at: Time.now)
end
after_transition started: :finished do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_finished, path: remote_mirror.project.full_path)
timestamp = Time.now
remote_mirror.update_attributes!(
last_update_at: timestamp, last_successful_update_at: timestamp, last_error: nil
)
end
after_transition started: :failed do |remote_mirror, _|
Gitlab::Metrics.add_event(:remote_mirrors_failed, path: remote_mirror.project.full_path)
remote_mirror.update(last_update_at: Time.now)
end
end
def remote_name
super || fallback_remote_name
end
def update_failed?
update_status == 'failed'
end
def update_in_progress?
update_status == 'started'
end
def update_repository(options)
raw.update(options)
end
def sync
return unless enabled?
return if Gitlab::Geo.secondary?
if recently_scheduled?
RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
else
RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now)
end
end
def enabled
return false unless project && super
return false unless project.remote_mirror_available?
return false unless project.repository_exists?
return false if project.pending_delete?
true
end
alias_method :enabled?, :enabled
def updated_since?(timestamp)
last_update_started_at && last_update_started_at > timestamp && !update_failed?
end
def mark_for_delete_if_blank_url
mark_for_destruction if url.blank?
end
def mark_as_failed(error_message)
update_fail
update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message))
end
def url=(value)
super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
mirror_url = Gitlab::UrlSanitizer.new(value)
self.credentials = mirror_url.credentials
super(mirror_url.sanitized_url)
end
def url
if super
Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url
end
rescue
super
end
def safe_url
return if url.nil?
result = URI.parse(url)
result.password = '*****' if result.password
result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user
result.to_s
end
private
def raw
@raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name)
end
def fallback_remote_name
return unless id
"remote_mirror_#{id}"
end
def recently_scheduled?
return false unless self.last_update_started_at
self.last_update_started_at >= Time.now - backoff_delay
end
def backoff_delay
if self.only_protected_branches
PROTECTED_BACKOFF_DELAY
else
UNPROTECTED_BACKOFF_DELAY
end
end
def reset_fields
update_columns(
last_error: nil,
last_update_at: nil,
last_successful_update_at: nil,
update_status: 'finished'
)
end
def set_override_remote_mirror_available
enabled = read_attribute(:enabled)
project.update(remote_mirror_available_overridden: enabled)
end
def set_new_remote_name
self.remote_name = "remote_mirror_#{SecureRandom.hex}"
end
def refresh_remote
return unless project
# Before adding a new remote we have to delete the data from
# the previous remote name
prev_remote_name = remote_name_was || fallback_remote_name
run_after_commit do
project.repository.async_remove_remote(prev_remote_name)
end
project.repository.add_remote(remote_name, url)
end
def remove_remote
return unless project # could be pending to delete so don't need to touch the git repository
project.repository.async_remove_remote(remote_name)
end
def mirror_url_changed?
url_changed? || encrypted_credentials_changed?
end
end
......@@ -861,6 +861,20 @@ class Repository
gitlab_shell.fetch_remote(raw_repository, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune)
end
def async_remove_remote(remote_name)
return unless remote_name
job_id = RepositoryRemoveRemoteWorker.perform_async(project.id, remote_name)
if job_id
Rails.logger.info("Remove remote job scheduled for #{project.id} with remote name: #{remote_name} job ID #{job_id}.")
else
Rails.logger.info("Remove remote job failed to create for #{project.id} with remote name #{remote_name}.")
end
job_id
end
def fetch_source_branch!(source_repository, source_branch, local_ref)
raw_repository.fetch_source_branch!(source_repository.raw_repository, source_branch, local_ref)
end
......
class ProjectMirrorEntity < Grape::Entity
prepend ::EE::ProjectMirrorEntity
expose :id
expose :remote_mirrors_attributes do |project|
next [] unless project.remote_mirrors.present?
project.remote_mirrors.map do |remote|
remote.as_json(only: %i[id url enabled])
end
end
end
......@@ -55,6 +55,7 @@ class GitPushService < BaseService
execute_related_hooks
perform_housekeeping
update_remote_mirrors
update_caches
update_signatures
......@@ -119,6 +120,13 @@ class GitPushService < BaseService
protected
def update_remote_mirrors
return unless @project.has_remote_mirror?
@project.mark_stuck_remote_mirrors_as_failed!
@project.update_remote_mirrors
end
def execute_related_hooks
# Update merge requests that may be affected by this push. A new branch
# could cause the last commit of a merge request to change.
......
module Projects
class UpdateRemoteMirrorService < BaseService
attr_reader :errors
def execute(remote_mirror)
@errors = []
return success unless remote_mirror.enabled?
begin
repository.fetch_remote(remote_mirror.remote_name, no_tags: true)
opts = {}
if remote_mirror.only_protected_branches?
opts[:only_branches_matching] = project.protected_branches.select(:name).map(&:name)
end
remote_mirror.update_repository(opts)
rescue => e
errors << e.message.strip
end
if errors.present?
error(errors.join("\n\n"))
else
success
end
end
end
end
- expanded = Rails.env.test?
%section.settings.no-animate{ class: ('expanded' if expanded) }
.settings-header
%h4
Push to a remote repository
%button.btn.js-settings-toggle
= expanded ? 'Collapse' : 'Expand'
%p
Set up the remote repository that you want to update with the content of the current repository
every time someone pushes to it.
= link_to 'Read more', help_page_path('workflow/repository_mirroring', anchor: 'pushing-to-a-remote-repository'), target: '_blank'
.settings-content
= form_for @project, url: project_mirror_path(@project) do |f|
%div
= form_errors(@project)
= render "shared/remote_mirror_update_button", remote_mirror: @remote_mirror
- if @remote_mirror.last_error.present?
.panel.panel-danger
.panel-heading
- if @remote_mirror.last_update_at
The remote repository failed to update #{time_ago_with_tooltip(@remote_mirror.last_update_at)}.
- else
The remote repository failed to update.
- if @remote_mirror.last_successful_update_at
Last successful update #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
.panel-body
%pre
:preserve
#{h(@remote_mirror.last_error.strip)}
= f.fields_for :remote_mirrors, @remote_mirror do |rm_form|
.form-group
= rm_form.check_box :enabled, class: "pull-left"
.prepend-left-20
= rm_form.label :enabled, "Remote mirror repository", class: "label-light append-bottom-0"
%p.light.append-bottom-0
Automatically update the remote mirror's branches, tags, and commits from this repository every time someone pushes to it.
.form-group.has-feedback
= rm_form.label :url, "Git repository URL", class: "label-light"
= rm_form.text_field :url, class: "form-control", placeholder: 'https://username:password@gitlab.company.com/group/project.git'
= render "projects/mirrors/instructions"
.form-group
= rm_form.check_box :only_protected_branches, class: 'pull-left'
.prepend-left-20
= rm_form.label :only_protected_branches, class: 'label-light'
= link_to icon('question-circle'), help_page_path('user/project/protected_branches')
= f.submit 'Save changes', class: 'btn btn-create', name: 'update_remote_mirror'
- if can?(current_user, :admin_mirror, @project)
= render 'projects/mirrors/push'
- if @project.has_remote_mirror?
.append-bottom-default
- if remote_mirror.update_in_progress?
%span.btn.disabled
= icon("refresh spin")
Updating&hellip;
- else
= link_to update_now_project_mirror_path(@project, sync_remote: true), method: :post, class: "btn" do
= icon("refresh")
Update Now
- if @remote_mirror.last_successful_update_at
%p.inline.prepend-left-10
Successfully updated #{time_ago_with_tooltip(@remote_mirror.last_successful_update_at)}.
......@@ -112,3 +112,4 @@
- update_user_activity
- upload_checksum
- web_hook
- repository_update_remote_mirror
class RepositoryRemoveRemoteWorker
include ApplicationWorker
include ExclusiveLeaseGuard
LEASE_TIMEOUT = 1.hour
attr_reader :project, :remote_name
def perform(project_id, remote_name)
@remote_name = remote_name
@project = Project.find_by_id(project_id)
return unless @project
logger.info("Removing remote #{remote_name} from project #{project.id}")
try_obtain_lease do
remove_remote = @project.repository.remove_remote(remote_name)
if remove_remote
logger.info("Remote #{remote_name} was successfully removed from project #{project.id}")
else
logger.error("Could not remove remote #{remote_name} from project #{project.id}")
end
end
end
def lease_timeout
LEASE_TIMEOUT
end
def lease_key
"remove_remote_#{project.id}_#{remote_name}"
end
end
class RepositoryUpdateRemoteMirrorWorker
UpdateAlreadyInProgressError = Class.new(StandardError)
UpdateError = Class.new(StandardError)
include ApplicationWorker
include Gitlab::ShellAdapter
sidekiq_options retry: 3, dead: false
sidekiq_retry_in { |count| 30 * count }
sidekiq_retries_exhausted do |msg, _|
Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}"
end
def perform(remote_mirror_id, scheduled_time)
remote_mirror = RemoteMirror.find(remote_mirror_id)
return if remote_mirror.updated_since?(scheduled_time)
raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress?
remote_mirror.update_start
project = remote_mirror.project
current_user = project.creator
result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror)
raise UpdateError, result[:message] if result[:status] == :error
remote_mirror.update_finish
rescue UpdateAlreadyInProgressError
raise
rescue UpdateError => ex
fail_remote_mirror(remote_mirror, ex.message)
raise
rescue => ex
return unless remote_mirror
fail_remote_mirror(remote_mirror, ex.message)
raise UpdateError, "#{ex.class}: #{ex.message}"
end
private
def fail_remote_mirror(remote_mirror, message)
remote_mirror.mark_as_failed(message)
Rails.logger.error(message)
end
end
......@@ -73,3 +73,6 @@
- [object_storage, 1]
- [plugin, 1]
- [pipeline_background, 1]
- [repository_update_remote_mirror, 1]
- [repository_remove_remote, 1]
......@@ -314,10 +314,10 @@ ActiveRecord::Schema.define(version: 20180503200320) do
t.integer "auto_canceled_by_id"
t.boolean "retried"
t.integer "stage_id"
t.integer "artifacts_file_store"
t.integer "artifacts_metadata_store"
t.boolean "protected"
t.integer "failure_reason"
t.integer "artifacts_file_store"
t.integer "artifacts_metadata_store"
end
add_index "ci_builds", ["artifacts_expire_at"], name: "index_ci_builds_on_artifacts_expire_at", where: "(artifacts_file <> ''::text)", using: :btree
......@@ -365,13 +365,13 @@ ActiveRecord::Schema.define(version: 20180503200320) do
t.integer "project_id", null: false
t.integer "job_id", null: false
t.integer "file_type", null: false
t.integer "file_store"
t.integer "size", limit: 8
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.datetime_with_timezone "expire_at"
t.string "file"
t.binary "file_sha256"
t.integer "file_store"
end
add_index "ci_job_artifacts", ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id", using: :btree
......
......@@ -106,6 +106,7 @@ excluded_attributes:
- :last_repository_updated_at
- :last_repository_check_at
- :storage_version
- :remote_mirror_available_overridden
- :description_html
snippets:
- :expired_at
......
......@@ -71,6 +71,7 @@ module Gitlab
projects_imported_from_github: Project.where(import_type: 'github').count,
protected_branches: ProtectedBranch.count,
releases: Release.count,
remote_mirrors: RemoteMirror.count,
snippets: Snippet.count,
todos: Todo.count,
uploads: Upload.count,
......
require 'spec_helper'
describe Projects::MirrorsController do
include ReactiveCachingHelpers
describe 'setting up a remote mirror' do
set(:project) { create(:project, :repository) }
context 'when the current project is not a mirror' do
it 'allows to create a remote mirror' do
sign_in(project.owner)
expect do
do_put(project, remote_mirrors_attributes: { '0' => { 'enabled' => 1, 'url' => 'http://foo.com' } })
end.to change { RemoteMirror.count }.to(1)
end
end
end
describe '#update' do
let(:project) { create(:project, :repository,