Commit 72a425fa authored by Dmitriy Zaporozhets's avatar Dmitriy Zaporozhets

Merge branch 'remove-satellites'

Signed-off-by: default avatarDmitriy Zaporozhets <dmitriy.zaporozhets@gmail.com>
parents a3e1d7cf 4a0fa270
Please view this file on the master branch, on stable branches it's out of date.
v 8.0.0 (unreleased)
- Remove satellites
- Better performance for web editor (switched from satellites to rugged)
- Faster merge
- Ability to fetch merge requests from refs/merge-requests/:id
v 7.14.0 (unreleased)
- Update default robots.txt rules to disallow crawling of irrelevant pages (Ben Bodenmiller)
......
......@@ -19,7 +19,7 @@ class @MergeRequestWidget
when 'merged'
location.reload()
else
setTimeout(merge_request_widget.mergeInProgress, 3000)
setTimeout(merge_request_widget.mergeInProgress, 2000)
dataType: 'json'
getMergeStatus: ->
......
......@@ -13,27 +13,20 @@ class Projects::BlobController < Projects::ApplicationController
before_action :commit, except: [:new, :create]
before_action :blob, except: [:new, :create]
before_action :from_merge_request, only: [:edit, :update]
before_action :after_edit_path, only: [:edit, :update]
before_action :require_branch_head, only: [:edit, :update]
before_action :editor_variables, except: [:show, :preview, :diff]
before_action :after_edit_path, only: [:edit, :update]
def new
commit unless @repository.empty?
end
def create
file_path = File.join(@path, File.basename(params[:file_name]))
result = Files::CreateService.new(
@project,
current_user,
params.merge(new_branch: sanitized_new_branch_name),
@ref,
file_path
).execute
result = Files::CreateService.new(@project, current_user, @commit_params).execute
if result[:status] == :success
flash[:notice] = "Your changes have been successfully committed"
ref = sanitized_new_branch_name.presence || @ref
redirect_to namespace_project_blob_path(@project.namespace, @project, File.join(ref, file_path))
redirect_to namespace_project_blob_path(@project.namespace, @project, File.join(@target_branch, @file_path))
else
flash[:alert] = result[:message]
render :new
......@@ -48,22 +41,10 @@ class Projects::BlobController < Projects::ApplicationController
end
def update
result = Files::UpdateService.
new(
@project,
current_user,
params.merge(new_branch: sanitized_new_branch_name),
@ref,
@path
).execute
result = Files::UpdateService.new(@project, current_user, @commit_params).execute
if result[:status] == :success
flash[:notice] = "Your changes have been successfully committed"
if from_merge_request
from_merge_request.reload_code
end
redirect_to after_edit_path
else
flash[:alert] = result[:message]
......@@ -80,12 +61,11 @@ class Projects::BlobController < Projects::ApplicationController
end
def destroy
result = Files::DeleteService.new(@project, current_user, params, @ref, @path).execute
result = Files::DeleteService.new(@project, current_user, @commit_params).execute
if result[:status] == :success
flash[:notice] = "Your changes have been successfully committed"
redirect_to namespace_project_tree_path(@project.namespace, @project,
@ref)
redirect_to namespace_project_tree_path(@project.namespace, @project, @target_branch)
else
flash[:alert] = result[:message]
render :show
......@@ -135,7 +115,6 @@ class Projects::BlobController < Projects::ApplicationController
@id = params[:id]
@ref, @path = extract_ref(@id)
rescue InvalidPathError
not_found!
end
......@@ -145,8 +124,8 @@ class Projects::BlobController < Projects::ApplicationController
if from_merge_request
diffs_namespace_project_merge_request_path(from_merge_request.target_project.namespace, from_merge_request.target_project, from_merge_request) +
"#file-path-#{hexdigest(@path)}"
elsif sanitized_new_branch_name.present?
namespace_project_blob_path(@project.namespace, @project, File.join(sanitized_new_branch_name, @path))
elsif @target_branch.present?
namespace_project_blob_path(@project.namespace, @project, File.join(@target_branch, @path))
else
namespace_project_blob_path(@project.namespace, @project, @id)
end
......@@ -160,4 +139,25 @@ class Projects::BlobController < Projects::ApplicationController
def sanitized_new_branch_name
@new_branch ||= sanitize(strip_tags(params[:new_branch]))
end
def editor_variables
@current_branch = @ref
@target_branch = (sanitized_new_branch_name || @ref)
@file_path =
if action_name.to_s == 'create'
File.join(@path, File.basename(params[:file_name]))
else
@path
end
@commit_params = {
file_path: @file_path,
current_branch: @current_branch,
target_branch: @target_branch,
commit_message: params[:commit_message],
file_content: params[:content],
file_content_encoding: params[:encoding]
}
end
end
......@@ -13,13 +13,8 @@ class Projects::CompareController < Projects::ApplicationController
base_ref = Addressable::URI.unescape(params[:from])
@ref = head_ref = Addressable::URI.unescape(params[:to])
compare_result = CompareService.new.execute(
current_user,
@project,
head_ref,
@project,
base_ref
)
compare_result = CompareService.new.
execute(@project, head_ref, @project, base_ref)
@commits = compare_result.commits
@diffs = compare_result.diffs
......
require 'gitlab/satellite/satellite'
class Projects::MergeRequestsController < Projects::ApplicationController
before_action :module_enabled
before_action :merge_request, only: [
:edit, :update, :show, :diffs, :commits, :automerge, :automerge_check,
:edit, :update, :show, :diffs, :commits, :merge, :merge_check,
:ci_status, :toggle_subscription
]
before_action :closes_issues, only: [:edit, :update, :show, :diffs, :commits]
......@@ -137,7 +135,7 @@ class Projects::MergeRequestsController < Projects::ApplicationController
end
end
def automerge_check
def merge_check
if @merge_request.unchecked?
@merge_request.check_if_can_be_merged
end
......@@ -147,11 +145,11 @@ class Projects::MergeRequestsController < Projects::ApplicationController
render partial: "projects/merge_requests/widget/show.html.haml", layout: false
end
def automerge
def merge
return access_denied! unless @merge_request.can_be_merged_by?(current_user)
if @merge_request.automergeable?
AutoMergeWorker.perform_async(@merge_request.id, current_user.id, params)
if @merge_request.mergeable?
MergeWorker.perform_async(@merge_request.id, current_user.id, params)
@status = true
else
@status = false
......
......@@ -41,8 +41,6 @@ class MergeRequest < ActiveRecord::Base
delegate :commits, :diffs, :last_commit, :last_commit_short_sha, to: :merge_request_diff, prefix: nil
attr_accessor :should_remove_source_branch
# When this attribute is true some MR validation is ignored
# It allows us to close or modify broken merge requests
attr_accessor :allow_broken
......@@ -57,7 +55,7 @@ class MergeRequest < ActiveRecord::Base
transition [:reopened, :opened] => :closed
end
event :merge do
event :mark_as_merged do
transition [:reopened, :opened, :locked] => :merged
end
......@@ -205,7 +203,10 @@ class MergeRequest < ActiveRecord::Base
end
def check_if_can_be_merged
if Gitlab::Satellite::MergeAction.new(self.author, self).can_be_merged?
can_be_merged =
project.repository.can_be_merged?(source_sha, target_branch)
if can_be_merged
mark_as_mergeable
else
mark_as_unmergeable
......@@ -220,18 +221,6 @@ class MergeRequest < ActiveRecord::Base
self.target_project.events.where(target_id: self.id, target_type: "MergeRequest", action: Event::CLOSED).last
end
def automerge!(current_user, commit_message = nil)
return unless automergeable?
MergeRequests::AutoMergeService.
new(target_project, current_user).
execute(self, commit_message)
end
def remove_source_branch?
self.should_remove_source_branch && !self.source_project.root_ref?(self.source_branch) && !self.for_fork?
end
def open?
opened? || reopened?
end
......@@ -240,11 +229,11 @@ class MergeRequest < ActiveRecord::Base
title =~ /\A\[?WIP\]?:? /i
end
def automergeable?
def mergeable?
open? && !work_in_progress? && can_be_merged?
end
def automerge_status
def gitlab_merge_status
if work_in_progress?
"work_in_progress"
else
......@@ -271,14 +260,14 @@ class MergeRequest < ActiveRecord::Base
#
# see "git diff"
def to_diff(current_user)
Gitlab::Satellite::MergeAction.new(current_user, self).diff_in_satellite
target_project.repository.diff_text(target_branch, source_sha)
end
# Returns the commit as a series of email patches.
#
# see "git format-patch"
def to_patch(current_user)
Gitlab::Satellite::MergeAction.new(current_user, self).format_patch
target_project.repository.format_patch(target_branch, source_sha)
end
def hook_attrs
......@@ -429,4 +418,30 @@ class MergeRequest < ActiveRecord::Base
"Open"
end
end
def target_sha
@target_sha ||= target_project.
repository.commit(target_branch).sha
end
def source_sha
commits.first.sha
end
def fetch_ref
target_project.repository.fetch_ref(
source_project.repository.path_to_repo,
"refs/heads/#{source_branch}",
"refs/merge-requests/#{iid}/head"
)
end
def in_locked_state
begin
lock_mr
yield
ensure
unlock_mr if locked?
end
end
end
......@@ -16,9 +16,8 @@ require Rails.root.join("app/models/commit")
class MergeRequestDiff < ActiveRecord::Base
include Sortable
# Prevent store of diff
# if commits amount more then 200
COMMITS_SAFE_SIZE = 200
# Prevent store of diff if commits amount more then 500
COMMITS_SAFE_SIZE = 500
attr_reader :commits, :diffs
......@@ -124,12 +123,12 @@ class MergeRequestDiff < ActiveRecord::Base
if new_diffs.any?
if new_diffs.size > Commit::DIFF_HARD_LIMIT_FILES
self.state = :overflow_diff_files_limit
new_diffs = []
new_diffs = new_diffs.first[Commit::DIFF_HARD_LIMIT_LINES]
end
if new_diffs.sum { |diff| diff.diff.lines.count } > Commit::DIFF_HARD_LIMIT_LINES
self.state = :overflow_diff_lines_limit
new_diffs = []
new_diffs = new_diffs.first[Commit::DIFF_HARD_LIMIT_LINES]
end
end
......@@ -160,12 +159,21 @@ class MergeRequestDiff < ActiveRecord::Base
private
def compare_result
@compare_result ||= CompareService.new.execute(
merge_request.author,
merge_request.source_project,
merge_request.source_branch,
merge_request.target_project,
merge_request.target_branch,
)
@compare_result ||=
begin
# Update ref for merge request
merge_request.fetch_ref
# Get latest sha of branch from source project
source_sha = merge_request.source_project.commit(source_branch).sha
Gitlab::CompareResult.new(
Gitlab::Git::Compare.new(
merge_request.target_project.repository.raw_repository,
merge_request.target_branch,
source_sha,
)
)
end
end
end
......@@ -118,12 +118,11 @@ class Namespace < ActiveRecord::Base
gitlab_shell.add_namespace(path_was)
if gitlab_shell.mv_namespace(path_was, path)
# If repositories moved successfully we need to remove old satellites
# and send update instructions to users.
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
gitlab_shell.rm_satellites(path_was)
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives
......
......@@ -520,14 +520,6 @@ class Project < ActiveRecord::Base
!repository.exists? || repository.empty?
end
def ensure_satellite_exists
self.satellite.create unless self.satellite.exists?
end
def satellite
@satellite ||= Gitlab::Satellite::Satellite.new(self)
end
def repo
repository.raw
end
......@@ -597,14 +589,11 @@ class Project < ActiveRecord::Base
new_path_with_namespace = File.join(namespace_dir, path)
if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to remove old satellite
# and send update instructions to users.
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
gitlab_shell.rm_satellites(old_path_with_namespace)
ensure_satellite_exists
send_move_instructions
reset_events_cache
rescue
......@@ -702,7 +691,6 @@ class Project < ActiveRecord::Base
def create_repository
if forked?
if gitlab_shell.fork_repository(forked_from_project.path_with_namespace, self.namespace.path)
ensure_satellite_exists
true
else
errors.add(:base, 'Failed to fork repository via gitlab-shell')
......
......@@ -74,6 +74,8 @@ class GitlabCiService < CiService
else
:error
end
rescue Errno::ECONNREFUSED
:error
end
def fork_registration(new_project, private_token)
......@@ -103,6 +105,8 @@ class GitlabCiService < CiService
if response.code == 200 and response["coverage"]
response["coverage"]
end
rescue Errno::ECONNREFUSED
nil
end
def build_page(sha, ref)
......
require 'securerandom'
class Repository
class PreReceiveError < StandardError; end
class CommitError < StandardError; end
include Gitlab::ShellAdapter
attr_accessor :raw_repository, :path_with_namespace, :project
......@@ -368,6 +373,89 @@ class Repository
@root_ref ||= raw_repository.root_ref
end
def commit_file(user, path, content, message, branch)
commit_with_hooks(user, branch) do |ref|
path[0] = '' if path[0] == '/'
committer = user_to_comitter(user)
options = {}
options[:committer] = committer
options[:author] = committer
options[:commit] = {
message: message,
branch: ref,
}
options[:file] = {
content: content,
path: path
}
Gitlab::Git::Blob.commit(raw_repository, options)
end
end
def remove_file(user, path, message, branch)
commit_with_hooks(user, branch) do |ref|
path[0] = '' if path[0] == '/'
committer = user_to_comitter(user)
options = {}
options[:committer] = committer
options[:author] = committer
options[:commit] = {
message: message,
branch: ref
}
options[:file] = {
path: path
}
Gitlab::Git::Blob.remove(raw_repository, options)
end
end
def user_to_comitter(user)
{
email: user.email,
name: user.name,
time: Time.now
}
end
def can_be_merged?(source_sha, target_branch)
our_commit = rugged.branches[target_branch].target
their_commit = rugged.lookup(source_sha)
if our_commit && their_commit
!rugged.merge_commits(our_commit, their_commit).conflicts?
else
false
end
end
def merge(user, source_sha, target_branch, options = {})
our_commit = rugged.branches[target_branch].target
their_commit = rugged.lookup(source_sha)
raise "Invalid merge target" if our_commit.nil?
raise "Invalid merge source" if their_commit.nil?
merge_index = rugged.merge_commits(our_commit, their_commit)
return false if merge_index.conflicts?
commit_with_hooks(user, target_branch) do |ref|
actual_options = options.merge(
parents: [our_commit, their_commit],
tree: merge_index.write_tree(rugged),
update_ref: ref
)
Rugged::Commit.create(rugged, actual_options)
end
end
def merged_to_root_ref?(branch_name)
branch_commit = commit(branch_name)
root_ref_commit = commit(root_ref)
......@@ -412,6 +500,64 @@ class Repository
)
end
def fetch_ref(source_path, source_ref, target_ref)
args = %W(git fetch #{source_path} #{source_ref}:#{target_ref})
Gitlab::Popen.popen(args, path_to_repo)
end
def commit_with_hooks(current_user, branch)
oldrev = Gitlab::Git::BLANK_SHA
ref = Gitlab::Git::BRANCH_REF_PREFIX + branch
gl_id = Gitlab::ShellEnv.gl_id(current_user)
was_empty = empty?
# Create temporary ref
random_string = SecureRandom.hex
tmp_ref = "refs/tmp/#{random_string}/head"
unless was_empty
oldrev = find_branch(branch).target
rugged.references.create(tmp_ref, oldrev)
end
# Make commit in tmp ref
newrev = yield(tmp_ref)
unless newrev
raise CommitError.new('Failed to create commit')
end
# Run GitLab pre-receive hook
pre_receive_hook = Gitlab::Git::Hook.new('pre-receive', path_to_repo)
status = pre_receive_hook.trigger(gl_id, oldrev, newrev, ref)
if status
if was_empty
# Create branch
rugged.references.create(ref, newrev)
else
# Update head
current_head = find_branch(branch).target
# Make sure target branch was not changed during pre-receive hook
if current_head == oldrev
rugged.references.update(ref, newrev)
else
raise CommitError.new('Commit was rejected because branch received new push')
end
end
# Run GitLab post receive hook
post_receive_hook = Gitlab::Git::Hook.new('post-receive', path_to_repo)
status = post_receive_hook.trigger(gl_id, oldrev, newrev, ref)
else
# Remove tmp ref and return error to user
rugged.references.delete(tmp_ref)
raise PreReceiveError.new('Commit was rejected by pre-receive hook')
end
end
private
def cache
......
......@@ -31,6 +31,10 @@ class BaseService
SystemHooksService.new
end
def repository
project.repository
end
# Add an error to the specified model for restricted visibility levels
def deny_visibility_level(model, denied_visibility_level = nil)
denied_visibility_level ||= model.visibility_level
......
require 'securerandom'
# Compare 2 branches for one repo or between repositories
# and return Gitlab::CompareResult object that responds to commits and diffs
class CompareService
def execute(current_user, source_project, source_branch, target_project, target_branch)
# Try to compare branches to get commits list and diffs
#
# Note: Use satellite only when need to compare between two repos
# because satellites are slower than operations on bare repo
if target_project == source_project
Gitlab::CompareResult.new(
Gitlab::Git::Compare.new(
target_project.repository.raw_repository,
target_branch,
source_branch,
)
def execute(source_project, source_branch, target_project, target_branch)
source_sha = source_project.commit(source_branch).sha
# If compare with other project we need to fetch ref first
unless target_project == source_project
random_string = SecureRandom.hex
target_project.repository.fetch_ref(
source_project.repository.path_to_repo,
"refs/heads/#{source_branch}",
"refs/tmp/#{random_string}/head"
)
else
Gitlab::Satellite::CompareAction.new(
current_user,
target_project,
target_branch,
source_project,
source_branch
).result
end
Gitlab::CompareResult.new(
Gitlab::Git::Compare.new(
target_project.repository.raw_repository,
target_branch,
source_sha,
)
)
end
end
module Files
class BaseService < ::BaseService
attr_reader :ref, :path
class ValidationError < StandardError; end
def initialize(project, user, params, ref, path = nil)
@project, @current_user, @params = project, user, params.dup
@ref = ref
@path = path
def execute
@current_branch = params[:current_branch]
@target_branch = params[:target_branch]
@commit_message = params[:commit_message]
@file_path = params[:file_path]
@file_content = if params[:file_content_encoding] == 'base64'
Base64.decode64(params[:file_content])
else
params[:file_content]
end
# Validate parameters
validate
# Create new branch if it different from current_branch
if @target_branch != @current_branch
create_target_branch
end
if sha = commit
success
else
error("Something went wrong. Your changes were not committed")
end
rescue Repository::CommitError, Repository::PreReceiveError, ValidationError => ex
error(ex.message)
end
private