Commit f20912df authored by Francisco Javier López's avatar Francisco Javier López Committed by Douwe Maan

Extend API for importing a project export with overwrite support

parent 44f4a674
......@@ -286,6 +286,10 @@ class Group < Namespace
false
end
def refresh_project_authorizations
refresh_members_authorized_projects(blocking: false)
end
private
def update_two_factor_requirement
......
......@@ -252,6 +252,10 @@ class Namespace < ActiveRecord::Base
[]
end
def refresh_project_authorizations
owner.refresh_authorized_projects
end
private
def path_or_parent_changed?
......
......@@ -1472,7 +1472,9 @@ class Project < ActiveRecord::Base
end
def rename_repo_notify!
send_move_instructions(full_path_was)
# When we import a project overwriting the original project, there
# is a move operation. In that case we don't want to send the instructions.
send_move_instructions(full_path_was) unless started?
expires_full_path_cache
self.old_path_with_namespace = full_path_was
......
module Projects
class BaseMoveRelationsService < BaseService
attr_reader :source_project
def execute(source_project, remove_remaining_elements: true)
return if source_project.blank?
@source_project = source_project
true
end
private
def prepare_relation(relation, id_param = :id)
if Gitlab::Database.postgresql?
relation
else
relation.model.where("#{id_param}": relation.pluck(id_param))
end
end
end
end
......@@ -46,6 +46,20 @@ module Projects
raise
end
def attempt_repositories_rollback
return unless @project
flush_caches(@project)
unless mv_repository(removal_path(repo_path), repo_path)
raise_error('Failed to restore project repository. Please contact the administrator.')
end
unless mv_repository(removal_path(wiki_path), wiki_path)
raise_error('Failed to restore wiki repository. Please contact the administrator.')
end
end
private
def repo_path
......@@ -70,12 +84,9 @@ module Projects
# Skip repository removal. We use this flag when remove user or group
return true if params[:skip_repo] == true
# There is a possibility project does not have repository or wiki
return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
new_path = removal_path(path)
if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
if mv_repository(path, new_path)
log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
project.run_after_commit do
......@@ -87,6 +98,13 @@ module Projects
end
end
def mv_repository(from_path, to_path)
# There is a possibility project does not have repository or wiki
return true unless gitlab_shell.exists?(project.repository_storage_path, from_path + '.git')
gitlab_shell.mv_repository(project.repository_storage_path, from_path, to_path)
end
def attempt_rollback(project, message)
return unless project
......
......@@ -15,9 +15,18 @@ module Projects
file = params.delete(:file)
FileUtils.copy_entry(file.path, import_upload_path)
@overwrite = params.delete(:overwrite)
data = {}
data[:override_params] = @override_params if @override_params
if overwrite_project?
data[:original_path] = params[:path]
params[:path] += "-#{tmp_filename}"
end
params[:import_type] = 'gitlab_project'
params[:import_source] = import_upload_path
params[:import_data] = { data: { override_params: @override_params } } if @override_params
params[:import_data] = { data: data } if data.present?
::Projects::CreateService.new(current_user, params).execute
end
......@@ -31,5 +40,17 @@ module Projects
def tmp_filename
SecureRandom.hex
end
def overwrite_project?
@overwrite && project_with_same_full_path?
end
def project_with_same_full_path?
Project.find_by_full_path("#{current_namespace.full_path}/#{params[:path]}").present?
end
def current_namespace
@current_namespace ||= Namespace.find_by(id: params[:namespace_id])
end
end
end
module Projects
class MoveAccessService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
@project.with_transaction_returning_status do
if @project.namespace != source_project.namespace
@project.run_after_commit do
source_project.namespace.refresh_project_authorizations
self.namespace.refresh_project_authorizations
end
end
::Projects::MoveProjectMembersService.new(@project, @current_user)
.execute(source_project, remove_remaining_elements: remove_remaining_elements)
::Projects::MoveProjectGroupLinksService.new(@project, @current_user)
.execute(source_project, remove_remaining_elements: remove_remaining_elements)
::Projects::MoveProjectAuthorizationsService.new(@project, @current_user)
.execute(source_project, remove_remaining_elements: remove_remaining_elements)
success
end
end
end
end
module Projects
class MoveDeployKeysProjectsService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_deploy_keys_projects
remove_remaining_deploy_keys_projects if remove_remaining_elements
success
end
end
private
def move_deploy_keys_projects
prepare_relation(non_existent_deploy_keys_projects)
.update_all(project_id: @project.id)
end
def non_existent_deploy_keys_projects
source_project.deploy_keys_projects
.joins(:deploy_key)
.where.not(keys: { fingerprint: @project.deploy_keys.select(:fingerprint) })
end
def remove_remaining_deploy_keys_projects
source_project.deploy_keys_projects.destroy_all
end
end
end
module Projects
class MoveForksService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super && source_project.fork_network
Project.transaction(requires_new: true) do
move_forked_project_links
move_fork_network_members
update_root_project
refresh_forks_count
success
end
end
private
def move_forked_project_links
# Update ancestor
ForkedProjectLink.where(forked_to_project: source_project)
.update_all(forked_to_project_id: @project.id)
# Update the descendants
ForkedProjectLink.where(forked_from_project: source_project)
.update_all(forked_from_project_id: @project.id)
end
def move_fork_network_members
ForkNetworkMember.where(project: source_project).update_all(project_id: @project.id)
ForkNetworkMember.where(forked_from_project: source_project).update_all(forked_from_project_id: @project.id)
end
def update_root_project
# Update root network project
ForkNetwork.where(root_project: source_project).update_all(root_project_id: @project.id)
end
def refresh_forks_count
Projects::ForksCountService.new(@project).refresh_cache
end
end
end
module Projects
class MoveLfsObjectsProjectsService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_lfs_objects_projects
remove_remaining_lfs_objects_project if remove_remaining_elements
success
end
end
private
def move_lfs_objects_projects
prepare_relation(non_existent_lfs_objects_projects)
.update_all(project_id: @project.lfs_storage_project.id)
end
def remove_remaining_lfs_objects_project
source_project.lfs_objects_projects.destroy_all
end
def non_existent_lfs_objects_projects
source_project.lfs_objects_projects.where.not(lfs_object: @project.lfs_objects)
end
end
end
module Projects
class MoveNotificationSettingsService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_notification_settings
remove_remaining_notification_settings if remove_remaining_elements
success
end
end
private
def move_notification_settings
prepare_relation(non_existent_notifications)
.update_all(source_id: @project.id)
end
# Remove remaining notification settings from source_project
def remove_remaining_notification_settings
source_project.notification_settings.destroy_all
end
# Get users of current notification_settings
def users_in_target_project
@project.notification_settings.select(:user_id)
end
# Look for notification_settings in source_project that are not in the target project
def non_existent_notifications
source_project.notification_settings
.select(:id)
.where.not(user_id: users_in_target_project)
end
end
end
# NOTE: This service cannot be used directly because it is part of a
# a bigger process. Instead, use the service MoveAccessService which moves
# project memberships, project group links, authorizations and refreshes
# the authorizations if neccessary
module Projects
class MoveProjectAuthorizationsService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_project_authorizations
remove_remaining_authorizations if remove_remaining_elements
success
end
end
private
def move_project_authorizations
prepare_relation(non_existent_authorization, :user_id)
.update_all(project_id: @project.id)
end
def remove_remaining_authorizations
# I think because the Project Authorization table does not have a primary key
# it brings a lot a problems/bugs. First, Rails raises PG::SyntaxException if we use
# destroy_all instead of delete_all.
source_project.project_authorizations.delete_all(:delete_all)
end
# Look for authorizations in source_project that are not in the target project
def non_existent_authorization
source_project.project_authorizations
.select(:user_id)
.where.not(user: @project.authorized_users)
end
end
end
# NOTE: This service cannot be used directly because it is part of a
# a bigger process. Instead, use the service MoveAccessService which moves
# project memberships, project group links, authorizations and refreshes
# the authorizations if neccessary
module Projects
class MoveProjectGroupLinksService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_group_links
remove_remaining_project_group_links if remove_remaining_elements
success
end
end
private
def move_group_links
prepare_relation(non_existent_group_links)
.update_all(project_id: @project.id)
end
# Remove remaining project group links from source_project
def remove_remaining_project_group_links
source_project.reload.project_group_links.destroy_all
end
def group_links_in_target_project
@project.project_group_links.select(:group_id)
end
# Look for groups in source_project that are not in the target project
def non_existent_group_links
source_project.project_group_links
.where.not(group_id: group_links_in_target_project)
end
end
end
# NOTE: This service cannot be used directly because it is part of a
# a bigger process. Instead, use the service MoveAccessService which moves
# project memberships, project group links, authorizations and refreshes
# the authorizations if neccessary
module Projects
class MoveProjectMembersService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
Project.transaction(requires_new: true) do
move_project_members
remove_remaining_members if remove_remaining_elements
success
end
end
private
def move_project_members
prepare_relation(non_existent_members).update_all(source_id: @project.id)
end
def remove_remaining_members
# Remove remaining members and authorizations from source_project
source_project.project_members.destroy_all
end
def project_members_in_target_project
@project.project_members.select(:user_id)
end
# Look for members in source_project that are not in the target project
def non_existent_members
source_project.members
.select(:id)
.where.not(user_id: @project.project_members.select(:user_id))
end
end
end
module Projects
class MoveUsersStarProjectsService < BaseMoveRelationsService
def execute(source_project, remove_remaining_elements: true)
return unless super
user_stars = source_project.users_star_projects
return unless user_stars.any?
Project.transaction(requires_new: true) do
user_stars.update_all(project_id: @project.id)
Project.reset_counters @project.id, :users_star_projects
Project.reset_counters source_project.id, :users_star_projects
success
end
end
end
end
module Projects
class OverwriteProjectService < BaseService
def execute(source_project)
return unless source_project && source_project.namespace == @project.namespace
Project.transaction do
move_before_destroy_relationships(source_project)
destroy_old_project(source_project)
rename_project(source_project.name, source_project.path)
@project
end
# Projects::DestroyService can raise Exceptions, but we don't want
# to pass that kind of exception to the caller. Instead, we change it
# for a StandardError exception
rescue Exception => e # rubocop:disable Lint/RescueException
attempt_restore_repositories(source_project)
if e.class == Exception
raise StandardError, e.message
else
raise
end
end
private
def move_before_destroy_relationships(source_project)
options = { remove_remaining_elements: false }
::Projects::MoveUsersStarProjectsService.new(@project, @current_user).execute(source_project, options)
::Projects::MoveAccessService.new(@project, @current_user).execute(source_project, options)
::Projects::MoveDeployKeysProjectsService.new(@project, @current_user).execute(source_project, options)
::Projects::MoveNotificationSettingsService.new(@project, @current_user).execute(source_project, options)
::Projects::MoveForksService.new(@project, @current_user).execute(source_project, options)
::Projects::MoveLfsObjectsProjectsService.new(@project, @current_user).execute(source_project, options)
add_source_project_to_fork_network(source_project)
end
def destroy_old_project(source_project)
# Delete previous project (synchronously) and unlink relations
::Projects::DestroyService.new(source_project, @current_user).execute
end
def rename_project(name, path)
# Update de project's name and path to the original name/path
::Projects::UpdateService.new(@project,
@current_user,
{ name: name, path: path })
.execute
end
def attempt_restore_repositories(project)
::Projects::DestroyService.new(project, @current_user).attempt_repositories_rollback
end
def add_source_project_to_fork_network(source_project)
return unless @project.fork_network
# Because he have moved all references in the fork network from the source_project
# we won't be able to query the database (only through its cached data),
# for its former relationships. That's why we're adding it to the network
# as a fork of the target project
ForkNetworkMember.create!(fork_network: @project.fork_network,
project: source_project,
forked_from_project: @project)
end
end
end
---
title: Extend API for importing a project export with overwrite support
merge_request: 17883
author:
type: added
......@@ -111,6 +111,7 @@ POST /projects/import
| `namespace` | integer/string | no | The ID or path of the namespace that the project will be imported to. Defaults to the current user's namespace |
| `file` | string | yes | The file to be uploaded |
| `path` | string | yes | Name and path for new project |
| `overwrite` | boolean | no | If there is a project with the same path the import will overwrite it. Default to false |
| `override_params` | Hash | no | Supports all fields defined in the [Project API](projects.md)] |
The override params passed will take precendence over all values defined inside the export file.
......
......@@ -26,6 +26,7 @@ module API
requires :path, type: String, desc: 'The new project path and name'
requires :file, type: File, desc: 'The project export file to be imported'
optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace."
optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it'
optional :override_params,
type: Hash,
desc: 'New project params to override values in the export' do
......@@ -50,7 +51,8 @@ module API
project_params = {
path: import_params[:path],
namespace_id: namespace.id,
file: import_params[:file]['tempfile']
file: import_params[:file]['tempfile'],
overwrite: import_params[:overwrite]
}
override_params = import_params.delete(:override_params)
......
module Gitlab
module ImportExport
class Importer
include Gitlab::Allowable
include Gitlab::Utils::StrongMemoize
def self.imports_repository?
true
end
......@@ -13,12 +16,14 @@ module Gitlab
end
def execute
if import_file && check_version! && restorers.all?(&:restore)
if import_file && check_version! && restorers.all?(&:restore) && overwrite_project
project_tree.restored_project
else
raise Projects::ImportService::Error.new(@shared.errors.join(', '))
end
rescue => e
raise Projects::ImportService::Error.new(e.message)
ensure
remove_import_file
end
......@@ -26,7 +31,7 @@ module Gitlab
def restorers
[repo_restorer, wiki_restorer, project_tree, avatar_restorer,
uploads_restorer, lfs_restorer]
uploads_restorer, lfs_restorer, statistics_restorer]
end
def import_file
......@@ -69,6 +74,10 @@ module Gitlab
Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared)
end
def statistics_restorer
Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared)
end
def path_with_namespace
File.join(@project.namespace.full_path, @project.path)
end
......@@ -84,6 +93,33 @@ module Gitlab
def remove_import_file
FileUtils.rm_rf(@archive_file)
end
def overwrite_project
project = project_tree.restored_project
return unless can?(@current_user, :admin_namespace, project.namespace)
if overwrite_project?
::Projects::OverwriteProjectService.new(project, @current_user)
.execute(project_to_overwrite)
end
true
end
def original_path
@project.import_data&.data&.fetch('original_path', nil)
end
def overwrite_project?
original_path.present? && project_to_overwrite.present?
end
def project_to_overwrite
strong_memoize(:project_to_overwrite) do
Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}")
end
end
end
end
end
......@@ -92,7 +92,7 @@ module Gitlab
end
def override_params
return {} unless params = @project.import_data&.data&.fetch('override_params')
return {} unless params = @project.import_data&.data&.fetch('override_params', nil)
@override_params ||= params.select do |key, _value|
Project.column_names.include?(key.to_s) &&
......
module Gitlab
module ImportExport
class StatisticsRestorer
def initialize(project:, shared:)
@project = project
@shared = shared
end
def restore
@project.statistics.refresh!
rescue => e
@shared.error(e)
false
end
end
end
end
FactoryBot.define do
factory :users_star_project do
project
user
end
end
require 'spec_helper'
describe Gitlab::ImportExport::Importer do
let(:user) { create(:user) }
let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared }
let(:project) { create(:project, import_source: File.join(test_path, 'exported-project.gz')) }
......@@ -11,6 +12,7 @@ describe Gitlab::ImportExport::Importer do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
FileUtils.mkdir_p(shared.export_path)
FileUtils.cp(Rails.root.join('spec', 'fixtures', 'exported-project.gz'), test_path)
allow(subject).to receive(:remove_import_file)
end
after do
......@@ -42,7 +44,8 @@ describe Gitlab::ImportExport::Importer do