Commit b529d04b authored by Jacob Vosmaer (GitLab)'s avatar Jacob Vosmaer (GitLab) Committed by Douwe Maan

Use Gitaly's DeleteAllRepositories RPC during backup restore

parent a45b9fc9
......@@ -412,7 +412,7 @@ group :ed25519 do
end
# Gitaly GRPC client
gem 'gitaly-proto', '~> 0.99.0', require: 'gitaly'
gem 'gitaly-proto', '~> 0.100.0', require: 'gitaly'
gem 'grpc', '~> 1.11.0'
# Locked until https://github.com/google/protobuf/issues/4210 is closed
......
......@@ -281,7 +281,7 @@ GEM
gettext_i18n_rails (>= 0.7.1)
po_to_json (>= 1.0.0)
rails (>= 3.2.0)
gitaly-proto (0.99.0)
gitaly-proto (0.100.0)
google-protobuf (~> 3.1)
grpc (~> 1.10)
github-linguist (5.3.3)
......@@ -1036,7 +1036,7 @@ DEPENDENCIES
gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3)
gitaly-proto (~> 0.99.0)
gitaly-proto (~> 0.100.0)
github-linguist (~> 5.3.3)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-gollum-lib (~> 4.2)
......
......@@ -73,29 +73,40 @@ module Backup
end
def prepare_directories
# TODO: Need to find a way to do this for gitaly
# Gitaly discussion issue: https://gitlab.com/gitlab-org/gitaly/issues/1194
Gitlab.config.repositories.storages.each do |name, repository_storage|
path = repository_storage.legacy_disk_path
next unless File.exist?(path)
# Move all files in the existing repos directory except . and .. to
# repositories.old.<timestamp> directory
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
FileUtils.mkdir_p(bk_repos_path, mode: 0700)
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
begin
FileUtils.mv(files, bk_repos_path)
rescue Errno::EACCES
access_denied_error(path)
rescue Errno::EBUSY
resource_busy_error(path)
delete_all_repositories(name, repository_storage)
end
end
def delete_all_repositories(name, repository_storage)
gitaly_migrate(:delete_all_repositories) do |is_enabled|
if is_enabled
Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories
else
local_delete_all_repositories(name, repository_storage)
end
end
end
def local_delete_all_repositories(name, repository_storage)
path = repository_storage.legacy_disk_path
return unless File.exist?(path)
# Move all files in the existing repos directory except . and .. to
# repositories.old.<timestamp> directory
bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s)
FileUtils.mkdir_p(bk_repos_path, mode: 0700)
files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")]
begin
FileUtils.mv(files, bk_repos_path)
rescue Errno::EACCES
access_denied_error(path)
rescue Errno::EBUSY
resource_busy_error(path)
end
end
def restore_custom_hooks(project)
# TODO: Need to find a way to do this for gitaly
# Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/1195
......@@ -113,6 +124,7 @@ module Backup
def restore
prepare_directories
gitlab_shell = Gitlab::Shell.new
Project.find_each(batch_size: 1000) do |project|
progress.print " * #{project.full_path} ... "
path_to_project_bundle = path_to_bundle(project)
......@@ -121,7 +133,6 @@ module Backup
restore_repo_success = nil
if File.exist?(path_to_project_bundle)
begin
gitlab_shell.remove_repository(project.repository_storage, project.disk_path) if project.repository_exists?
project.repository.create_from_bundle path_to_project_bundle
restore_repo_success = true
rescue => e
......@@ -146,7 +157,6 @@ module Backup
if File.exist?(path_to_wiki_bundle)
progress.print " * #{wiki.full_path} ... "
begin
gitlab_shell.remove_repository(wiki.repository_storage, wiki.disk_path) if wiki.repository_exists?
wiki.repository.create_from_bundle(path_to_wiki_bundle)
progress.puts "[DONE]".color(:green)
rescue => e
......@@ -224,5 +234,11 @@ module Backup
def display_repo_path(project)
project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
end
def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block)
Gitlab::GitalyClient.migrate(method, status: status, &block)
rescue GRPC::NotFound, GRPC::BadStatus => e
raise Error, e
end
end
end
module Gitlab
module GitalyClient
class StorageService
def initialize(storage)
@storage = storage
end
# Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation.
def delete_all_repositories
request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage)
GitalyClient.call(@storage, :storage_service, :delete_all_repositories, request)
end
end
end
end
......@@ -71,6 +71,40 @@ describe Backup::Repository do
end
end
describe '#delete_all_repositories', :seed_helper do
shared_examples('delete_all_repositories') do
before do
allow(FileUtils).to receive(:mkdir_p).and_call_original
allow(FileUtils).to receive(:mv).and_call_original
end
after(:all) do
ensure_seeds
end
it 'removes all repositories' do
# Sanity check: there should be something for us to delete
expect(list_repositories).to include(File.join(SEED_STORAGE_PATH, TEST_REPO_PATH))
subject.delete_all_repositories('default', Gitlab.config.repositories.storages['default'])
expect(list_repositories).to be_empty
end
def list_repositories
Dir[SEED_STORAGE_PATH + '/*.git']
end
end
context 'with gitaly' do
it_behaves_like 'delete_all_repositories'
end
context 'without gitaly', :skip_gitaly_mock do
it_behaves_like 'delete_all_repositories'
end
end
describe '#empty_repo?' do
context 'for a wiki' do
let(:wiki) { create(:project_wiki) }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment