GitLab steht Mittwoch, den 08. Juli, zwischen 09:00 und 13:00 Uhr aufgrund von Wartungsarbeiten nicht zur Verfügung.

Commit 1592b583 authored by Gabriel Mazetto's avatar Gabriel Mazetto

Adds Rollback functionality to HashedStorage migration

We are adding sidekiq workers and service classes to allow to rollback
a hashed storage migration. There are some refactoring involved as well
as part of the code can be reused by both the migration and the rollback
logic.
parent ff2ca356
......@@ -1976,6 +1976,16 @@ class Project < ActiveRecord::Base
end
end
def rollback_to_legacy_storage!
return if legacy_storage?
if git_transfer_in_progress?
ProjectRollbackHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id)
else
ProjectRollbackHashedStorageWorker.perform_async(id)
end
end
def git_transfer_in_progress?
repo_reference_count > 0 || wiki_reference_count > 0
end
......
# frozen_string_literal: true
module Projects
module HashedStorage
AttachmentRollbackError = Class.new(StandardError)
class RollbackAttachmentsService < BaseService
attr_reader :logger, :old_disk_path
def initialize(project, logger: nil)
@project = project
@logger = logger || Rails.logger
end
def execute
origin = FileUploader.absolute_base_dir(project)
project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
target = FileUploader.absolute_base_dir(project)
result = move_folder!(origin, target)
project.save!
if result && block_given?
yield
end
result
end
private
def move_folder!(old_path, new_path)
unless File.directory?(old_path)
logger.info("Skipped attachments rollback from '#{old_path}' to '#{new_path}', source path doesn't exist or is not a directory (PROJECT_ID=#{project.id})")
return true
end
if File.exist?(new_path)
logger.error("Cannot rollback attachments from '#{old_path}' to '#{new_path}', target path already exist (PROJECT_ID=#{project.id})")
raise AttachmentRollbackError, "Target path '#{new_path}' already exist"
end
# Create hashed storage base path folder
FileUtils.mkdir_p(File.dirname(new_path))
FileUtils.mv(old_path, new_path)
logger.info("Rolledback project attachments from '#{old_path}' to '#{new_path}' (PROJECT_ID=#{project.id})")
true
end
end
end
end
# frozen_string_literal: true
module Projects
module HashedStorage
class RollbackRepositoryService < BaseRepositoryService
def execute
try_to_set_repository_read_only!
@old_storage_version = project.storage_version
project.storage_version = nil
project.ensure_storage_path_exists
@new_disk_path = project.disk_path
result = move_repository(old_disk_path, new_disk_path)
if move_wiki
result &&= move_repository("#{old_wiki_disk_path}", "#{new_disk_path}.wiki")
end
if result
project.write_repository_config
project.track_project_repository
else
rollback_folder_move
project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
end
project.repository_read_only = false
project.save!
if result && block_given?
yield
end
result
end
private
def try_to_set_repository_read_only!
# Mitigate any push operation to start during migration
unless project.set_repository_read_only!
migration_error = "Target repository '#{old_disk_path}' cannot be made read-only as there is a git transfer in progress"
logger.error migration_error
raise RepositoryRollbackError, migration_error
end
end
end
end
end
# frozen_string_literal: true
module Projects
module HashedStorage
class RollbackService < BaseService
attr_reader :logger, :old_disk_path
def initialize(project, old_disk_path, logger: nil)
@project = project
@old_disk_path = old_disk_path
@logger = logger || Rails.logger
end
def execute
# Rollback attachments from Hashed Storage to Legacy
if project.hashed_storage?(:attachments)
return false unless rollback_attachments
end
# Rollback repository from Hashed Storage to Legacy
if project.hashed_storage?(:repository)
rollback_repository
end
end
private
def rollback_attachments
HashedStorage::RollbackAttachmentsService.new(project, logger: logger).execute
end
def rollback_repository
HashedStorage::RollbackRepositoryService.new(project, old_disk_path, logger: logger).execute
end
end
end
end
......@@ -127,6 +127,7 @@
- project_destroy
- project_export
- project_migrate_hashed_storage
- project_rollback_hashed_storage
- project_service
- propagate_service_template
- reactive_caching
......
# frozen_string_literal: true
class ProjectRollbackHashedStorageWorker
include ApplicationWorker
LEASE_TIMEOUT = 30.seconds.to_i
# rubocop: disable CodeReuse/ActiveRecord
def perform(project_id, old_disk_path = nil)
uuid = lease_for(project_id).try_obtain
if uuid
project = Project.find_by(id: project_id)
return if project.nil? || project.pending_delete?
old_disk_path ||= project.disk_path
::Projects::HashedStorage::RollbackService.new(project, old_disk_path, logger: logger).execute
else
return false
end
ensure
cancel_lease_for(project_id, uuid) if uuid
end
# rubocop: enable CodeReuse/ActiveRecord
def lease_for(project_id)
Gitlab::ExclusiveLease.new(lease_key(project_id), timeout: LEASE_TIMEOUT)
end
private
def lease_key(project_id)
# we share the same lease key for both migration and rollback so they don't run simultaneously
"#{ProjectMigrateHashedStorageWorker::LEASE_KEY_SEGMENT}:#{project_id}"
end
def cancel_lease_for(project_id, uuid)
Gitlab::ExclusiveLease.cancel(lease_key(project_id), uuid)
end
end
---
title: Hashed Storage rollback mechanism
merge_request: 23955
author:
type: added
......@@ -68,6 +68,7 @@
- [background_migration, 1]
- [gcp_cluster, 1]
- [project_migrate_hashed_storage, 1]
- [project_rollback_hashed_storage, 1]
- [hashed_storage, 1]
- [pages_domain_verification, 1]
- [object_storage_upload, 1]
......
......@@ -45,8 +45,15 @@ module Gitlab
Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
end
# Flag a project to be rolled-back to Legacy Storage
#
# @param [Project] project that will be rolled-back
def rollback(project)
# TODO: implement rollback strategy
Rails.logger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..."
project.rollback_to_legacy_storage!
rescue => err
Rails.logger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
end
private
......
......@@ -88,4 +88,50 @@ describe Gitlab::HashedStorage::Migrator do
end
end
end
describe '#rollback' do
let(:project) { create(:project, :empty_repo) }
it 'enqueues project rollback job' do
Sidekiq::Testing.fake! do
expect { subject.rollback(project) }.to change(ProjectRollbackHashedStorageWorker.jobs, :size).by(1)
end
end
it 'rescues and log exceptions' do
allow(project).to receive(:rollback_to_hashed_storage!).and_raise(StandardError)
expect { subject.rollback(project) }.not_to raise_error
end
it 'rolls-back project storage' do
perform_enqueued_jobs do
subject.rollback(project)
end
expect(project.reload.legacy_storage?).to be_truthy
end
it 'has rolled-back project set as writable' do
perform_enqueued_jobs do
subject.rollback(project)
end
expect(project.reload.repository_read_only?).to be_falsey
end
context 'when project is already on legacy storage' do
let(:project) { create(:project, :legacy_storage, :empty_repo) }
it 'doesnt enqueue any rollback job' do
Sidekiq::Testing.fake! do
expect { subject.rollback(project) }.not_to change(ProjectRollbackHashedStorageWorker.jobs, :size)
end
end
it 'returns false' do
expect(subject.rollback(project)).to be_falsey
end
end
end
end
......@@ -3452,6 +3452,20 @@ describe Project do
project.migrate_to_hashed_storage!
end
end
describe '#rollback_to_legacy_storage!' do
let(:project) { create(:project, :empty_repo, :legacy_storage) }
it 'returns nil' do
expect(project.rollback_to_legacy_storage!).to be_nil
end
it 'does not run validations' do
expect(project).not_to receive(:valid?)
project.rollback_to_legacy_storage!
end
end
end
context 'hashed storage' do
......@@ -3532,6 +3546,30 @@ describe Project do
end
end
end
describe '#rollback_to_legacy_storage!' do
let(:project) { create(:project, :repository, skip_disk_validation: true) }
it 'returns true' do
expect(project.rollback_to_legacy_storage!).to be_truthy
end
it 'does not run validations' do
expect(project).not_to receive(:valid?)
project.rollback_to_legacy_storage!
end
it 'does not flag as read-only' do
expect { project.rollback_to_legacy_storage! }.not_to change { project.repository_read_only }
end
it 'enqueues a job' do
Sidekiq::Testing.fake! do
expect { project.rollback_to_legacy_storage! }.to change(ProjectRollbackHashedStorageWorker.jobs, :size).by(1)
end
end
end
end
describe '#gl_repository' do
......
# frozen_string_literal: true
require 'spec_helper'
describe Projects::HashedStorage::RollbackRepositoryService, :clean_gitlab_redis_shared_state do
include GitHelpers
let(:gitlab_shell) { Gitlab::Shell.new }
let(:project) { create(:project, :repository, :wiki_repo, storage_version: ::Project::HASHED_STORAGE_FEATURES[:repository]) }
let(:legacy_storage) { Storage::LegacyProject.new(project) }
let(:hashed_storage) { Storage::HashedProject.new(project) }
subject(:service) { described_class.new(project, project.disk_path) }
describe '#execute' do
let(:old_disk_path) { hashed_storage.disk_path }
let(:new_disk_path) { legacy_storage.disk_path }
before do
allow(service).to receive(:gitlab_shell) { gitlab_shell }
end
context 'repository lock' do
it 'tries to lock the repository' do
expect(service).to receive(:try_to_set_repository_read_only!)
service.execute
end
it 'fails when a git operation is in progress' do
allow(project).to receive(:repo_reference_count) { 1 }
expect { service.execute }.to raise_error(Projects::HashedStorage::RepositoryRollbackError)
end
end
context 'when succeeds' do
it 'renames project and wiki repositories' do
service.execute
expect(gitlab_shell.exists?(project.repository_storage, "#{new_disk_path}.git")).to be_truthy
expect(gitlab_shell.exists?(project.repository_storage, "#{new_disk_path}.wiki.git")).to be_truthy
end
it 'updates project to be legacy and not read-only' do
service.execute
expect(project.legacy_storage?).to be_truthy
expect(project.repository_read_only).to be_falsey
end
it 'move operation is called for both repositories' do
expect_move_repository(old_disk_path, new_disk_path)
expect_move_repository("#{old_disk_path}.wiki", "#{new_disk_path}.wiki")
service.execute
end
it 'writes project full path to .git/config' do
service.execute
rugged_config = rugged_repo(project.repository).config['gitlab.fullpath']
expect(rugged_config).to eq project.full_path
end
end
context 'when one move fails' do
it 'rollsback repositories to original name' do
allow(service).to receive(:move_repository).and_call_original
allow(service).to receive(:move_repository).with(old_disk_path, new_disk_path).once { false } # will disable first move only
expect(service).to receive(:rollback_folder_move).and_call_original
service.execute
expect(gitlab_shell.exists?(project.repository_storage, "#{new_disk_path}.git")).to be_falsey
expect(gitlab_shell.exists?(project.repository_storage, "#{new_disk_path}.wiki.git")).to be_falsey
expect(project.repository_read_only?).to be_falsey
end
context 'when rollback fails' do
before do
legacy_storage.ensure_storage_path_exists
gitlab_shell.mv_repository(project.repository_storage, old_disk_path, new_disk_path)
end
it 'does not try to move nil repository over existing' do
expect(gitlab_shell).not_to receive(:mv_repository).with(project.repository_storage, old_disk_path, new_disk_path)
expect_move_repository("#{old_disk_path}.wiki", "#{new_disk_path}.wiki")
service.execute
end
end
end
def expect_move_repository(from_name, to_name)
expect(gitlab_shell).to receive(:mv_repository).with(project.repository_storage, from_name, to_name).and_call_original
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Projects::HashedStorage::RollbackService do
let(:project) { create(:project, :empty_repo, :wiki_repo) }
let(:logger) { double }
subject(:service) { described_class.new(project, project.full_path, logger: logger) }
describe '#execute' do
context 'attachments rollback' do
let(:attachments_service_class) { Projects::HashedStorage::RollbackAttachmentsService }
let(:attachments_service) { attachments_service_class.new(project, logger: logger) }
it 'delegates rollback to Projects::HashedStorage::RollbackAttachmentsService' do
expect(attachments_service_class).to receive(:new)
.with(project, logger: logger)
.and_return(attachments_service)
expect(attachments_service).to receive(:execute)
service.execute
end
it 'does not delegate rollback if repository is in legacy storage already' do
project.storage_version = nil
expect(attachments_service_class).not_to receive(:new)
service.execute
end
end
context 'repository rollback' do
let(:repository_service_class) { Projects::HashedStorage::RollbackRepositoryService }
let(:repository_service) { repository_service_class.new(project, project.full_path, logger: logger) }
it 'delegates rollback to RollbackRepositoryService' do
project.storage_version = ::Project::HASHED_STORAGE_FEATURES[:repository]
expect(repository_service_class).to receive(:new)
.with(project, project.full_path, logger: logger)
.and_return(repository_service)
expect(repository_service).to receive(:execute)
service.execute
end
it 'does not delegate rollback if repository is in legacy storage already' do
project.storage_version = nil
expect(repository_service_class).not_to receive(:new)
service.execute
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe ProjectRollbackHashedStorageWorker, :clean_gitlab_redis_shared_state do
include ExclusiveLeaseHelpers
describe '#perform' do
let(:project) { create(:project, :empty_repo) }
let(:lease_key) { "project_migrate_hashed_storage_worker:#{project.id}" }
let(:lease_timeout) { described_class::LEASE_TIMEOUT }
let(:rollback_service) { ::Projects::HashedStorage::RollbackService }
it 'skips when project no longer exists' do
expect(rollback_service).not_to receive(:new)
subject.perform(-1)
end
it 'skips when project is pending delete' do
pending_delete_project = create(:project, :empty_repo, pending_delete: true)
expect(rollback_service).not_to receive(:new)
subject.perform(pending_delete_project.id)
end
it 'delegates rollback to service class when have exclusive lease' do
stub_exclusive_lease(lease_key, 'uuid', timeout: lease_timeout)
service_spy = spy
allow(rollback_service)
.to receive(:new).with(project, project.disk_path, logger: subject.logger)
.and_return(service_spy)
subject.perform(project.id)
expect(service_spy).to have_received(:execute)
end
it 'skips when it cant acquire the exclusive lease' do
stub_exclusive_lease_taken(lease_key, timeout: lease_timeout)
expect(rollback_service).not_to receive(:new)
subject.perform(project.id)
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment