Commit d63380fa authored by Gabriel Mazetto's avatar Gabriel Mazetto

Refactor ProjectMigrate and ProjectRollback workers

Moved to HashedStorage namespace, and added them to the
`:hashed_storage` queue namespace
parent 1592b583
......@@ -1970,9 +1970,9 @@ class Project < ActiveRecord::Base
return unless storage_upgradable?
if git_transfer_in_progress?
ProjectMigrateHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id)
HashedStorage::ProjectMigrateWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id)
else
ProjectMigrateHashedStorageWorker.perform_async(id)
HashedStorage::ProjectMigrateWorker.perform_async(id)
end
end
......@@ -1980,9 +1980,9 @@ class Project < ActiveRecord::Base
return if legacy_storage?
if git_transfer_in_progress?
ProjectRollbackHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id)
HashedStorage::ProjectRollbackWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id)
else
ProjectRollbackHashedStorageWorker.perform_async(id)
HashedStorage::ProjectRollbackWorker.perform_async(id)
end
end
......
......@@ -47,6 +47,8 @@
- github_importer:github_import_stage_import_repository
- hashed_storage:hashed_storage_migrator
- hashed_storage:hashed_storage_project_migrate
- hashed_storage:hashed_storage_project_rollback
- mail_scheduler:mail_scheduler_issue_due
- mail_scheduler:mail_scheduler_notification_service
......@@ -126,8 +128,6 @@
- project_cache
- project_destroy
- project_export
- project_migrate_hashed_storage
- project_rollback_hashed_storage
- project_service
- propagate_service_template
- reactive_caching
......
# frozen_string_literal: true
module HashedStorage
class ProjectMigrateWorker
include ApplicationWorker
LEASE_TIMEOUT = 30.seconds.to_i
LEASE_KEY_SEGMENT = 'project_migrate_hashed_storage_worker'.freeze
queue_namespace :hashed_storage
# rubocop: disable CodeReuse/ActiveRecord
def perform(project_id, old_disk_path = nil)
uuid = lease_for(project_id).try_obtain
if uuid
project = Project.find_by(id: project_id)
return if project.nil? || project.pending_delete?
old_disk_path ||= project.disk_path
::Projects::HashedStorage::MigrationService.new(project, old_disk_path, logger: logger).execute
else
return false
end
ensure
cancel_lease_for(project_id, uuid) if uuid
end
# rubocop: enable CodeReuse/ActiveRecord
def lease_for(project_id)
Gitlab::ExclusiveLease.new(lease_key(project_id), timeout: LEASE_TIMEOUT)
end
private
def lease_key(project_id)
# we share the same lease key for both migration and rollback so they don't run simultaneously
"#{LEASE_KEY_SEGMENT}:#{project_id}"
end
def cancel_lease_for(project_id, uuid)
Gitlab::ExclusiveLease.cancel(lease_key(project_id), uuid)
end
end
end
# frozen_string_literal: true
module HashedStorage
class ProjectRollbackWorker
include ApplicationWorker
LEASE_TIMEOUT = 30.seconds.to_i
queue_namespace :hashed_storage
# rubocop: disable CodeReuse/ActiveRecord
def perform(project_id, old_disk_path = nil)
uuid = lease_for(project_id).try_obtain
if uuid
project = Project.find_by(id: project_id)
return if project.nil? || project.pending_delete?
old_disk_path ||= project.disk_path
::Projects::HashedStorage::RollbackService.new(project, old_disk_path, logger: logger).execute
else
return false
end
ensure
cancel_lease_for(project_id, uuid) if uuid
end
# rubocop: enable CodeReuse/ActiveRecord
def lease_for(project_id)
Gitlab::ExclusiveLease.new(lease_key(project_id), timeout: LEASE_TIMEOUT)
end
private
def lease_key(project_id)
# we share the same lease key for both migration and rollback so they don't run simultaneously
"#{ProjectMigrateWorker::LEASE_KEY_SEGMENT}:#{project_id}"
end
def cancel_lease_for(project_id, uuid)
Gitlab::ExclusiveLease.cancel(lease_key(project_id), uuid)
end
end
end
# frozen_string_literal: true
class ProjectMigrateHashedStorageWorker
include ApplicationWorker
LEASE_TIMEOUT = 30.seconds.to_i
LEASE_KEY_SEGMENT = 'project_migrate_hashed_storage_worker'.freeze
# rubocop: disable CodeReuse/ActiveRecord
def perform(project_id, old_disk_path = nil)
uuid = lease_for(project_id).try_obtain
if uuid
project = Project.find_by(id: project_id)
return if project.nil? || project.pending_delete?
old_disk_path ||= project.disk_path
::Projects::HashedStorage::MigrationService.new(project, old_disk_path, logger: logger).execute
else
return false
end
ensure
cancel_lease_for(project_id, uuid) if uuid
end
# rubocop: enable CodeReuse/ActiveRecord
def lease_for(project_id)
Gitlab::ExclusiveLease.new(lease_key(project_id), timeout: LEASE_TIMEOUT)
end
private
def lease_key(project_id)
# we share the same lease key for both migration and rollback so they don't run simultaneously
"#{LEASE_KEY_SEGMENT}:#{project_id}"
end
def cancel_lease_for(project_id, uuid)
Gitlab::ExclusiveLease.cancel(lease_key(project_id), uuid)
end
end
# frozen_string_literal: true
class ProjectRollbackHashedStorageWorker
include ApplicationWorker
LEASE_TIMEOUT = 30.seconds.to_i
# rubocop: disable CodeReuse/ActiveRecord
def perform(project_id, old_disk_path = nil)
uuid = lease_for(project_id).try_obtain
if uuid
project = Project.find_by(id: project_id)
return if project.nil? || project.pending_delete?
old_disk_path ||= project.disk_path
::Projects::HashedStorage::RollbackService.new(project, old_disk_path, logger: logger).execute
else
return false
end
ensure
cancel_lease_for(project_id, uuid) if uuid
end
# rubocop: enable CodeReuse/ActiveRecord
def lease_for(project_id)
Gitlab::ExclusiveLease.new(lease_key(project_id), timeout: LEASE_TIMEOUT)
end
private
def lease_key(project_id)
# we share the same lease key for both migration and rollback so they don't run simultaneously
"#{ProjectMigrateHashedStorageWorker::LEASE_KEY_SEGMENT}:#{project_id}"
end
def cancel_lease_for(project_id, uuid)
Gitlab::ExclusiveLease.cancel(lease_key(project_id), uuid)
end
end
......@@ -13,9 +13,9 @@ describe Gitlab::HashedStorage::Migrator do
let(:projects) { create_list(:project, 2, :legacy_storage) }
let(:ids) { projects.map(&:id) }
it 'enqueue jobs to ProjectMigrateHashedStorageWorker' do
it 'enqueue jobs to HashedStorage::ProjectMigrateWorker' do
Sidekiq::Testing.fake! do
expect { subject.bulk_migrate(start: ids.min, finish: ids.max) }.to change(ProjectMigrateHashedStorageWorker.jobs, :size).by(2)
expect { subject.bulk_migrate(start: ids.min, finish: ids.max) }.to change(HashedStorage::ProjectMigrateWorker.jobs, :size).by(2)
end
end
......@@ -48,7 +48,7 @@ describe Gitlab::HashedStorage::Migrator do
it 'enqueues project migration job' do
Sidekiq::Testing.fake! do
expect { subject.migrate(project) }.to change(ProjectMigrateHashedStorageWorker.jobs, :size).by(1)
expect { subject.migrate(project) }.to change(HashedStorage::ProjectMigrateWorker.jobs, :size).by(1)
end
end
......@@ -79,7 +79,7 @@ describe Gitlab::HashedStorage::Migrator do
it 'doesnt enqueue any migration job' do
Sidekiq::Testing.fake! do
expect { subject.migrate(project) }.not_to change(ProjectMigrateHashedStorageWorker.jobs, :size)
expect { subject.migrate(project) }.not_to change(HashedStorage::ProjectMigrateWorker.jobs, :size)
end
end
......@@ -94,7 +94,7 @@ describe Gitlab::HashedStorage::Migrator do
it 'enqueues project rollback job' do
Sidekiq::Testing.fake! do
expect { subject.rollback(project) }.to change(ProjectRollbackHashedStorageWorker.jobs, :size).by(1)
expect { subject.rollback(project) }.to change(HashedStorage::ProjectRollbackWorker.jobs, :size).by(1)
end
end
......@@ -125,7 +125,7 @@ describe Gitlab::HashedStorage::Migrator do
it 'doesnt enqueue any rollback job' do
Sidekiq::Testing.fake! do
expect { subject.rollback(project) }.not_to change(ProjectRollbackHashedStorageWorker.jobs, :size)
expect { subject.rollback(project) }.not_to change(HashedStorage::ProjectRollbackWorker.jobs, :size)
end
end
......
......@@ -3430,24 +3430,24 @@ describe Project do
project.migrate_to_hashed_storage!
end
it 'schedules ProjectMigrateHashedStorageWorker with delayed start when the project repo is in use' do
it 'schedules HashedStorage::ProjectMigrateWorker with delayed start when the project repo is in use' do
Gitlab::ReferenceCounter.new(project.gl_repository(is_wiki: false)).increase
expect(ProjectMigrateHashedStorageWorker).to receive(:perform_in)
expect(HashedStorage::ProjectMigrateWorker).to receive(:perform_in)
project.migrate_to_hashed_storage!
end
it 'schedules ProjectMigrateHashedStorageWorker with delayed start when the wiki repo is in use' do
it 'schedules HashedStorage::ProjectMigrateWorker with delayed start when the wiki repo is in use' do
Gitlab::ReferenceCounter.new(project.gl_repository(is_wiki: true)).increase
expect(ProjectMigrateHashedStorageWorker).to receive(:perform_in)
expect(HashedStorage::ProjectMigrateWorker).to receive(:perform_in)
project.migrate_to_hashed_storage!
end
it 'schedules ProjectMigrateHashedStorageWorker' do
expect(ProjectMigrateHashedStorageWorker).to receive(:perform_async).with(project.id)
it 'schedules HashedStorage::ProjectMigrateWorker' do
expect(HashedStorage::ProjectMigrateWorker).to receive(:perform_async).with(project.id)
project.migrate_to_hashed_storage!
end
......@@ -3541,7 +3541,7 @@ describe Project do
project = create(:project, storage_version: 1, skip_disk_validation: true)
Sidekiq::Testing.fake! do
expect { project.migrate_to_hashed_storage! }.to change(ProjectMigrateHashedStorageWorker.jobs, :size).by(1)
expect { project.migrate_to_hashed_storage! }.to change(HashedStorage::ProjectMigrateWorker.jobs, :size).by(1)
end
end
end
......@@ -3566,7 +3566,7 @@ describe Project do
it 'enqueues a job' do
Sidekiq::Testing.fake! do
expect { project.rollback_to_legacy_storage! }.to change(ProjectRollbackHashedStorageWorker.jobs, :size).by(1)
expect { project.rollback_to_legacy_storage! }.to change(HashedStorage::ProjectRollbackWorker.jobs, :size).by(1)
end
end
end
......
require 'spec_helper'
describe ProjectMigrateHashedStorageWorker, :clean_gitlab_redis_shared_state do
describe HashedStorage::ProjectMigrateWorker, :clean_gitlab_redis_shared_state do
include ExclusiveLeaseHelpers
describe '#perform' do
......
......@@ -2,7 +2,7 @@
require 'spec_helper'
describe ProjectRollbackHashedStorageWorker, :clean_gitlab_redis_shared_state do
describe HashedStorage::ProjectRollbackWorker, :clean_gitlab_redis_shared_state do
include ExclusiveLeaseHelpers
describe '#perform' do
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment