Commit 7b223816 authored by Jan Provaznik's avatar Jan Provaznik

Reschedule commits_count background migration

We still have >100K unmigrated MergeRequestDiffs
which don't have commits_count set yet (see
https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/17567#note_61904891)

This migration re-schedules the original background migration.
To assure that records are not processed twice, records with
commits_count set are skipped.

Related to #41698 and !17567
parent 47b9854d
class RescheduleCommitsCountForMergeRequestDiff < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
MIGRATION = 'AddMergeRequestDiffCommitsCount'.freeze
BATCH_SIZE = 5000
DELAY_INTERVAL = 5.minutes.to_i
class MergeRequestDiff < ActiveRecord::Base
self.table_name = 'merge_request_diffs'
include ::EachBatch
end
disable_ddl_transaction!
def up
say 'Populating the MergeRequestDiff `commits_count` (reschedule)'
execute("SET statement_timeout TO '60s'") if Gitlab::Database.postgresql?
MergeRequestDiff.where(commits_count: nil).each_batch(of: BATCH_SIZE) do |relation, index|
start_id, end_id = relation.pluck('MIN(id), MAX(id)').first
delay = index * DELAY_INTERVAL
BackgroundMigrationWorker.perform_in(delay, MIGRATION, [start_id, end_id])
end
end
end
......@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20180308052825) do
ActiveRecord::Schema.define(version: 20180309121820) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......
......@@ -19,7 +19,7 @@ module Gitlab
WHERE merge_request_diffs.id = merge_request_diff_commits.merge_request_diff_id
)'.squish
MergeRequestDiff.where(id: start_id..stop_id).update_all(update)
MergeRequestDiff.where(id: start_id..stop_id).where(commits_count: nil).update_all(update)
end
end
end
......
......@@ -37,6 +37,18 @@ describe Gitlab::BackgroundMigration::AddMergeRequestDiffCommitsCount, :migratio
expect(diff.reload.commits_count).to eq(0)
end
it 'skips diffs that have commits_count already set' do
timestamp = 2.days.ago
diff = merge_request_diffs_table.create!(
merge_request_id: merge_request.id,
commits_count: 0,
updated_at: timestamp)
subject.perform(diff.id, diff.id)
expect(diff.reload.updated_at).to be_within(1.second).of(timestamp)
end
it 'migrates multiple diffs to the correct values' do
diffs = Array.new(3).map.with_index { |_, i| create_diff!(i, commits: 3) }
......
require 'spec_helper'
require Rails.root.join('db', 'migrate', '20180309121820_reschedule_commits_count_for_merge_request_diff')
describe RescheduleCommitsCountForMergeRequestDiff, :migration, :sidekiq do
let(:merge_request_diffs) { table(:merge_request_diffs) }
let(:merge_requests) { table(:merge_requests) }
let(:projects) { table(:projects) }
let(:namespaces) { table(:namespaces) }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 1)
namespaces.create!(id: 1, name: 'gitlab', path: 'gitlab')
projects.create!(id: 1, namespace_id: 1)
merge_requests.create!(id: 1, target_project_id: 1, source_project_id: 1, target_branch: 'feature', source_branch: 'master')
merge_request_diffs.create!(id: 1, merge_request_id: 1)
merge_request_diffs.create!(id: 2, merge_request_id: 1)
merge_request_diffs.create!(id: 3, merge_request_id: 1, commits_count: 0)
merge_request_diffs.create!(id: 4, merge_request_id: 1)
end
it 'correctly schedules background migrations' do
Sidekiq::Testing.fake! do
Timecop.freeze do
migrate!
expect(described_class::MIGRATION).to be_scheduled_delayed_migration(5.minutes, 1, 1)
expect(described_class::MIGRATION).to be_scheduled_delayed_migration(10.minutes, 2, 2)
expect(described_class::MIGRATION).to be_scheduled_delayed_migration(15.minutes, 4, 4)
expect(BackgroundMigrationWorker.jobs.size).to eq 3
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment