Finish migration to the new events setup

This finishes the procedure for migrating events from the old format
into the new format. Code no longer uses the old setup and the database
tables used during the migration process are swapped, with the old table
being dropped.

While the database migration can be reversed this will 1) take a lot of
time as data has to be coped around 2) won't restore data in the
"events.data" column as we have no way of restoring this.

Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/37241
parent 1632ffa6
class Event < ActiveRecord::Base
include Sortable
include IgnorableColumn
default_scope { reorder(nil).where.not(author_id: nil) }
CREATED = 1
......@@ -50,13 +51,9 @@ class Event < ActiveRecord::Base
belongs_to :target, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
has_one :push_event_payload, foreign_key: :event_id
# For Hash only
serialize :data # rubocop:disable Cop/ActiveRecordSerialize
# Callbacks
after_create :reset_project_activity
after_create :set_last_repository_updated_at, if: :push?
after_create :replicate_event_for_push_events_migration
# Scopes
scope :recent, -> { reorder(id: :desc) }
......@@ -82,6 +79,10 @@ class Event < ActiveRecord::Base
self.inheritance_column = 'action'
# "data" will be removed in 10.0 but it may be possible that JOINs happen that
# include this column, hence we're ignoring it as well.
ignore_column :data
class << self
def model_name
ActiveModel::Name.new(self, nil, 'event')
......@@ -159,7 +160,7 @@ class Event < ActiveRecord::Base
end
def push?
action == PUSHED && valid_push?
false
end
def merged?
......@@ -272,87 +273,6 @@ class Event < ActiveRecord::Base
end
end
def valid_push?
data[:ref] && ref_name.present?
rescue
false
end
def tag?
Gitlab::Git.tag_ref?(data[:ref])
end
def branch?
Gitlab::Git.branch_ref?(data[:ref])
end
def new_ref?
Gitlab::Git.blank_ref?(commit_from)
end
def rm_ref?
Gitlab::Git.blank_ref?(commit_to)
end
def md_ref?
!(rm_ref? || new_ref?)
end
def commit_from
data[:before]
end
def commit_to
data[:after]
end
def ref_name
if tag?
tag_name
else
branch_name
end
end
def branch_name
@branch_name ||= Gitlab::Git.ref_name(data[:ref])
end
def tag_name
@tag_name ||= Gitlab::Git.ref_name(data[:ref])
end
# Max 20 commits from push DESC
def commits
@commits ||= (data[:commits] || []).reverse
end
def commit_title
commit = commits.last
commit[:message] if commit
end
def commit_id
commit_to || commit_from
end
def commits_count
data[:total_commits_count] || commits.count || 0
end
def ref_type
tag? ? "tag" : "branch"
end
def push_with_commits?
!commits.empty? && commit_from && commit_to
end
def last_push_to_non_root?
branch? && project.default_branch != branch_name
end
def target_iid
target.respond_to?(:iid) ? target.iid : target_id
end
......@@ -432,16 +352,6 @@ class Event < ActiveRecord::Base
user ? author_id == user.id : false
end
# We're manually replicating data into the new table since database triggers
# are not dumped to db/schema.rb. This could mean that a new installation
# would not have the triggers in place, thus losing events data in GitLab
# 10.0.
def replicate_event_for_push_events_migration
new_attributes = attributes.with_indifferent_access.except(:title, :data)
EventForMigration.create!(new_attributes)
end
def to_partial_path
# We are intentionally using `Event` rather than `self.class` so that
# subclasses also use the `Event` implementation.
......
# This model is used to replicate events between the old "events" table and the
# new "events_for_migration" table that will replace "events" in GitLab 10.0.
class EventForMigration < ActiveRecord::Base
self.table_name = 'events_for_migration'
end
......@@ -15,15 +15,21 @@ class PushEvent < Event
# should ensure the ID points to a valid project.
validates :project_id, presence: true
# The "data" field must not be set for push events since it's not used and a
# waste of space.
validates :data, absence: true
# These fields are also not used for push events, thus storing them would be a
# waste.
validates :target_id, absence: true
validates :target_type, absence: true
delegate :branch?, to: :push_event_payload
delegate :tag?, to: :push_event_payload
delegate :commit_from, to: :push_event_payload
delegate :commit_to, to: :push_event_payload
delegate :ref_type, to: :push_event_payload
delegate :commit_title, to: :push_event_payload
delegate :commit_count, to: :push_event_payload
alias_method :commits_count, :commit_count
def self.sti_name
PUSHED
end
......@@ -36,86 +42,35 @@ class PushEvent < Event
!!(commit_from && commit_to)
end
def tag?
return super unless push_event_payload
push_event_payload.tag?
end
def branch?
return super unless push_event_payload
push_event_payload.branch?
end
def valid_push?
return super unless push_event_payload
push_event_payload.ref.present?
end
def new_ref?
return super unless push_event_payload
push_event_payload.created?
end
def rm_ref?
return super unless push_event_payload
push_event_payload.removed?
end
def commit_from
return super unless push_event_payload
push_event_payload.commit_from
end
def commit_to
return super unless push_event_payload
push_event_payload.commit_to
def md_ref?
!(rm_ref? || new_ref?)
end
def ref_name
return super unless push_event_payload
push_event_payload.ref
end
def ref_type
return super unless push_event_payload
push_event_payload.ref_type
end
def branch_name
return super unless push_event_payload
ref_name
end
def tag_name
return super unless push_event_payload
ref_name
end
def commit_title
return super unless push_event_payload
push_event_payload.commit_title
end
alias_method :branch_name, :ref_name
alias_method :tag_name, :ref_name
def commit_id
commit_to || commit_from
end
def commits_count
return super unless push_event_payload
push_event_payload.commit_count
def last_push_to_non_root?
branch? && project.default_branch != branch_name
end
def validate_push_action
......
---
title: Finish migration to the new events setup
merge_request:
author:
type: changed
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class StealRemainingEventMigrationJobs < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
disable_ddl_transaction!
def up
Gitlab::BackgroundMigration.steal('MigrateEventsToPushEventPayloads')
end
def down
end
end
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class SwapEventMigrationTables < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
def up
rename_tables
end
def down
rename_tables
end
def rename_tables
rename_table :events, :events_old
rename_table :events_for_migration, :events
rename_table :events_old, :events_for_migration
end
end
......@@ -7,6 +7,5 @@ class LimitsToMysql < ActiveRecord::Migration
change_column :merge_request_diffs, :st_diffs, :text, limit: 2147483647
change_column :snippets, :content, :text, limit: 2147483647
change_column :notes, :st_diff, :text, limit: 2147483647
change_column :events, :data, :text, limit: 2147483647
end
end
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class DropEventsForMigrationTable < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
class Event < ActiveRecord::Base
include EachBatch
end
def up
transaction do
drop_table :events_for_migration
end
end
# rubocop: disable Migration/Datetime
def down
create_table :events_for_migration do |t|
t.string :target_type, index: true
t.integer :target_id, index: true
t.string :title
t.text :data
t.integer :project_id
t.datetime :created_at, index: true
t.datetime :updated_at
t.integer :action, index: true
t.integer :author_id, index: true
t.index [:project_id, :id]
end
Event.all.each_batch do |relation|
start_id, stop_id = relation.pluck('MIN(id), MAX(id)').first
execute <<-EOF.strip_heredoc
INSERT INTO events_for_migration (target_type, target_id, project_id, created_at, updated_at, action, author_id)
SELECT target_type, target_id, project_id, created_at, updated_at, action, author_id
FROM events
WHERE id BETWEEN #{start_id} AND #{stop_id}
EOF
end
end
end
......@@ -531,25 +531,6 @@ ActiveRecord::Schema.define(version: 20170901071411) do
add_index "environments", ["project_id", "slug"], name: "index_environments_on_project_id_and_slug", unique: true, using: :btree
create_table "events", force: :cascade do |t|
t.string "target_type"
t.integer "target_id"
t.string "title"
t.text "data"
t.integer "project_id"
t.datetime "created_at"
t.datetime "updated_at"
t.integer "action"
t.integer "author_id"
end
add_index "events", ["action"], name: "index_events_on_action", using: :btree
add_index "events", ["author_id"], name: "index_events_on_author_id", using: :btree
add_index "events", ["created_at"], name: "index_events_on_created_at", using: :btree
add_index "events", ["project_id", "id"], name: "index_events_on_project_id_and_id", using: :btree
add_index "events", ["target_id"], name: "index_events_on_target_id", using: :btree
add_index "events", ["target_type"], name: "index_events_on_target_type", using: :btree
create_table "events_for_migration", force: :cascade do |t|
t.integer "project_id"
t.integer "author_id", null: false
t.integer "target_id"
......@@ -559,10 +540,10 @@ ActiveRecord::Schema.define(version: 20170901071411) do
t.string "target_type"
end
add_index "events_for_migration", ["action"], name: "index_events_for_migration_on_action", using: :btree
add_index "events_for_migration", ["author_id"], name: "index_events_for_migration_on_author_id", using: :btree
add_index "events_for_migration", ["project_id", "id"], name: "index_events_for_migration_on_project_id_and_id", using: :btree
add_index "events_for_migration", ["target_type", "target_id"], name: "index_events_for_migration_on_target_type_and_target_id", using: :btree
add_index "events", ["action"], name: "index_events_on_action", using: :btree
add_index "events", ["author_id"], name: "index_events_on_author_id", using: :btree
add_index "events", ["project_id", "id"], name: "index_events_on_project_id_and_id", using: :btree
add_index "events", ["target_type", "target_id"], name: "index_events_on_target_type_and_target_id", using: :btree
create_table "feature_gates", force: :cascade do |t|
t.string "feature_key", null: false
......@@ -1697,9 +1678,8 @@ ActiveRecord::Schema.define(version: 20170901071411) do
add_foreign_key "deploy_keys_projects", "projects", name: "fk_58a901ca7e", on_delete: :cascade
add_foreign_key "deployments", "projects", name: "fk_b9a3851b82", on_delete: :cascade
add_foreign_key "environments", "projects", name: "fk_d1c8c1da6a", on_delete: :cascade
add_foreign_key "events", "projects", name: "fk_0434b48643", on_delete: :cascade
add_foreign_key "events_for_migration", "projects", on_delete: :cascade
add_foreign_key "events_for_migration", "users", column: "author_id", name: "fk_edfd187b6f", on_delete: :cascade
add_foreign_key "events", "projects", on_delete: :cascade
add_foreign_key "events", "users", column: "author_id", name: "fk_edfd187b6f", on_delete: :cascade
add_foreign_key "forked_project_links", "projects", column: "forked_to_project_id", name: "fk_434510edb0", on_delete: :cascade
add_foreign_key "gpg_keys", "users", on_delete: :cascade
add_foreign_key "gpg_signatures", "gpg_keys", on_delete: :nullify
......@@ -1743,7 +1723,7 @@ ActiveRecord::Schema.define(version: 20170901071411) do
add_foreign_key "protected_tag_create_access_levels", "protected_tags", name: "fk_f7dfda8c51", on_delete: :cascade
add_foreign_key "protected_tag_create_access_levels", "users"
add_foreign_key "protected_tags", "projects", name: "fk_8e4af87648", on_delete: :cascade
add_foreign_key "push_event_payloads", "events_for_migration", column: "event_id", name: "fk_36c74129da", on_delete: :cascade
add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade
add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade
add_foreign_key "services", "projects", name: "fk_71cce407f9", on_delete: :cascade
add_foreign_key "snippets", "projects", name: "fk_be41fd4bb7", on_delete: :cascade
......
......@@ -28,17 +28,18 @@ with all their related data and be moved into a new GitLab instance.
## Version history
| GitLab version | Import/Export version |
| -------- | -------- |
| 9.4.0 to current | 0.1.8 |
| 9.2.0 | 0.1.7 |
| 8.17.0 | 0.1.6 |
| 8.13.0 | 0.1.5 |
| 8.12.0 | 0.1.4 |
| 8.10.3 | 0.1.3 |
| 8.10.0 | 0.1.2 |
| 8.9.5 | 0.1.1 |
| 8.9.0 | 0.1.0 |
| GitLab version | Import/Export version |
| ---------------- | --------------------- |
| 10.0 to current | 0.2.0 |
| 9.4.0 | 0.1.8 |
| 9.2.0 | 0.1.7 |
| 8.17.0 | 0.1.6 |
| 8.13.0 | 0.1.5 |
| 8.12.0 | 0.1.4 |
| 8.10.3 | 0.1.3 |
| 8.10.0 | 0.1.2 |
| 8.9.5 | 0.1.1 |
| 8.9.0 | 0.1.0 |
> The table reflects what GitLab version we updated the Import/Export version at.
> For instance, 8.10.3 and 8.11 will have the same Import/Export version (0.1.3)
......
......@@ -17,14 +17,9 @@ class Spinach::Features::User < Spinach::FeatureSteps
Issues::CreateService.new(project, user, issue_params).execute
# Push code contribution
push_params = {
project: project,
action: Event::PUSHED,
author_id: user.id,
data: { commit_count: 3 }
}
Event.create(push_params)
event = create(:push_event, project: project, author: user)
create(:push_event_payload, event: event, commit_count: 3)
end
step 'I should see contributed projects' do
......@@ -38,6 +33,6 @@ class Spinach::Features::User < Spinach::FeatureSteps
end
def contributed_project
@contributed_project ||= create(:project, :public)
@contributed_project ||= create(:project, :public, :empty_repo)
end
end
......@@ -545,7 +545,7 @@ module API
end
class Event < Grape::Entity
expose :title, :project_id, :action_name
expose :project_id, :action_name
expose :target_id, :target_iid, :target_type, :author_id
expose :target_title
expose :created_at
......
......@@ -31,7 +31,7 @@ module API
end
class Event < Grape::Entity
expose :title, :project_id, :action_name
expose :project_id, :action_name
expose :target_id, :target_type, :author_id
expose :target_title
expose :created_at
......
......@@ -3,7 +3,7 @@ module Gitlab
extend self
# For every version update, the version history in import_export.md has to be kept up to date.
VERSION = '0.1.8'.freeze
VERSION = '0.2.0'.freeze
FILENAME_LIMIT = 50
def export_path(relative_path:)
......
......@@ -69,7 +69,6 @@ module Gitlab
reset_tokens!
remove_encrypted_attributes!
@relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data']
set_st_diff_commits if @relation_name == :merge_request_diff
set_diff if @relation_name == :merge_request_diff_files
end
......
......@@ -9,5 +9,16 @@ namespace :gitlab do
task data: :environment do
puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true)
end
desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz'
task bump_test_version: :environment do
Dir.mktmpdir do |tmp_dir|
system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null")
File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null")
end
puts "Updated to #{Gitlab::ImportExport.version}"
end
end
end
......@@ -84,25 +84,11 @@ feature 'Dashboard Projects' do
end
context 'last push widget' do
let(:push_event_data) do
{
before: Gitlab::Git::BLANK_SHA,
after: '0220c11b9a3e6c69dc8fd35321254ca9a7b98f7e',
ref: 'refs/heads/feature',
user_id: user.id,
user_name: user.name,
repository: {
name: project.name,
url: 'localhost/rubinius',
description: '',
homepage: 'localhost/rubinius',
private: true
}
}
end
let!(:push_event) { create(:event, :pushed, data: push_event_data, project: project, author: user) }
before do
event = create(:push_event, project: project, author: user)
create(:push_event_payload, event: event, ref: 'feature', action: :created)
visit dashboard_projects_path
end
......
require 'spec_helper'
describe Gitlab::BackgroundMigration::MigrateEventsToPushEventPayloads::Event do
describe Gitlab::BackgroundMigration::MigrateEventsToPushEventPayloads::Event, :migration, schema: 20170608152748 do
describe '#commit_title' do
it 'returns nil when there are no commits' do
expect(described_class.new.commit_title).to be_nil
......
......@@ -75,8 +75,6 @@
"id": 487,
"target_type": "Milestone",
"target_id": 1,
"title": null,
"data": null,
"project_id": 46,
"created_at": "2016-06-14T15:02:04.418Z",
"updated_at": "2016-06-14T15:02:04.418Z",
......@@ -364,8 +362,6 @@
"id": 487,
"target_type": "Milestone",
"target_id": 1,
"title": null,
"data": null,
"project_id": 46,
"created_at": "2016-06-14T15:02:04.418Z",
"updated_at": "2016-06-14T15:02:04.418Z",
......@@ -2311,8 +2307,6 @@
"id": 487,
"target_type": "Milestone",
"target_id": 1,
"title": null,
"data": null,
"project_id": 46,
"created_at": "2016-06-14T15:02:04.418Z",
"updated_at": "2016-06-14T15:02:04.418Z",
......@@ -2336,8 +2330,6 @@
"id": 240,
"target_type": "Milestone",
"target_id": 20,
"title": null,
"data": null,
"project_id": 36,
"created_at": "2016-06-14T15:02:04.593Z",
"updated_at": "2016-06-14T15:02:04.593Z",
......@@ -2348,8 +2340,6 @@
"id": 60,
"target_type": "Milestone",
"target_id": 20,
"title": null,
"data": null,
"project_id": 5,
"created_at": "2016-06-14T15:02:04.593Z",
"updated_at": "2016-06-14T15:02:04.593Z",
......@@ -2373,8 +2363,6 @@
"id": 241,
"target_type": "Milestone",
"target_id": 19,
"title": null,
"data": null,
"project_id": 36,
"created_at": "2016-06-14T15:02:04.585Z",
"updated_at": "2016-06-14T15:02:04.585Z",
......@@ -2385,41 +2373,6 @@
"id": 59,
"target_type": "Milestone",
"target_id": 19,
"title": null,
"data": {
"object_kind": "push",
"before": "0000000000000000000000000000000000000000",
"after": "de990aa15829d0ab182ad5a55b4c527846c0d39c",
"ref": "refs/heads/removable-group-owner",
"checkout_sha": "de990aa15829d0ab182ad5a55b4c527846c0d39c",
"message": null,
"user_id": 273486,
"user_name": "James Lopez",
"user_email": "james@jameslopez.es",
"project_id": 562317,
"repository": {
"name": "GitLab Community Edition",
"url": "git@gitlab.com:james11/gitlab-ce.git",
"description": "Version Control on your Server. See http://gitlab.org/gitlab-ce/ and the README for more information",
"homepage": "https://gitlab.com/james11/gitlab-ce",
"git_http_url": "https://gitlab.com/james11/gitlab-ce.git",
"git_ssh_url": "git@gitlab.com:james11/gitlab-ce.git",
"visibility_level": 20
},
"commits": [
{
"id": "de990aa15829d0ab182ad5a55b4c527846c0d39c",
"message": "fixed last group owner issue and added test\\n",
"timestamp": "2015-10-29T16:10:27+00:00",
"url": "https://gitlab.com/james11/gitlab-ce/commit/de990aa15829d0ab182ad5a55b4c527846c0d39c",
"author": {
"name": "James Lopez",
"email": "james.lopez@vodafone.com"
}
}
],
"total_commits_count": 1
},
"project_id": 5,
"created_at": "2016-06-14T15:02:04.585Z",
"updated_at": "2016-06-14T15:02:04.585Z",
......@@ -2947,8 +2900,6 @@
"id": 221,
"target_type": "MergeRequest",
"target_id": 27,
"title": null,
"data": null,
"project_id": 36,
"created_at": "2016-06-14T15:02:36.703Z",
"updated_at": "2016-06-14T15:02:36.703Z",
......@@ -2959,8 +2910,6 @@
"id": 187,
"target_type": "MergeRequest",
"target_id": 27,
"title": null,
"data": null,
"project_id": 5,
"created_at": "2016-06-14T15:02:36.703Z",
"updated_at": "2016-06-14T15:02:36.703Z",
......@@ -3230,8 +3179,6 @@
"id": 222,
"target_type": "MergeRequest",
"target_id": 26,
"title": null,
"data": null,
"project_id": 36,
"created_at": "2016-06-14T15:02:36.496Z",
"updated_at": "2016-06-14T15:02:36.496Z",
......@@ -3242,8 +3189,6 @@
"id": 186,
"target_type": "MergeRequest",
"target_id": 26,
"title": null,
"data": null,
"project_id": 5,
"created_at": "2016-06-14T15:02:36.496Z",
"updated_at": "2016-06-14T15:02:36.496Z",
......@@ -3513,8 +3458,6 @@
"id": 223,
"target_type": "MergeRequest",
"target_id": 15,
"title": null,
"data": null,
"project_id": 36,
"created_at": "2016-06-14T15:02:25.262Z",
"updated_at": "2016-06-14T15:02:25.262Z",
......@@ -3525,8 +3468,6 @@
"id": 175,
"target_type": "MergeRequest",
"target_id": 15,