Commit 971604de authored by Sean McGivern's avatar Sean McGivern

Merge branch 'bvl-rename-all-reserved-paths' into 'master'

Rename all forbidden paths again

Closes #32625

See merge request !11713
parents da6a3fb0 7c53fcf1
---
title: Rename all reserved paths that could have been created
merge_request: 11713
author:
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class RenameAllReservedPathsAgain < ActiveRecord::Migration
include Gitlab::Database::RenameReservedPathsMigration::V1
DOWNTIME = false
disable_ddl_transaction!
TOP_LEVEL_ROUTES = %w[
-
.well-known
abuse_reports
admin
all
api
assets
autocomplete
ci
dashboard
explore
files
groups
health_check
help
hooks
import
invites
issues
jwt
koding
member
merge_requests
new
notes
notification_settings
oauth
profile
projects
public
repository
robots.txt
s
search
sent_notifications
services
snippets
teams
u
unicorn_test
unsubscribes
uploads
users
].freeze
PROJECT_WILDCARD_ROUTES = %w[
badges
blame
blob
builds
commits
create
create_dir
edit
environments/folders
files
find_file
gitlab-lfs/objects
info/lfs/objects
new
preview
raw
refs
tree
update
wikis
].freeze
GROUP_ROUTES = %w[
activity
analytics
audit_events
avatar
edit
group_members
hooks
issues
labels
ldap
ldap_group_links
merge_requests
milestones
notification_setting
pipeline_quota
projects
subgroups
].freeze
def up
disable_statement_timeout
TOP_LEVEL_ROUTES.each { |route| rename_root_paths(route) }
PROJECT_WILDCARD_ROUTES.each { |route| rename_wildcard_paths(route) }
GROUP_ROUTES.each { |route| rename_child_paths(route) }
end
def down
disable_statement_timeout
revert_renames
end
end
...@@ -29,6 +29,11 @@ module Gitlab ...@@ -29,6 +29,11 @@ module Gitlab
paths = Array(paths) paths = Array(paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
end end
def revert_renames
RenameProjects.new([], self).revert_renames
RenameNamespaces.new([], self).revert_renames
end
end end
end end
end end
......
...@@ -6,7 +6,10 @@ module Gitlab ...@@ -6,7 +6,10 @@ module Gitlab
attr_reader :paths, :migration attr_reader :paths, :migration
delegate :update_column_in_batches, delegate :update_column_in_batches,
:execute,
:replace_sql, :replace_sql,
:quote_string,
:say,
to: :migration to: :migration
def initialize(paths, migration) def initialize(paths, migration)
...@@ -26,24 +29,45 @@ module Gitlab ...@@ -26,24 +29,45 @@ module Gitlab
new_path = rename_path(namespace_path, old_path) new_path = rename_path(namespace_path, old_path)
new_full_path = join_routable_path(namespace_path, new_path) new_full_path = join_routable_path(namespace_path, new_path)
perform_rename(routable, old_full_path, new_full_path)
[old_full_path, new_full_path]
end
def perform_rename(routable, old_full_path, new_full_path)
# skips callbacks & validations # skips callbacks & validations
new_path = new_full_path.split('/').last
routable.class.where(id: routable) routable.class.where(id: routable)
.update_all(path: new_path) .update_all(path: new_path)
rename_routes(old_full_path, new_full_path) rename_routes(old_full_path, new_full_path)
[old_full_path, new_full_path]
end end
def rename_routes(old_full_path, new_full_path) def rename_routes(old_full_path, new_full_path)
routes = Route.arel_table
quoted_old_full_path = quote_string(old_full_path)
quoted_old_wildcard_path = quote_string("#{old_full_path}/%")
filter = if Database.mysql?
"lower(routes.path) = lower('#{quoted_old_full_path}') "\
"OR routes.path LIKE '#{quoted_old_wildcard_path}'"
else
"routes.id IN "\
"( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\
"UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )"
end
replace_statement = replace_sql(Route.arel_table[:path], replace_statement = replace_sql(Route.arel_table[:path],
old_full_path, old_full_path,
new_full_path) new_full_path)
update_column_in_batches(:routes, :path, replace_statement) do |table, query| update = Arel::UpdateManager.new(ActiveRecord::Base)
path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"]) .table(routes)
query.where(path_or_children) .set([[routes[:path], replace_statement]])
end .where(Arel::Nodes::SqlLiteral.new(filter))
execute(update.to_sql)
end end
def rename_path(namespace_path, path_was) def rename_path(namespace_path, path_was)
...@@ -86,32 +110,74 @@ module Gitlab ...@@ -86,32 +110,74 @@ module Gitlab
def move_folders(directory, old_relative_path, new_relative_path) def move_folders(directory, old_relative_path, new_relative_path)
old_path = File.join(directory, old_relative_path) old_path = File.join(directory, old_relative_path)
return unless File.directory?(old_path) unless File.directory?(old_path)
say "#{old_path} doesn't exist, skipping"
return
end
new_path = File.join(directory, new_relative_path) new_path = File.join(directory, new_relative_path)
FileUtils.mv(old_path, new_path) FileUtils.mv(old_path, new_path)
end end
def remove_cached_html_for_projects(project_ids) def remove_cached_html_for_projects(project_ids)
update_column_in_batches(:projects, :description_html, nil) do |table, query| project_ids.each do |project_id|
query.where(table[:id].in(project_ids)) update_column_in_batches(:projects, :description_html, nil) do |table, query|
end query.where(table[:id].eq(project_id))
end
update_column_in_batches(:issues, :description_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids)) update_column_in_batches(:issues, :description_html, nil) do |table, query|
query.where(table[:project_id].eq(project_id))
end
update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
query.where(table[:target_project_id].eq(project_id))
end
update_column_in_batches(:notes, :note_html, nil) do |table, query|
query.where(table[:project_id].eq(project_id))
end
update_column_in_batches(:milestones, :description_html, nil) do |table, query|
query.where(table[:project_id].eq(project_id))
end
end end
end
update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| def track_rename(type, old_path, new_path)
query.where(table[:target_project_id].in(project_ids)) key = redis_key_for_type(type)
Gitlab::Redis.with do |redis|
redis.lpush(key, [old_path, new_path].to_json)
redis.expire(key, 2.weeks.to_i)
end end
say "tracked rename: #{key}: #{old_path} -> #{new_path}"
end
update_column_in_batches(:notes, :note_html, nil) do |table, query| def reverts_for_type(type)
query.where(table[:project_id].in(project_ids)) key = redis_key_for_type(type)
Gitlab::Redis.with do |redis|
failed_reverts = []
while rename_info = redis.lpop(key)
path_before_rename, path_after_rename = JSON.parse(rename_info)
say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}"
begin
yield(path_before_rename, path_after_rename)
rescue StandardError => e
failed_reverts << rename_info
say "Renaming #{type} from #{path_after_rename} back to "\
"#{path_before_rename} failed. Review the error and try "\
"again by running the `down` action. \n"\
"#{e.message}: \n #{e.backtrace.join("\n")}"
end
end
failed_reverts.each { |rename_info| redis.lpush(key, rename_info) }
end end
end
update_column_in_batches(:milestones, :description_html, nil) do |table, query| def redis_key_for_type(type)
query.where(table[:project_id].in(project_ids)) "rename:#{migration.name}:#{type}"
end
end end
def file_storage? def file_storage?
......
...@@ -26,6 +26,12 @@ module Gitlab ...@@ -26,6 +26,12 @@ module Gitlab
def rename_namespace(namespace) def rename_namespace(namespace)
old_full_path, new_full_path = rename_path_for_routable(namespace) old_full_path, new_full_path = rename_path_for_routable(namespace)
track_rename('namespace', old_full_path, new_full_path)
rename_namespace_dependencies(namespace, old_full_path, new_full_path)
end
def rename_namespace_dependencies(namespace, old_full_path, new_full_path)
move_repositories(namespace, old_full_path, new_full_path) move_repositories(namespace, old_full_path, new_full_path)
move_uploads(old_full_path, new_full_path) move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path) move_pages(old_full_path, new_full_path)
...@@ -33,6 +39,23 @@ module Gitlab ...@@ -33,6 +39,23 @@ module Gitlab
remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
end end
def revert_renames
reverts_for_type('namespace') do |path_before_rename, current_path|
matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
namespace = MigrationClasses::Namespace.joins(:route)
.where(matches_path).first&.becomes(MigrationClasses::Namespace)
if namespace
perform_rename(namespace, current_path, path_before_rename)
rename_namespace_dependencies(namespace, current_path, path_before_rename)
else
say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\
"namespace was renamed, or no longer exists at the expected path"
end
end
end
def rename_user(old_username, new_username) def rename_user(old_username, new_username)
MigrationClasses::User.where(username: old_username) MigrationClasses::User.where(username: old_username)
.update_all(username: new_username) .update_all(username: new_username)
......
...@@ -16,12 +16,37 @@ module Gitlab ...@@ -16,12 +16,37 @@ module Gitlab
def rename_project(project) def rename_project(project)
old_full_path, new_full_path = rename_path_for_routable(project) old_full_path, new_full_path = rename_path_for_routable(project)
track_rename('project', old_full_path, new_full_path)
move_project_folders(project, old_full_path, new_full_path)
end
def move_project_folders(project, old_full_path, new_full_path)
move_repository(project, old_full_path, new_full_path) move_repository(project, old_full_path, new_full_path)
move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
move_uploads(old_full_path, new_full_path) move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path) move_pages(old_full_path, new_full_path)
end end
def revert_renames
reverts_for_type('project') do |path_before_rename, current_path|
matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
project = MigrationClasses::Project.joins(:route)
.where(matches_path).first
if project
perform_rename(project, current_path, path_before_rename)
move_project_folders(project, current_path, path_before_rename)
else
say "Couldn't rename project from #{current_path} back to "\
"#{path_before_rename}, project was renamed or no longer "\
"exists at the expected path."
end
end
end
def move_repository(project, old_path, new_path) def move_repository(project, old_path, new_path)
unless gitlab_shell.mv_repository(project.repository_storage_path, unless gitlab_shell.mv_repository(project.repository_storage_path,
old_path, old_path,
......
...@@ -6,6 +6,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca ...@@ -6,6 +6,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca
before do before do
allow(migration).to receive(:say) allow(migration).to receive(:say)
TestEnv.clean_test_path
end end
def migration_namespace(namespace) def migration_namespace(namespace)
...@@ -153,6 +154,30 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca ...@@ -153,6 +154,30 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca
end end
end end
describe '#perform_rename' do
describe 'for namespaces' do
let(:namespace) { create(:namespace, path: 'the-path') }
it 'renames the path' do
subject.perform_rename(migration_namespace(namespace), 'the-path', 'renamed')
expect(namespace.reload.path).to eq('renamed')
end
it 'renames all the routes for the namespace' do
child = create(:group, path: 'child', parent: namespace)
project = create(:project, namespace: child, path: 'the-project')
other_one = create(:namespace, path: 'the-path-is-similar')
subject.perform_rename(migration_namespace(namespace), 'the-path', 'renamed')
expect(namespace.reload.route.path).to eq('renamed')
expect(child.reload.route.path).to eq('renamed/child')
expect(project.reload.route.path).to eq('renamed/child/the-project')
expect(other_one.reload.route.path).to eq('the-path-is-similar')
end
end
end
describe '#move_pages' do describe '#move_pages' do
it 'moves the pages directory' do it 'moves the pages directory' do
expect(subject).to receive(:move_folders) expect(subject).to receive(:move_folders)
...@@ -203,4 +228,53 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca ...@@ -203,4 +228,53 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca
expect(File.exist?(expected_file)).to be(true) expect(File.exist?(expected_file)).to be(true)
end end
end end
describe '#track_rename', redis: true do
it 'tracks a rename in redis' do
key = 'rename:FakeRenameReservedPathMigrationV1:namespace'
subject.track_rename('namespace', 'path/to/namespace', 'path/to/renamed')
old_path, new_path = [nil, nil]
Gitlab::Redis.with do |redis|
rename_info = redis.lpop(key)
old_path, new_path = JSON.parse(rename_info)
end
expect(old_path).to eq('path/to/namespace')
expect(new_path).to eq('path/to/renamed')
end
end
describe '#reverts_for_type', redis: true do
it 'yields for each tracked rename' do
subject.track_rename('project', 'old_path', 'new_path')
subject.track_rename('project', 'old_path2', 'new_path2')
subject.track_rename('namespace', 'namespace_path', 'new_namespace_path')
expect { |b| subject.reverts_for_type('project', &b) }
.to yield_successive_args(%w(old_path2 new_path2), %w(old_path new_path))
expect { |b| subject.reverts_for_type('namespace', &b) }
.to yield_with_args('namespace_path', 'new_namespace_path')
end
it 'keeps the revert in redis if it failed' do
subject.track_rename('project', 'old_path', 'new_path')
subject.reverts_for_type('project') do
raise 'whatever happens, keep going!'
end
key = 'rename:FakeRenameReservedPathMigrationV1:project'
stored_renames = nil
rename_count = 0
Gitlab::Redis.with do |redis|
stored_renames = redis.lrange(key, 0, 1)
rename_count = redis.llen(key)
end
expect(rename_count).to eq(1)
expect(JSON.parse(stored_renames.first)).to eq(%w(old_path new_path))
end
end
end end
...@@ -3,9 +3,11 @@ require 'spec_helper' ...@@ -3,9 +3,11 @@ require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :truncate do describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :truncate do
let(:migration) { FakeRenameReservedPathMigrationV1.new } let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) } let(:subject) { described_class.new(['the-path'], migration) }
let(:namespace) { create(:group, name: 'the-path') }
before do before do
allow(migration).to receive(:say) allow(migration).to receive(:say)
TestEnv.clean_test_path
end end
def migration_namespace(namespace) def migration_namespace(namespace)
...@@ -137,8 +139,6 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : ...@@ -137,8 +139,6 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
end end
describe "#rename_namespace" do describe "#rename_namespace" do
let(:namespace) { create(:group, name: 'the-path') }
it 'renames paths & routes for the namespace' do it 'renames paths & routes for the namespace' do
expect(subject).to receive(:rename_path_for_routable) expect(subject).to receive(:rename_path_for_routable)
.with(namespace) .with(namespace)
...@@ -149,11 +149,27 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : ...@@ -149,11 +149,27 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
expect(namespace.reload.path).to eq('the-path0') expect(namespace.reload.path).to eq('the-path0')
end end
it 'tracks the rename' do
expect(subject).to receive(:track_rename)
.with('namespace', 'the-path', 'the-path0')
subject.rename_namespace(namespace)
end
it 'renames things related to the namespace' do
expect(subject).to receive(:rename_namespace_dependencies)
.with(namespace, 'the-path', 'the-path0')
subject.rename_namespace(namespace)
end
end
describe '#rename_namespace_dependencies' do
it "moves the the repository for a project in the namespace" do it "moves the the repository for a project in the namespace" do
create(:project, namespace: namespace, path: "the-path-project") create(:project, namespace: namespace, path: "the-path-project")
expected_repo = File.join(TestEnv.repos_path, "the-path0", "the-path-project.git") expected_repo = File.join(TestEnv.repos_path, "the-path0", "the-path-project.git")
subject.rename_namespace(namespace) subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
expect(File.directory?(expected_repo)).to be(true) expect(File.directory?(expected_repo)).to be(true)
end end
...@@ -161,13 +177,13 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : ...@@ -161,13 +177,13 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
it "moves the uploads for the namespace" do it "moves the uploads for the namespace" do
expect(subject).to receive(:move_uploads).with("the-path", "the-path0") expect(subject).to receive(:move_uploads).with("the-path", "the-path0")
subject.rename_namespace(namespace) subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
end end
it "moves the pages for the namespace" do it "moves the pages for the namespace" do
expect(subject).to receive(:move_pages).with("the-path", "the-path0") expect(subject).to receive(:move_pages).with("the-path", "the-path0")
subject.rename_namespace(namespace) subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
end end
it 'invalidates the markdown cache of related projects' do it 'invalidates the markdown cache of related projects' do
...@@ -175,13 +191,13 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : ...@@ -175,13 +191,13 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
expect(subject).to receive(:remove_cached_html_for_projects).with([project.id]) expect(subject).to receive(:remove_cached_html_for_projects).with([project.id])
subject.rename_namespace(namespace) subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
end end
it "doesn't rename users for other namespaces" do it "doesn't rename users for other namespaces" do
expect(subject).not_to receive(:rename_user) expect(subject).not_to receive(:rename_user)
subject.rename_namespace(namespace) subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
end end
it 'renames the username of a namespace for a user' do it 'renames the username of a namespace for a user' do
...@@ -189,7 +205,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : ...@@ -189,7 +205,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
expect(subject).to receive(:rename_user).with('the-path', 'the-path0') expect(subject).to receive(:rename_user).with('the-path', 'the-path0')
subject.rename_namespace(user.namespace) subject.rename_namespace_dependencies(user.namespace, 'the-path', 'the-path0')
end end
end end
...@@ -224,4 +240,50 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, : ...@@ -224,4 +240,50 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
subject.rename_namespaces(type: :child) subject.rename_namespaces(type: :child)
end end
end end
describe '#revert_renames', redis: true do
it 'renames the routes back to the previous values' do
project = create(:project, path: 'a-project', namespace: namespace)
subject.rename_namespace(namespace)
expect(subject).to receive(:perform_rename)
.with(
kind_of(Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::Namespace),
'the-path0',
'the-path'
).and_call_original
subject.revert_renames
expect(namespace.reload.path).to eq('the-path')
expect(namespace.reload.route.path).to eq('the-path')
expect(project.reload.route.path).to eq('the-path/a-project')
end
it 'moves the repositories back to their original place' do
project = create(:project, path: 'a-project', namespace: namespace)
project.create_repository
subject.rename_namespace(namespace)
expected_path = File.join(TestEnv.repos_path, 'the-path', 'a-project.git')
expect(subject).to receive(:rename_namespace_dependencies)
.with(
kind_of(Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::Namespace),