Commit 971604de authored by Sean McGivern's avatar Sean McGivern

Merge branch 'bvl-rename-all-reserved-paths' into 'master'

Rename all forbidden paths again

Closes #32625

See merge request !11713
parents da6a3fb0 7c53fcf1
---
title: Rename all reserved paths that could have been created
merge_request: 11713
author:
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class RenameAllReservedPathsAgain < ActiveRecord::Migration
include Gitlab::Database::RenameReservedPathsMigration::V1
DOWNTIME = false
disable_ddl_transaction!
TOP_LEVEL_ROUTES = %w[
-
.well-known
abuse_reports
admin
all
api
assets
autocomplete
ci
dashboard
explore
files
groups
health_check
help
hooks
import
invites
issues
jwt
koding
member
merge_requests
new
notes
notification_settings
oauth
profile
projects
public
repository
robots.txt
s
search
sent_notifications
services
snippets
teams
u
unicorn_test
unsubscribes
uploads
users
].freeze
PROJECT_WILDCARD_ROUTES = %w[
badges
blame
blob
builds
commits
create
create_dir
edit
environments/folders
files
find_file
gitlab-lfs/objects
info/lfs/objects
new
preview
raw
refs
tree
update
wikis
].freeze
GROUP_ROUTES = %w[
activity
analytics
audit_events
avatar
edit
group_members
hooks
issues
labels
ldap
ldap_group_links
merge_requests
milestones
notification_setting
pipeline_quota
projects
subgroups
].freeze
def up
disable_statement_timeout
TOP_LEVEL_ROUTES.each { |route| rename_root_paths(route) }
PROJECT_WILDCARD_ROUTES.each { |route| rename_wildcard_paths(route) }
GROUP_ROUTES.each { |route| rename_child_paths(route) }
end
def down
disable_statement_timeout
revert_renames
end
end
......@@ -29,6 +29,11 @@ module Gitlab
paths = Array(paths)
RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level)
end
def revert_renames
RenameProjects.new([], self).revert_renames
RenameNamespaces.new([], self).revert_renames
end
end
end
end
......
......@@ -6,7 +6,10 @@ module Gitlab
attr_reader :paths, :migration
delegate :update_column_in_batches,
:execute,
:replace_sql,
:quote_string,
:say,
to: :migration
def initialize(paths, migration)
......@@ -26,24 +29,45 @@ module Gitlab
new_path = rename_path(namespace_path, old_path)
new_full_path = join_routable_path(namespace_path, new_path)
perform_rename(routable, old_full_path, new_full_path)
[old_full_path, new_full_path]
end
def perform_rename(routable, old_full_path, new_full_path)
# skips callbacks & validations
new_path = new_full_path.split('/').last
routable.class.where(id: routable)
.update_all(path: new_path)
rename_routes(old_full_path, new_full_path)
[old_full_path, new_full_path]
end
def rename_routes(old_full_path, new_full_path)
routes = Route.arel_table
quoted_old_full_path = quote_string(old_full_path)
quoted_old_wildcard_path = quote_string("#{old_full_path}/%")
filter = if Database.mysql?
"lower(routes.path) = lower('#{quoted_old_full_path}') "\
"OR routes.path LIKE '#{quoted_old_wildcard_path}'"
else
"routes.id IN "\
"( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\
"UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )"
end
replace_statement = replace_sql(Route.arel_table[:path],
old_full_path,
new_full_path)
update_column_in_batches(:routes, :path, replace_statement) do |table, query|
path_or_children = table[:path].matches_any([old_full_path, "#{old_full_path}/%"])
query.where(path_or_children)
end
update = Arel::UpdateManager.new(ActiveRecord::Base)
.table(routes)
.set([[routes[:path], replace_statement]])
.where(Arel::Nodes::SqlLiteral.new(filter))
execute(update.to_sql)
end
def rename_path(namespace_path, path_was)
......@@ -86,32 +110,74 @@ module Gitlab
def move_folders(directory, old_relative_path, new_relative_path)
old_path = File.join(directory, old_relative_path)
return unless File.directory?(old_path)
unless File.directory?(old_path)
say "#{old_path} doesn't exist, skipping"
return
end
new_path = File.join(directory, new_relative_path)
FileUtils.mv(old_path, new_path)
end
def remove_cached_html_for_projects(project_ids)
update_column_in_batches(:projects, :description_html, nil) do |table, query|
query.where(table[:id].in(project_ids))
end
update_column_in_batches(:issues, :description_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
project_ids.each do |project_id|
update_column_in_batches(:projects, :description_html, nil) do |table, query|
query.where(table[:id].eq(project_id))
end
update_column_in_batches(:issues, :description_html, nil) do |table, query|
query.where(table[:project_id].eq(project_id))
end
update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
query.where(table[:target_project_id].eq(project_id))
end
update_column_in_batches(:notes, :note_html, nil) do |table, query|
query.where(table[:project_id].eq(project_id))
end
update_column_in_batches(:milestones, :description_html, nil) do |table, query|
query.where(table[:project_id].eq(project_id))
end
end
end
update_column_in_batches(:merge_requests, :description_html, nil) do |table, query|
query.where(table[:target_project_id].in(project_ids))
def track_rename(type, old_path, new_path)
key = redis_key_for_type(type)
Gitlab::Redis.with do |redis|
redis.lpush(key, [old_path, new_path].to_json)
redis.expire(key, 2.weeks.to_i)
end
say "tracked rename: #{key}: #{old_path} -> #{new_path}"
end
update_column_in_batches(:notes, :note_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
def reverts_for_type(type)
key = redis_key_for_type(type)
Gitlab::Redis.with do |redis|
failed_reverts = []
while rename_info = redis.lpop(key)
path_before_rename, path_after_rename = JSON.parse(rename_info)
say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}"
begin
yield(path_before_rename, path_after_rename)
rescue StandardError => e
failed_reverts << rename_info
say "Renaming #{type} from #{path_after_rename} back to "\
"#{path_before_rename} failed. Review the error and try "\
"again by running the `down` action. \n"\
"#{e.message}: \n #{e.backtrace.join("\n")}"
end
end
failed_reverts.each { |rename_info| redis.lpush(key, rename_info) }
end
end
update_column_in_batches(:milestones, :description_html, nil) do |table, query|
query.where(table[:project_id].in(project_ids))
end
def redis_key_for_type(type)
"rename:#{migration.name}:#{type}"
end
def file_storage?
......
......@@ -26,6 +26,12 @@ module Gitlab
def rename_namespace(namespace)
old_full_path, new_full_path = rename_path_for_routable(namespace)
track_rename('namespace', old_full_path, new_full_path)
rename_namespace_dependencies(namespace, old_full_path, new_full_path)
end
def rename_namespace_dependencies(namespace, old_full_path, new_full_path)
move_repositories(namespace, old_full_path, new_full_path)
move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path)
......@@ -33,6 +39,23 @@ module Gitlab
remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id))
end
def revert_renames
reverts_for_type('namespace') do |path_before_rename, current_path|
matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
namespace = MigrationClasses::Namespace.joins(:route)
.where(matches_path).first&.becomes(MigrationClasses::Namespace)
if namespace
perform_rename(namespace, current_path, path_before_rename)
rename_namespace_dependencies(namespace, current_path, path_before_rename)
else
say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\
"namespace was renamed, or no longer exists at the expected path"
end
end
end
def rename_user(old_username, new_username)
MigrationClasses::User.where(username: old_username)
.update_all(username: new_username)
......
......@@ -16,12 +16,37 @@ module Gitlab
def rename_project(project)
old_full_path, new_full_path = rename_path_for_routable(project)
track_rename('project', old_full_path, new_full_path)
move_project_folders(project, old_full_path, new_full_path)
end
def move_project_folders(project, old_full_path, new_full_path)
move_repository(project, old_full_path, new_full_path)
move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki")
move_uploads(old_full_path, new_full_path)
move_pages(old_full_path, new_full_path)
end
def revert_renames
reverts_for_type('project') do |path_before_rename, current_path|
matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
project = MigrationClasses::Project.joins(:route)
.where(matches_path).first
if project
perform_rename(project, current_path, path_before_rename)
move_project_folders(project, current_path, path_before_rename)
else
say "Couldn't rename project from #{current_path} back to "\
"#{path_before_rename}, project was renamed or no longer "\
"exists at the expected path."
end
end
end
def move_repository(project, old_path, new_path)
unless gitlab_shell.mv_repository(project.repository_storage_path,
old_path,
......
......@@ -6,6 +6,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca
before do
allow(migration).to receive(:say)
TestEnv.clean_test_path
end
def migration_namespace(namespace)
......@@ -153,6 +154,30 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca
end
end
describe '#perform_rename' do
describe 'for namespaces' do
let(:namespace) { create(:namespace, path: 'the-path') }
it 'renames the path' do
subject.perform_rename(migration_namespace(namespace), 'the-path', 'renamed')
expect(namespace.reload.path).to eq('renamed')
end
it 'renames all the routes for the namespace' do
child = create(:group, path: 'child', parent: namespace)
project = create(:project, namespace: child, path: 'the-project')
other_one = create(:namespace, path: 'the-path-is-similar')
subject.perform_rename(migration_namespace(namespace), 'the-path', 'renamed')
expect(namespace.reload.route.path).to eq('renamed')
expect(child.reload.route.path).to eq('renamed/child')
expect(project.reload.route.path).to eq('renamed/child/the-project')
expect(other_one.reload.route.path).to eq('the-path-is-similar')
end
end
end
describe '#move_pages' do
it 'moves the pages directory' do
expect(subject).to receive(:move_folders)
......@@ -203,4 +228,53 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca
expect(File.exist?(expected_file)).to be(true)
end
end
describe '#track_rename', redis: true do
it 'tracks a rename in redis' do
key = 'rename:FakeRenameReservedPathMigrationV1:namespace'
subject.track_rename('namespace', 'path/to/namespace', 'path/to/renamed')
old_path, new_path = [nil, nil]
Gitlab::Redis.with do |redis|
rename_info = redis.lpop(key)
old_path, new_path = JSON.parse(rename_info)
end
expect(old_path).to eq('path/to/namespace')
expect(new_path).to eq('path/to/renamed')
end
end
describe '#reverts_for_type', redis: true do
it 'yields for each tracked rename' do
subject.track_rename('project', 'old_path', 'new_path')
subject.track_rename('project', 'old_path2', 'new_path2')
subject.track_rename('namespace', 'namespace_path', 'new_namespace_path')
expect { |b| subject.reverts_for_type('project', &b) }
.to yield_successive_args(%w(old_path2 new_path2), %w(old_path new_path))
expect { |b| subject.reverts_for_type('namespace', &b) }
.to yield_with_args('namespace_path', 'new_namespace_path')
end
it 'keeps the revert in redis if it failed' do
subject.track_rename('project', 'old_path', 'new_path')
subject.reverts_for_type('project') do
raise 'whatever happens, keep going!'
end
key = 'rename:FakeRenameReservedPathMigrationV1:project'
stored_renames = nil
rename_count = 0
Gitlab::Redis.with do |redis|
stored_renames = redis.lrange(key, 0, 1)
rename_count = redis.llen(key)
end
expect(rename_count).to eq(1)
expect(JSON.parse(stored_renames.first)).to eq(%w(old_path new_path))
end
end
end
......@@ -3,9 +3,11 @@ require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :truncate do
let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) }
let(:namespace) { create(:group, name: 'the-path') }
before do
allow(migration).to receive(:say)
TestEnv.clean_test_path
end
def migration_namespace(namespace)
......@@ -137,8 +139,6 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
end
describe "#rename_namespace" do
let(:namespace) { create(:group, name: 'the-path') }
it 'renames paths & routes for the namespace' do
expect(subject).to receive(:rename_path_for_routable)
.with(namespace)
......@@ -149,11 +149,27 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
expect(namespace.reload.path).to eq('the-path0')
end
it 'tracks the rename' do
expect(subject).to receive(:track_rename)
.with('namespace', 'the-path', 'the-path0')
subject.rename_namespace(namespace)
end
it 'renames things related to the namespace' do
expect(subject).to receive(:rename_namespace_dependencies)
.with(namespace, 'the-path', 'the-path0')
subject.rename_namespace(namespace)
end
end
describe '#rename_namespace_dependencies' do
it "moves the the repository for a project in the namespace" do
create(:project, namespace: namespace, path: "the-path-project")
expected_repo = File.join(TestEnv.repos_path, "the-path0", "the-path-project.git")
subject.rename_namespace(namespace)
subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
expect(File.directory?(expected_repo)).to be(true)
end
......@@ -161,13 +177,13 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
it "moves the uploads for the namespace" do
expect(subject).to receive(:move_uploads).with("the-path", "the-path0")
subject.rename_namespace(namespace)
subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
end
it "moves the pages for the namespace" do
expect(subject).to receive(:move_pages).with("the-path", "the-path0")
subject.rename_namespace(namespace)
subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
end
it 'invalidates the markdown cache of related projects' do
......@@ -175,13 +191,13 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
expect(subject).to receive(:remove_cached_html_for_projects).with([project.id])
subject.rename_namespace(namespace)
subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
end
it "doesn't rename users for other namespaces" do
expect(subject).not_to receive(:rename_user)
subject.rename_namespace(namespace)
subject.rename_namespace_dependencies(namespace, 'the-path', 'the-path0')
end
it 'renames the username of a namespace for a user' do
......@@ -189,7 +205,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
expect(subject).to receive(:rename_user).with('the-path', 'the-path0')
subject.rename_namespace(user.namespace)
subject.rename_namespace_dependencies(user.namespace, 'the-path', 'the-path0')
end
end
......@@ -224,4 +240,50 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :
subject.rename_namespaces(type: :child)
end
end
describe '#revert_renames', redis: true do
it 'renames the routes back to the previous values' do
project = create(:project, path: 'a-project', namespace: namespace)
subject.rename_namespace(namespace)
expect(subject).to receive(:perform_rename)
.with(
kind_of(Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::Namespace),
'the-path0',
'the-path'
).and_call_original
subject.revert_renames
expect(namespace.reload.path).to eq('the-path')
expect(namespace.reload.route.path).to eq('the-path')
expect(project.reload.route.path).to eq('the-path/a-project')
end
it 'moves the repositories back to their original place' do
project = create(:project, path: 'a-project', namespace: namespace)
project.create_repository
subject.rename_namespace(namespace)
expected_path = File.join(TestEnv.repos_path, 'the-path', 'a-project.git')
expect(subject).to receive(:rename_namespace_dependencies)
.with(
kind_of(Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::Namespace),
'the-path0',
'the-path'
).and_call_original
subject.revert_renames
expect(File.directory?(expected_path)).to be_truthy
end
it "doesn't break when the namespace was renamed" do
subject.rename_namespace(namespace)
namespace.update_attributes!(path: 'renamed-afterwards')
expect { subject.revert_renames }.not_to raise_error
end
end
end
......@@ -3,9 +3,15 @@ require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :truncate do
let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) }
let(:project) do
create(:empty_project,
path: 'the-path',
namespace: create(:namespace, path: 'known-parent' ))
end
before do
allow(migration).to receive(:say)
TestEnv.clean_test_path
end
describe '#projects_for_paths' do
......@@ -47,12 +53,6 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :tr
end
describe '#rename_project' do
let(:project) do
create(:empty_project,
path: 'the-path',
namespace: create(:namespace, path: 'known-parent' ))
end
it 'renames path & route for the project' do
expect(subject).to receive(:rename_path_for_routable)
.with(project)
......@@ -63,27 +63,42 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :tr
expect(project.reload.path).to eq('the-path0')
end
it 'tracks the rename' do
expect(subject).to receive(:track_rename)
.with('project', 'known-parent/the-path', 'known-parent/the-path0')
subject.rename_project(project)
end
it 'renames the folders for the project' do
expect(subject).to receive(:move_project_folders).with(project, 'known-parent/the-path', 'known-parent/the-path0')
subject.rename_project(project)
end
end
describe '#move_project_folders' do
it 'moves the wiki & the repo' do
expect(subject).to receive(:move_repository)
.with(project, 'known-parent/the-path.wiki', 'known-parent/the-path0.wiki')
expect(subject).to receive(:move_repository)
.with(project, 'known-parent/the-path', 'known-parent/the-path0')
subject.rename_project(project)
subject.move_project_folders(project, 'known-parent/the-path', 'known-parent/the-path0')
end
it 'moves uploads' do
expect(subject).to receive(:move_uploads)
.with('known-parent/the-path', 'known-parent/the-path0')
subject.rename_project(project)
subject.move_project_folders(project, 'known-parent/the-path', 'known-parent/the-path0')
end
it 'moves pages' do
expect(subject).to receive(:move_pages)
.with('known-parent/the-path', 'known-parent/the-path0')
subject.rename_project(project)
subject.move_project_folders(project, 'known-parent/the-path', 'known-parent/the-path0')
end
end
......@@ -99,4 +114,47 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :tr
expect(File.directory?(expected_path)).to be(true)
end
end
describe '#revert_renames', redis: true do
it 'renames the routes back to the previous values' do
subject.rename_project(project)
expect(subject).to receive(:perform_rename)
.with(
kind_of(Gitlab::Database::RenameReservedPathsMigration::V1::MigrationClasses::Project),
'known-parent/the-path0',
'known-parent/the-path'
).and_call_original
subject.revert_renames
expect(project.reload.