Improve the GitHub and Gitea import feature table interface

These are backend changes.
Use Vue for the import feature UI for "githubish"
providers (GitHub and Gitea).
Add "Go to project" button after a successful import.
Use CI-style status icons and improve spacing of the
table and its component.
Adds ETag polling to the github and gitea import
jobs endpoint.
parent bc881b84
# frozen_string_literal: true # frozen_string_literal: true
class Import::GiteaController < Import::GithubController class Import::GiteaController < Import::GithubController
extend ::Gitlab::Utils::Override
def new def new
if session[access_token_key].present? && session[host_key].present? if session[access_token_key].present? && provider_url.present?
redirect_to status_import_url redirect_to status_import_url
end end
end end
...@@ -12,8 +14,8 @@ class Import::GiteaController < Import::GithubController ...@@ -12,8 +14,8 @@ class Import::GiteaController < Import::GithubController
super super
end end
# Must be defined or it will 404
def status def status
@gitea_host_url = session[host_key]
super super
end end
...@@ -23,25 +25,33 @@ class Import::GiteaController < Import::GithubController ...@@ -23,25 +25,33 @@ class Import::GiteaController < Import::GithubController
:"#{provider}_host_url" :"#{provider}_host_url"
end end
# Overridden methods override :provider
def provider def provider
:gitea :gitea
end end
override :provider_url
def provider_url
session[host_key]
end
# Gitea is not yet an OAuth provider # Gitea is not yet an OAuth provider
# See https://github.com/go-gitea/gitea/issues/27 # See https://github.com/go-gitea/gitea/issues/27
override :logged_in_with_provider?
def logged_in_with_provider? def logged_in_with_provider?
false false
end end
override :provider_auth
def provider_auth def provider_auth
if session[access_token_key].blank? || session[host_key].blank? if session[access_token_key].blank? || provider_url.blank?
redirect_to new_import_gitea_url, redirect_to new_import_gitea_url,
alert: 'You need to specify both an Access Token and a Host URL.' alert: 'You need to specify both an Access Token and a Host URL.'
end end
end end
override :client_options
def client_options def client_options
{ host: session[host_key], api_version: 'v1' } { host: provider_url, api_version: 'v1' }
end end
end end
# frozen_string_literal: true # frozen_string_literal: true
class Import::GithubController < Import::BaseController class Import::GithubController < Import::BaseController
include ImportHelper
before_action :verify_import_enabled before_action :verify_import_enabled
before_action :provider_auth, only: [:status, :jobs, :create] before_action :provider_auth, only: [:status, :realtime_changes, :create]
before_action :expire_etag_cache, only: [:status, :create]
rescue_from Octokit::Unauthorized, with: :provider_unauthorized rescue_from Octokit::Unauthorized, with: :provider_unauthorized
...@@ -24,30 +27,37 @@ class Import::GithubController < Import::BaseController ...@@ -24,30 +27,37 @@ class Import::GithubController < Import::BaseController
redirect_to status_import_url redirect_to status_import_url
end end
# rubocop: disable CodeReuse/ActiveRecord
def status def status
@repos = client.repos # Request repos to display error page if provider token is invalid
@already_added_projects = find_already_added_projects(provider) # Improving in https://gitlab.com/gitlab-org/gitlab-ce/issues/55585
already_added_projects_names = @already_added_projects.pluck(:import_source) client_repos
@repos.reject! { |repo| already_added_projects_names.include? repo.full_name } respond_to do |format|
end format.json do
# rubocop: enable CodeReuse/ActiveRecord render json: { imported_projects: serialized_imported_projects,
provider_repos: serialized_provider_repos,
def jobs namespaces: serialized_namespaces }
render json: find_jobs(provider) end
format.html
end
end end
def create def create
result = Import::GithubService.new(client, current_user, import_params).execute(access_params, provider) result = Import::GithubService.new(client, current_user, import_params).execute(access_params, provider)
if result[:status] == :success if result[:status] == :success
render json: ProjectSerializer.new.represent(result[:project]) render json: serialized_imported_projects(result[:project])
else else
render json: { errors: result[:message] }, status: result[:http_status] render json: { errors: result[:message] }, status: result[:http_status]
end end
end end
def realtime_changes
Gitlab::PollingInterval.set_header(response, interval: 3_000)
render json: find_jobs(provider)
end
private private
def import_params def import_params
...@@ -58,10 +68,45 @@ class Import::GithubController < Import::BaseController ...@@ -58,10 +68,45 @@ class Import::GithubController < Import::BaseController
[:repo_id, :new_name, :target_namespace] [:repo_id, :new_name, :target_namespace]
end end
def serialized_imported_projects(projects = already_added_projects)
ProjectSerializer.new.represent(projects, serializer: :import, provider_url: provider_url)
end
def serialized_provider_repos
repos = client_repos.reject { |repo| already_added_project_names.include? repo.full_name }
ProviderRepoSerializer.new(current_user: current_user).represent(repos, provider: provider, provider_url: provider_url)
end
def serialized_namespaces
NamespaceSerializer.new.represent(namespaces)
end
def already_added_projects
@already_added_projects ||= find_already_added_projects(provider)
end
def already_added_project_names
@already_added_projects_names ||= already_added_projects.pluck(:import_source) # rubocop:disable CodeReuse/ActiveRecord
end
def namespaces
current_user.manageable_groups_with_routes
end
def expire_etag_cache
Gitlab::EtagCaching::Store.new.tap do |store|
store.touch(realtime_changes_path)
end
end
def client def client
@client ||= Gitlab::LegacyGithubImport::Client.new(session[access_token_key], client_options) @client ||= Gitlab::LegacyGithubImport::Client.new(session[access_token_key], client_options)
end end
def client_repos
@client_repos ||= client.repos
end
def verify_import_enabled def verify_import_enabled
render_404 unless import_enabled? render_404 unless import_enabled?
end end
...@@ -74,6 +119,10 @@ class Import::GithubController < Import::BaseController ...@@ -74,6 +119,10 @@ class Import::GithubController < Import::BaseController
__send__("#{provider}_import_enabled?") # rubocop:disable GitlabSecurity/PublicSend __send__("#{provider}_import_enabled?") # rubocop:disable GitlabSecurity/PublicSend
end end
def realtime_changes_path
public_send("realtime_changes_import_#{provider}_path", format: :json) # rubocop:disable GitlabSecurity/PublicSend
end
def new_import_url def new_import_url
public_send("new_import_#{provider}_url", extra_import_params) # rubocop:disable GitlabSecurity/PublicSend public_send("new_import_#{provider}_url", extra_import_params) # rubocop:disable GitlabSecurity/PublicSend
end end
...@@ -105,6 +154,14 @@ class Import::GithubController < Import::BaseController ...@@ -105,6 +154,14 @@ class Import::GithubController < Import::BaseController
:github :github
end end
def provider_url
strong_memoize(:provider_url) do
provider = Gitlab::Auth::OAuth::Provider.config_for('github')
provider&.dig('url').presence || 'https://github.com'
end
end
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def logged_in_with_provider? def logged_in_with_provider?
current_user.identities.exists?(provider: provider) current_user.identities.exists?(provider: provider)
......
...@@ -18,10 +18,8 @@ module ImportHelper ...@@ -18,10 +18,8 @@ module ImportHelper
"#{namespace}/#{name}" "#{namespace}/#{name}"
end end
def provider_project_link(provider, full_path) def provider_project_link_url(provider_url, full_path)
url = __send__("#{provider}_project_url", full_path) # rubocop:disable GitlabSecurity/PublicSend Gitlab::Utils.append_path(provider_url, full_path)
link_to full_path, url, target: '_blank', rel: 'noopener noreferrer'
end end
def import_will_timeout_message(_ci_cd_only) def import_will_timeout_message(_ci_cd_only)
...@@ -81,22 +79,4 @@ module ImportHelper ...@@ -81,22 +79,4 @@ module ImportHelper
def import_all_githubish_repositories_button_label def import_all_githubish_repositories_button_label
_('Import all repositories') _('Import all repositories')
end end
private
def github_project_url(full_path)
Gitlab::Utils.append_path(github_root_url, full_path)
end
def github_root_url
strong_memoize(:github_url) do
provider = Gitlab::Auth::OAuth::Provider.config_for('github')
provider&.dig('url').presence || 'https://github.com'
end
end
def gitea_project_url(full_path)
Gitlab::Utils.append_path(@gitea_host_url, full_path)
end
end end
...@@ -5,11 +5,8 @@ module NamespacesHelper ...@@ -5,11 +5,8 @@ module NamespacesHelper
params.dig(:project, :namespace_id) || params[:namespace_id] params.dig(:project, :namespace_id) || params[:namespace_id]
end end
# rubocop: disable CodeReuse/ActiveRecord
def namespaces_options(selected = :current_user, display_path: false, groups: nil, extra_group: nil, groups_only: false) def namespaces_options(selected = :current_user, display_path: false, groups: nil, extra_group: nil, groups_only: false)
groups ||= current_user.manageable_groups groups ||= current_user.manageable_groups_with_routes
.eager_load(:route)
.order('routes.path')
users = [current_user.namespace] users = [current_user.namespace]
selected_id = selected selected_id = selected
...@@ -43,7 +40,6 @@ module NamespacesHelper ...@@ -43,7 +40,6 @@ module NamespacesHelper
grouped_options_for_select(options, selected_id) grouped_options_for_select(options, selected_id)
end end
# rubocop: enable CodeReuse/ActiveRecord
def namespace_icon(namespace, size = 40) def namespace_icon(namespace, size = 40)
if namespace.is_a?(Group) if namespace.is_a?(Group)
......
...@@ -1167,6 +1167,10 @@ class User < ApplicationRecord ...@@ -1167,6 +1167,10 @@ class User < ApplicationRecord
Gitlab::ObjectHierarchy.new(owned_or_maintainers_groups).base_and_descendants Gitlab::ObjectHierarchy.new(owned_or_maintainers_groups).base_and_descendants
end end
def manageable_groups_with_routes
manageable_groups.eager_load(:route).order('routes.path')
end
def namespaces def namespaces
namespace_ids = groups.pluck(:id) namespace_ids = groups.pluck(:id)
namespace_ids.push(namespace.id) namespace_ids.push(namespace.id)
......
# frozen_string_literal: true
class NamespaceBasicEntity < Grape::Entity
expose :id
expose :full_path
end
# frozen_string_literal: true
class NamespaceSerializer < BaseSerializer
entity NamespaceBasicEntity
end
# frozen_string_literal: true
class ProjectImportEntity < ProjectEntity
include ImportHelper
expose :import_source
expose :import_status
expose :human_import_status_name
expose :provider_link do |project, options|
provider_project_link_url(options[:provider_url], project[:import_source])
end
end
# frozen_string_literal: true # frozen_string_literal: true
class ProjectSerializer < BaseSerializer class ProjectSerializer < BaseSerializer
entity ProjectEntity def represent(project, opts = {})
entity =
case opts[:serializer]
when :import
ProjectImportEntity
else
ProjectEntity
end
super(project, opts, entity)
end
end end
# frozen_string_literal: true
class ProviderRepoEntity < Grape::Entity
include ImportHelper
expose :id
expose :full_name
expose :owner_name do |provider_repo, options|
owner_name(provider_repo, options[:provider])
end
expose :sanitized_name do |provider_repo|
sanitize_project_name(provider_repo[:name])
end
expose :provider_link do |provider_repo, options|
provider_project_link_url(options[:provider_url], provider_repo[:full_name])
end
private
def owner_name(provider_repo, provider)
provider_repo.dig(:owner, :login) if provider == :github
end
end
# frozen_string_literal: true
class ProviderRepoSerializer < BaseSerializer
entity ProviderRepoEntity
end
---
title: Improve GitHub and Gitea project import table UI
merge_request: 24606
author:
type: other
...@@ -12,13 +12,13 @@ namespace :import do ...@@ -12,13 +12,13 @@ namespace :import do
post :personal_access_token post :personal_access_token
get :status get :status
get :callback get :callback
get :jobs get :realtime_changes
end end
resource :gitea, only: [:create, :new], controller: :gitea do resource :gitea, only: [:create, :new], controller: :gitea do
post :personal_access_token post :personal_access_token
get :status get :status
get :jobs get :realtime_changes
end end
resource :gitlab, only: [:create], controller: :gitlab do resource :gitlab, only: [:create], controller: :gitlab do
......
...@@ -52,6 +52,14 @@ module Gitlab ...@@ -52,6 +52,14 @@ module Gitlab
Gitlab::EtagCaching::Router::Route.new( Gitlab::EtagCaching::Router::Route.new(
%r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z), %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z),
'environments' 'environments'
),
Gitlab::EtagCaching::Router::Route.new(
%r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/import/github/realtime_changes\.json\z),
'realtime_changes_import_github'
),
Gitlab::EtagCaching::Router::Route.new(
%r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/import/gitea/realtime_changes\.json\z),
'realtime_changes_import_gitea'
) )
].freeze ].freeze
......
...@@ -40,4 +40,12 @@ describe Import::GiteaController do ...@@ -40,4 +40,12 @@ describe Import::GiteaController do
end end
end end
end end
describe "GET realtime_changes" do
it_behaves_like 'a GitHub-ish import controller: GET realtime_changes' do
before do
assign_host_url
end
end
end
end end
...@@ -60,4 +60,8 @@ describe Import::GithubController do ...@@ -60,4 +60,8 @@ describe Import::GithubController do
describe "POST create" do describe "POST create" do
it_behaves_like 'a GitHub-ish import controller: POST create' it_behaves_like 'a GitHub-ish import controller: POST create'
end end
describe "GET realtime_changes" do
it_behaves_like 'a GitHub-ish import controller: GET realtime_changes'
end
end end
...@@ -39,59 +39,12 @@ describe ImportHelper do ...@@ -39,59 +39,12 @@ describe ImportHelper do
end end
end end
describe '#provider_project_link' do describe '#provider_project_link_url' do
context 'when provider is "github"' do let(:full_path) { '/repo/path' }
let(:github_server_url) { nil } let(:host_url) { 'http://provider.com/' }
let(:provider) { OpenStruct.new(name: 'github', url: github_server_url) }
before do it 'appends repo full path to provider host url' do
stub_omniauth_setting(providers: [provider]) expect(helper.provider_project_link_url(host_url, full_path)).to match('http://provider.com/repo/path')
end
context 'when provider does not specify a custom URL' do
it 'uses default GitHub URL' do
expect(helper.provider_project_link('github', 'octocat/Hello-World'))
.to include('href="https://github.com/octocat/Hello-World"')
end
end
context 'when provider specify a custom URL' do
let(:github_server_url) { 'https://github.company.com' }
it 'uses custom URL' do
expect(helper.provider_project_link('github', 'octocat/Hello-World'))
.to include('href="https://github.company.com/octocat/Hello-World"')
end
end
context "when custom URL contains a '/' char at the end" do
let(:github_server_url) { 'https://github.company.com/' }
it "doesn't render double slash" do
expect(helper.provider_project_link('github', 'octocat/Hello-World'))
.to include('href="https://github.company.com/octocat/Hello-World"')
end
end
context 'when provider is missing' do
it 'uses the default URL' do
allow(Gitlab.config.omniauth).to receive(:providers).and_return([])
expect(helper.provider_project_link('github', 'octocat/Hello-World'))
.to include('href="https://github.com/octocat/Hello-World"')
end
end
end
context 'when provider is "gitea"' do
before do
assign(:gitea_host_url, 'https://try.gitea.io/')
end
it 'uses given host' do
expect(helper.provider_project_link('gitea', 'octocat/Hello-World'))
.to include('href="https://try.gitea.io/octocat/Hello-World"')
end
end end
end end
end end
...@@ -925,6 +925,21 @@ describe User do ...@@ -925,6 +925,21 @@ describe User do
expect(user.manageable_groups).to contain_exactly(group, subgroup) expect(user.manageable_groups).to contain_exactly(group, subgroup)
end end
end end
describe '#manageable_groups_with_routes' do
it 'eager loads routes from manageable groups' do
control_count =
ActiveRecord::QueryRecorder.new(skip_cached: false) do
user.manageable_groups_with_routes.map(&:route)
end.count
create(:group, parent: subgroup)
expect do
user.manageable_groups_with_routes.map(&:route)
end.not_to exceed_all_query_limit(control_count)
end
end
end end
end end
......
...@@ -23,6 +23,11 @@ require 'spec_helper' ...@@ -23,6 +23,11 @@ require 'spec_helper'
# end # end
shared_examples 'importer routing' do shared_examples 'importer routing' do
let(:except_actions) { [] } let(:except_actions) { [] }
let(:is_realtime) { false }
before do
except_actions.push(is_realtime ? :jobs : :realtime_changes)
end
it 'to #create' do it 'to #create' do
expect(post("/import/#{provider}")).to route_to("import/#{provider}#create") unless except_actions.include?(:create) expect(post("/import/#{provider}")).to route_to("import/#{provider}#create") unless except_actions.include?(:create)
...@@ -43,17 +48,22 @@ shared_examples 'importer routing' do ...@@ -43,17 +48,22 @@ shared_examples 'importer routing' do
it 'to #jobs' do it 'to #jobs' do
expect(get("/import/#{provider}/jobs")).to route_to("import/#{provider}#jobs") unless except_actions.include?(:jobs) expect(get("/import/#{provider}/jobs")).to route_to("import/#{provider}#jobs") unless except_actions.include?(:jobs)
end end
it 'to #realtime_changes' do
expect(get("/import/#{provider}/realtime_changes")).to route_to("import/#{provider}#realtime_changes") unless except_actions.include?(:realtime_changes)
end
end end
# personal_access_token_import_github POST /import/github/personal_access_token(.:format) import/github#personal_access_token # personal_access_token_import_github POST /import/github/personal_access_token(.:format) import/github#personal_access_token
# status_import_github GET /import/github/status(.:format) import/github#status # status_import_github GET /import/github/status(.:format) import/github#status
# callback_import_github GET /import/github/callback(.:format) import/github#callback # callback_import_github GET /import/github/callback(.:format) import/github#callback
# jobs_import_github GET /import/github/jobs(.:format) import/github#jobs # realtime_changes_import_github GET /import/github/realtime_changes(.:format) import/github#jobs
# import_github POST /import/github(.:format) import/github#create # import_github POST /import/github(.:format) import/github#create
# new_import_github GET /import/github/new(.:format) import/github#new # new_import_github GET /import/github/new(.:format) import/github#new
describe Import::GithubController, 'routing' do describe Import::GithubController, 'routing' do
it_behaves_like 'importer routing' do it_behaves_like 'importer routing' do
let(:provider) { 'github' } let(:provider) { 'github' }
let(:is_realtime) { true }
end end
it 'to #personal_access_token' do it 'to #personal_access_token' do
...@@ -63,13 +73,14 @@ end ...@@ -63,13 +73,14 @@ end
# personal_access_token_import_gitea POST /import/gitea/personal_access_token(.:format) import/gitea#personal_access_token # personal_access_token_import_gitea POST /import/gitea/personal_access_token(.:format) import/gitea#personal_access_token
# status_import_gitea GET /import/gitea/status(.:format) import/gitea#status # status_import_gitea GET /import/gitea/status(.:format) import/gitea#status
# jobs_import_gitea GET /import/gitea/jobs(.:format) import/gitea#jobs # realtime_changes_import_gitea GET /import/gitea/realtime_changes(.:format) import/gitea#jobs
# import_gitea POST /import/gitea(.:format) import/gitea#create # import_gitea POST /import/gitea(.:format) import/gitea#create
# new_import_gitea GET /import/gitea/new(.:format) import/gitea#new # new_import_gitea GET /import/gitea/new(.:format) import/gitea#new
describe Import::GiteaController, 'routing' do describe Import::GiteaController, 'routing' do
it_behaves_like 'importer routing' do it_behaves_like 'importer routing' do
let(:except_actions) { [:callback] } let(:except_actions) { [:callback] }
let(:provider) { 'gitea' } let(:provider) { 'gitea' }
let(:is_realtime) { true }
end end
it 'to #personal_access_token' do it 'to #personal_access_token' do
......
# frozen_string_literal: true
require 'spec_helper'
describe NamespaceBasicEntity do
set(:group) { create(:group) }
let(:entity) do
described_class.represent(group)
end
describe '#as_json' do
subject { entity.as_json }
it 'includes required fields' do
expect(subject).to include :id, :full_path
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe NamespaceSerializer do
it 'represents NamespaceBasicEntity entities' do
expect(described_class.entity_class).to eq(NamespaceBasicEntity)
end
end
# frozen_string_literal: true
require 'spec_helper'
describe ProjectImportEntity do
include ImportHelper
set(:project) { create(:project, import_status: :started, import_source: 'namespace/project') }
let(:provider_url) { 'https://provider.com' }
let(:entity) { described_class.represent(project, provider_url: provider_url) }
describe '#as_json' do
subject { entity.as_json }
it 'includes required fields' do