Improve the GitHub and Gitea import feature table interface

These are backend changes.
Use Vue for the import feature UI for "githubish"
providers (GitHub and Gitea).
Add "Go to project" button after a successful import.
Use CI-style status icons and improve spacing of the
table and its component.
Adds ETag polling to the github and gitea import
jobs endpoint.
parent bc881b84
# frozen_string_literal: true
class Import::GiteaController < Import::GithubController
extend ::Gitlab::Utils::Override
def new
if session[access_token_key].present? && session[host_key].present?
if session[access_token_key].present? && provider_url.present?
redirect_to status_import_url
end
end
......@@ -12,8 +14,8 @@ class Import::GiteaController < Import::GithubController
super
end
# Must be defined or it will 404
def status
@gitea_host_url = session[host_key]
super
end
......@@ -23,25 +25,33 @@ class Import::GiteaController < Import::GithubController
:"#{provider}_host_url"
end
# Overridden methods
override :provider
def provider
:gitea
end
override :provider_url
def provider_url
session[host_key]
end
# Gitea is not yet an OAuth provider
# See https://github.com/go-gitea/gitea/issues/27
override :logged_in_with_provider?
def logged_in_with_provider?
false
end
override :provider_auth
def provider_auth
if session[access_token_key].blank? || session[host_key].blank?
if session[access_token_key].blank? || provider_url.blank?
redirect_to new_import_gitea_url,
alert: 'You need to specify both an Access Token and a Host URL.'
end
end
override :client_options
def client_options
{ host: session[host_key], api_version: 'v1' }
{ host: provider_url, api_version: 'v1' }
end
end
# frozen_string_literal: true
class Import::GithubController < Import::BaseController
include ImportHelper
before_action :verify_import_enabled
before_action :provider_auth, only: [:status, :jobs, :create]
before_action :provider_auth, only: [:status, :realtime_changes, :create]
before_action :expire_etag_cache, only: [:status, :create]
rescue_from Octokit::Unauthorized, with: :provider_unauthorized
......@@ -24,30 +27,37 @@ class Import::GithubController < Import::BaseController
redirect_to status_import_url
end
# rubocop: disable CodeReuse/ActiveRecord
def status
@repos = client.repos
@already_added_projects = find_already_added_projects(provider)
already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.reject! { |repo| already_added_projects_names.include? repo.full_name }
end
# rubocop: enable CodeReuse/ActiveRecord
def jobs
render json: find_jobs(provider)
# Request repos to display error page if provider token is invalid
# Improving in https://gitlab.com/gitlab-org/gitlab-ce/issues/55585
client_repos
respond_to do |format|
format.json do
render json: { imported_projects: serialized_imported_projects,
provider_repos: serialized_provider_repos,
namespaces: serialized_namespaces }
end
format.html
end
end
def create
result = Import::GithubService.new(client, current_user, import_params).execute(access_params, provider)
if result[:status] == :success
render json: ProjectSerializer.new.represent(result[:project])
render json: serialized_imported_projects(result[:project])
else
render json: { errors: result[:message] }, status: result[:http_status]
end
end
def realtime_changes
Gitlab::PollingInterval.set_header(response, interval: 3_000)
render json: find_jobs(provider)
end
private
def import_params
......@@ -58,10 +68,45 @@ class Import::GithubController < Import::BaseController
[:repo_id, :new_name, :target_namespace]
end
def serialized_imported_projects(projects = already_added_projects)
ProjectSerializer.new.represent(projects, serializer: :import, provider_url: provider_url)
end
def serialized_provider_repos
repos = client_repos.reject { |repo| already_added_project_names.include? repo.full_name }
ProviderRepoSerializer.new(current_user: current_user).represent(repos, provider: provider, provider_url: provider_url)
end
def serialized_namespaces
NamespaceSerializer.new.represent(namespaces)
end
def already_added_projects
@already_added_projects ||= find_already_added_projects(provider)
end
def already_added_project_names
@already_added_projects_names ||= already_added_projects.pluck(:import_source) # rubocop:disable CodeReuse/ActiveRecord
end
def namespaces
current_user.manageable_groups_with_routes
end
def expire_etag_cache
Gitlab::EtagCaching::Store.new.tap do |store|
store.touch(realtime_changes_path)
end
end
def client
@client ||= Gitlab::LegacyGithubImport::Client.new(session[access_token_key], client_options)
end
def client_repos
@client_repos ||= client.repos
end
def verify_import_enabled
render_404 unless import_enabled?
end
......@@ -74,6 +119,10 @@ class Import::GithubController < Import::BaseController
__send__("#{provider}_import_enabled?") # rubocop:disable GitlabSecurity/PublicSend
end
def realtime_changes_path
public_send("realtime_changes_import_#{provider}_path", format: :json) # rubocop:disable GitlabSecurity/PublicSend
end
def new_import_url
public_send("new_import_#{provider}_url", extra_import_params) # rubocop:disable GitlabSecurity/PublicSend
end
......@@ -105,6 +154,14 @@ class Import::GithubController < Import::BaseController
:github
end
def provider_url
strong_memoize(:provider_url) do
provider = Gitlab::Auth::OAuth::Provider.config_for('github')
provider&.dig('url').presence || 'https://github.com'
end
end
# rubocop: disable CodeReuse/ActiveRecord
def logged_in_with_provider?
current_user.identities.exists?(provider: provider)
......
......@@ -18,10 +18,8 @@ module ImportHelper
"#{namespace}/#{name}"
end
def provider_project_link(provider, full_path)
url = __send__("#{provider}_project_url", full_path) # rubocop:disable GitlabSecurity/PublicSend
link_to full_path, url, target: '_blank', rel: 'noopener noreferrer'
def provider_project_link_url(provider_url, full_path)
Gitlab::Utils.append_path(provider_url, full_path)
end
def import_will_timeout_message(_ci_cd_only)
......@@ -81,22 +79,4 @@ module ImportHelper
def import_all_githubish_repositories_button_label
_('Import all repositories')
end
private
def github_project_url(full_path)
Gitlab::Utils.append_path(github_root_url, full_path)
end
def github_root_url
strong_memoize(:github_url) do
provider = Gitlab::Auth::OAuth::Provider.config_for('github')
provider&.dig('url').presence || 'https://github.com'
end
end
def gitea_project_url(full_path)
Gitlab::Utils.append_path(@gitea_host_url, full_path)
end
end
......@@ -5,11 +5,8 @@ module NamespacesHelper
params.dig(:project, :namespace_id) || params[:namespace_id]
end
# rubocop: disable CodeReuse/ActiveRecord
def namespaces_options(selected = :current_user, display_path: false, groups: nil, extra_group: nil, groups_only: false)
groups ||= current_user.manageable_groups
.eager_load(:route)
.order('routes.path')
groups ||= current_user.manageable_groups_with_routes
users = [current_user.namespace]
selected_id = selected
......@@ -43,7 +40,6 @@ module NamespacesHelper
grouped_options_for_select(options, selected_id)
end
# rubocop: enable CodeReuse/ActiveRecord
def namespace_icon(namespace, size = 40)
if namespace.is_a?(Group)
......
......@@ -1167,6 +1167,10 @@ class User < ApplicationRecord
Gitlab::ObjectHierarchy.new(owned_or_maintainers_groups).base_and_descendants
end
def manageable_groups_with_routes
manageable_groups.eager_load(:route).order('routes.path')
end
def namespaces
namespace_ids = groups.pluck(:id)
namespace_ids.push(namespace.id)
......
# frozen_string_literal: true
class NamespaceBasicEntity < Grape::Entity
expose :id
expose :full_path
end
# frozen_string_literal: true
class NamespaceSerializer < BaseSerializer
entity NamespaceBasicEntity
end
# frozen_string_literal: true
class ProjectImportEntity < ProjectEntity
include ImportHelper
expose :import_source
expose :import_status
expose :human_import_status_name
expose :provider_link do |project, options|
provider_project_link_url(options[:provider_url], project[:import_source])
end
end
# frozen_string_literal: true
class ProjectSerializer < BaseSerializer
entity ProjectEntity
def represent(project, opts = {})
entity =
case opts[:serializer]
when :import
ProjectImportEntity
else
ProjectEntity
end
super(project, opts, entity)
end
end
# frozen_string_literal: true
class ProviderRepoEntity < Grape::Entity
include ImportHelper
expose :id
expose :full_name
expose :owner_name do |provider_repo, options|
owner_name(provider_repo, options[:provider])
end
expose :sanitized_name do |provider_repo|
sanitize_project_name(provider_repo[:name])
end
expose :provider_link do |provider_repo, options|
provider_project_link_url(options[:provider_url], provider_repo[:full_name])
end
private
def owner_name(provider_repo, provider)
provider_repo.dig(:owner, :login) if provider == :github
end
end
# frozen_string_literal: true
class ProviderRepoSerializer < BaseSerializer
entity ProviderRepoEntity
end
---
title: Improve GitHub and Gitea project import table UI
merge_request: 24606
author:
type: other
......@@ -12,13 +12,13 @@ namespace :import do
post :personal_access_token
get :status
get :callback
get :jobs
get :realtime_changes
end
resource :gitea, only: [:create, :new], controller: :gitea do
post :personal_access_token
get :status
get :jobs
get :realtime_changes
end
resource :gitlab, only: [:create], controller: :gitlab do
......
......@@ -52,6 +52,14 @@ module Gitlab
Gitlab::EtagCaching::Router::Route.new(
%r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z),
'environments'
),
Gitlab::EtagCaching::Router::Route.new(
%r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/import/github/realtime_changes\.json\z),
'realtime_changes_import_github'
),
Gitlab::EtagCaching::Router::Route.new(
%r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/import/gitea/realtime_changes\.json\z),
'realtime_changes_import_gitea'
)
].freeze
......
......@@ -40,4 +40,12 @@ describe Import::GiteaController do
end
end
end
describe "GET realtime_changes" do
it_behaves_like 'a GitHub-ish import controller: GET realtime_changes' do
before do
assign_host_url
end
end
end
end
......@@ -60,4 +60,8 @@ describe Import::GithubController do
describe "POST create" do
it_behaves_like 'a GitHub-ish import controller: POST create'
end
describe "GET realtime_changes" do
it_behaves_like 'a GitHub-ish import controller: GET realtime_changes'
end
end
......@@ -39,59 +39,12 @@ describe ImportHelper do
end
end
describe '#provider_project_link' do
context 'when provider is "github"' do
let(:github_server_url) { nil }
let(:provider) { OpenStruct.new(name: 'github', url: github_server_url) }
describe '#provider_project_link_url' do
let(:full_path) { '/repo/path' }
let(:host_url) { 'http://provider.com/' }
before do
stub_omniauth_setting(providers: [provider])
end
context 'when provider does not specify a custom URL' do
it 'uses default GitHub URL' do
expect(helper.provider_project_link('github', 'octocat/Hello-World'))
.to include('href="https://github.com/octocat/Hello-World"')
end
end
context 'when provider specify a custom URL' do
let(:github_server_url) { 'https://github.company.com' }
it 'uses custom URL' do
expect(helper.provider_project_link('github', 'octocat/Hello-World'))
.to include('href="https://github.company.com/octocat/Hello-World"')
end
end
context "when custom URL contains a '/' char at the end" do
let(:github_server_url) { 'https://github.company.com/' }
it "doesn't render double slash" do
expect(helper.provider_project_link('github', 'octocat/Hello-World'))
.to include('href="https://github.company.com/octocat/Hello-World"')
end
end
context 'when provider is missing' do
it 'uses the default URL' do
allow(Gitlab.config.omniauth).to receive(:providers).and_return([])
expect(helper.provider_project_link('github', 'octocat/Hello-World'))
.to include('href="https://github.com/octocat/Hello-World"')
end
end
end
context 'when provider is "gitea"' do
before do
assign(:gitea_host_url, 'https://try.gitea.io/')
end
it 'uses given host' do
expect(helper.provider_project_link('gitea', 'octocat/Hello-World'))
.to include('href="https://try.gitea.io/octocat/Hello-World"')
end
it 'appends repo full path to provider host url' do
expect(helper.provider_project_link_url(host_url, full_path)).to match('http://provider.com/repo/path')
end
end
end
......@@ -925,6 +925,21 @@ describe User do
expect(user.manageable_groups).to contain_exactly(group, subgroup)
end
end
describe '#manageable_groups_with_routes' do
it 'eager loads routes from manageable groups' do
control_count =
ActiveRecord::QueryRecorder.new(skip_cached: false) do
user.manageable_groups_with_routes.map(&:route)
end.count
create(:group, parent: subgroup)
expect do
user.manageable_groups_with_routes.map(&:route)
end.not_to exceed_all_query_limit(control_count)
end
end
end
end
......
......@@ -23,6 +23,11 @@ require 'spec_helper'
# end
shared_examples 'importer routing' do
let(:except_actions) { [] }
let(:is_realtime) { false }
before do
except_actions.push(is_realtime ? :jobs : :realtime_changes)
end
it 'to #create' do
expect(post("/import/#{provider}")).to route_to("import/#{provider}#create") unless except_actions.include?(:create)
......@@ -43,17 +48,22 @@ shared_examples 'importer routing' do
it 'to #jobs' do
expect(get("/import/#{provider}/jobs")).to route_to("import/#{provider}#jobs") unless except_actions.include?(:jobs)
end
it 'to #realtime_changes' do
expect(get("/import/#{provider}/realtime_changes")).to route_to("import/#{provider}#realtime_changes") unless except_actions.include?(:realtime_changes)
end
end
# personal_access_token_import_github POST /import/github/personal_access_token(.:format) import/github#personal_access_token
# status_import_github GET /import/github/status(.:format) import/github#status
# callback_import_github GET /import/github/callback(.:format) import/github#callback
# jobs_import_github GET /import/github/jobs(.:format) import/github#jobs
# realtime_changes_import_github GET /import/github/realtime_changes(.:format) import/github#jobs
# import_github POST /import/github(.:format) import/github#create
# new_import_github GET /import/github/new(.:format) import/github#new
describe Import::GithubController, 'routing' do
it_behaves_like 'importer routing' do
let(:provider) { 'github' }
let(:is_realtime) { true }
end
it 'to #personal_access_token' do
......@@ -63,13 +73,14 @@ end
# personal_access_token_import_gitea POST /import/gitea/personal_access_token(.:format) import/gitea#personal_access_token
# status_import_gitea GET /import/gitea/status(.:format) import/gitea#status
# jobs_import_gitea GET /import/gitea/jobs(.:format) import/gitea#jobs
# realtime_changes_import_gitea GET /import/gitea/realtime_changes(.:format) import/gitea#jobs
# import_gitea POST /import/gitea(.:format) import/gitea#create
# new_import_gitea GET /import/gitea/new(.:format) import/gitea#new
describe Import::GiteaController, 'routing' do
it_behaves_like 'importer routing' do
let(:except_actions) { [:callback] }
let(:provider) { 'gitea' }
let(:is_realtime) { true }
end
it 'to #personal_access_token' do
......
# frozen_string_literal: true
require 'spec_helper'
describe NamespaceBasicEntity do
set(:group) { create(:group) }
let(:entity) do
described_class.represent(group)
end
describe '#as_json' do
subject { entity.as_json }
it 'includes required fields' do
expect(subject).to include :id, :full_path
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe NamespaceSerializer do
it 'represents NamespaceBasicEntity entities' do
expect(described_class.entity_class).to eq(NamespaceBasicEntity)
end
end
# frozen_string_literal: true
require 'spec_helper'
describe ProjectImportEntity do
include ImportHelper
set(:project) { create(:project, import_status: :started, import_source: 'namespace/project') }
let(:provider_url) { 'https://provider.com' }
let(:entity) { described_class.represent(project, provider_url: provider_url) }
describe '#as_json' do
subject { entity.as_json }
it 'includes required fields' do
expect(subject[:import_source]).to eq(project.import_source)
expect(subject[:import_status]).to eq(project.import_status)
expect(subject[:human_import_status_name]).to eq(project.human_import_status_name)
expect(subject[:provider_link]).to eq(provider_project_link_url(provider_url, project[:import_source]))
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe ProjectSerializer do
set(:project) { create(:project) }
let(:provider_url) { 'http://provider.com' }
context 'when serializer option is :import' do
subject do
described_class.new.represent(project, serializer: :import, provider_url: provider_url)
end
before do
allow(ProjectImportEntity).to receive(:represent)
end
it 'represents with ProjectImportEntity' do
subject
expect(ProjectImportEntity)
.to have_received(:represent)
.with(project, serializer: :import, provider_url: provider_url, request: an_instance_of(EntityRequest))
end
end
context 'when serializer option is omitted' do
subject do
described_class.new.represent(project)
end
before do
allow(ProjectEntity).to receive(:represent)
end
it 'represents with ProjectEntity' do
subject
expect(ProjectEntity)
.to have_received(:represent)
.with(project, request: an_instance_of(EntityRequest))
end
end
end