Commit ad6d6232 authored by Douwe Maan's avatar Douwe Maan
Browse files

Add Bitbucket importer.

parent 7561b1c2
......@@ -72,6 +72,8 @@ v 7.8.0
- Improve database performance for GitLab
- Add Asana service (Jeremy Benoist)
- Improve project web hooks with extra data
- Add Bitbucket omniauth provider.
- Add Bitbucket importer.
v 7.7.2
- Update GitLab Shell to version 2.4.2 that fixes a bug when developers can push to protected branch
......
......@@ -30,6 +30,7 @@ gem 'omniauth-github'
gem 'omniauth-shibboleth'
gem 'omniauth-kerberos'
gem 'omniauth-gitlab'
gem 'omniauth-bitbucket'
gem 'doorkeeper', '2.1.0'
gem "rack-oauth2", "~> 1.0.5"
......
......@@ -338,6 +338,10 @@ GEM
omniauth (1.1.4)
hashie (>= 1.2, < 3)
rack
omniauth-bitbucket (0.0.2)
multi_json (~> 1.7)
omniauth (~> 1.1)
omniauth-oauth (~> 1.0)
omniauth-github (1.1.1)
omniauth (~> 1.0)
omniauth-oauth2 (~> 1.1)
......@@ -701,6 +705,7 @@ DEPENDENCIES
nprogress-rails
octokit (= 3.7.0)
omniauth (~> 1.1.3)
omniauth-bitbucket
omniauth-github
omniauth-gitlab
omniauth-google-oauth2
......
class Import::BitbucketController < Import::BaseController
before_filter :bitbucket_auth, except: :callback
# rescue_from OAuth::Error, with: :bitbucket_unauthorized
def callback
request_token = session.delete(:oauth_request_token)
raise "Session expired!" if request_token.nil?
request_token.symbolize_keys!
access_token = client.get_token(request_token, params[:oauth_verifier], callback_import_bitbucket_url)
current_user.bitbucket_access_token = access_token.token
current_user.bitbucket_access_token_secret = access_token.secret
current_user.save
redirect_to status_import_bitbucket_url
end
def status
@repos = client.projects
@already_added_projects = current_user.created_projects.where(import_type: "bitbucket")
already_added_projects_names = @already_added_projects.pluck(:import_source)
@repos.to_a.reject!{ |repo| already_added_projects_names.include? "#{repo["owner"]}/#{repo["slug"]}" }
end
def jobs
jobs = current_user.created_projects.where(import_type: "bitbucket").to_json(only: [:id, :import_status])
render json: jobs
end
def create
@repo_id = params[:repo_id] || ""
repo = client.project(@repo_id.gsub("___", "/"))
@target_namespace = params[:new_namespace].presence || repo["owner"]
@project_name = repo["slug"]
namespace = get_or_create_namespace || (render and return)
unless Gitlab::BitbucketImport::KeyAdder.new(repo, current_user).execute
@access_denied = true
render
return
end
@project = Gitlab::BitbucketImport::ProjectCreator.new(repo, namespace, current_user).execute
end
private
def client
@client ||= Gitlab::BitbucketImport::Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret)
end
def bitbucket_auth
if current_user.bitbucket_access_token.blank?
go_to_bitbucket_for_permissions
end
end
def go_to_bitbucket_for_permissions
request_token = client.request_token(callback_import_bitbucket_url)
session[:oauth_request_token] = request_token
redirect_to client.authorize_url(request_token, callback_import_bitbucket_url)
end
def bitbucket_unauthorized
go_to_bitbucket_for_permissions
end
end
......@@ -4,7 +4,7 @@ def ldap_enabled?
end
def default_providers
[:twitter, :github, :gitlab, :google_oauth2, :ldap]
[:twitter, :github, :gitlab, :bitbucket, :google_oauth2, :ldap]
end
def enabled_oauth_providers
......@@ -13,7 +13,7 @@ def enabled_oauth_providers
def enabled_social_providers
enabled_oauth_providers.select do |name|
[:twitter, :gitlab, :github, :google_oauth2].include?(name.to_sym)
[:twitter, :gitlab, :github, :bitbucket, :google_oauth2].include?(name.to_sym)
end
end
......
......@@ -273,4 +273,8 @@ def github_import_enabled?
def gitlab_import_enabled?
enabled_oauth_providers.include?(:gitlab)
end
def bitbucket_import_enabled?
enabled_oauth_providers.include?(:bitbucket)
end
end
......@@ -130,7 +130,7 @@ class Project < ActiveRecord::Base
validates_uniqueness_of :name, scope: :namespace_id
validates_uniqueness_of :path, scope: :namespace_id
validates :import_url,
format: { with: URI::regexp(%w(git http https)), message: 'should be a valid url' },
format: { with: URI::regexp(%w(ssh git http https)), message: 'should be a valid url' },
if: :import?
validates :star_count, numericality: { greater_than_or_equal_to: 0 }
validate :check_limit, on: :create
......
......@@ -46,6 +46,7 @@
# github_access_token :string(255)
# notification_email :string(255)
# password_automatically_set :boolean default(FALSE)
# bitbucket_access_token :string(255)
#
require 'carrierwave/orm/activerecord'
......
......@@ -10,9 +10,16 @@
target_field.append("/" + project_name)
target_field.data("project_name", project_name)
target_field.find('input').prop("value", origin_namespace)
- elsif @access_denied
:plain
job = $("tr#repo_#{@repo_id}")
job.find(".import-actions").html("<p class='alert alert-danger'>Access denied! Please verify you can add deploy keys to this repository.</p>"")
- else
:plain
job = $("tr#repo_#{@repo_id}")
job.attr("id", "project_#{@project.id}")
target_field = job.find(".import-target")
target_field.empty()
target_field.append('<strong>#{link_to @project.path_with_namespace, @project}</strong>')
$("table.import-jobs tbody").prepend(job)
job.addClass("active").find(".import-actions").html("<i class='fa fa-spinner fa-spin'></i> started")
%h3.page-title
%i.fa.fa-bitbucket
Import repositories from Bitbucket
%p.light
Select projects you want to import.
%hr
%p
= button_tag 'Import all projects', class: "btn btn-success js-import-all"
%table.table.import-jobs
%thead
%tr
%th From Bitbucket
%th To GitLab
%th Status
%tbody
- @already_added_projects.each do |project|
%tr{id: "project_#{project.id}", class: "#{project_status_css_class(project.import_status)}"}
%td= project.import_source
%td
%strong= link_to project.path_with_namespace, project
%td.job-status
- if project.import_status == 'finished'
%span.cgreen
%i.fa.fa-check
done
- elsif project.import_status == 'started'
%i.fa.fa-spinner.fa-spin
started
- else
= project.human_import_status_name
- @repos.each do |repo|
%tr{id: "repo_#{repo["owner"]}___#{repo["slug"]}"}
%td= "#{repo["owner"]}/#{repo["slug"]}"
%td.import-target
= "#{repo["owner"]}/#{repo["slug"]}"
%td.import-actions.job-status
= button_tag "Import", class: "btn js-add-to-import"
:coffeescript
$ ->
new ImporterStatus("#{jobs_import_bitbucket_path}", "#{import_bitbucket_path}")
%h3.page-title
%i.fa.fa-github
Import repositories from GitHub.com
Import repositories from GitHub
%p.light
Select projects you want to import.
......
%h3.page-title
%i.fa.fa-github
%i.fa.fa-heart
Import repositories from GitLab.com
%p.light
......
%div#bitbucket_import_modal.modal.hide
.modal-dialog
.modal-content
.modal-header
%a.close{href: "#", "data-dismiss" => "modal"} ×
%h3 GitHub OAuth import
.modal-body
You need to setup integration with Bitbucket first.
= link_to 'How to setup integration with Bitbucket', 'https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/integration/bitbucket.md'
\ No newline at end of file
......@@ -53,6 +53,19 @@
Import projects from GitHub
= render 'github_import_modal'
.project-import.form-group
.col-sm-2
.col-sm-10
- if bitbucket_import_enabled?
= link_to status_import_bitbucket_path do
%i.fa.fa-bitbucket
Import projects from Bitbucket
- else
= link_to '#', class: 'how_to_import_link light' do
%i.fa.fa-bitbucket
Import projects from Bitbucket
= render 'bitbucket_import_modal'
- unless request.host == 'gitlab.com'
.project-import.form-group
.col-sm-2
......
......@@ -14,6 +14,8 @@ def perform(project_id)
Gitlab::GithubImport::Importer.new(project).execute
elsif project.import_type == 'gitlab'
Gitlab::GitlabImport::Importer.new(project).execute
elsif project.import_type == 'bitbucket'
Gitlab::BitbucketImport::Importer.new(project).execute
else
true
end
......
......@@ -207,17 +207,19 @@ production: &base
# arguments, followed by optional 'args' which can be either a hash or an array.
# Documentation for this is available at http://doc.gitlab.com/ce/integration/omniauth.html
providers:
# - { name: 'google_oauth2', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET',
# - { name: 'google_oauth2', app_id: 'YOUR_APP_ID',
# app_secret: 'YOUR_APP_SECRET',
# args: { access_type: 'offline', approval_prompt: '' } }
# - { name: 'twitter', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET'}
# - { name: 'github', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET',
# - { name: 'twitter', app_id: 'YOUR_APP_ID',
# app_secret: 'YOUR_APP_SECRET'}
# - { name: 'github', app_id: 'YOUR_APP_ID',
# app_secret: 'YOUR_APP_SECRET',
# args: { scope: 'user:email' } }
# - { name: 'gitlab', app_id: 'YOUR APP ID',
# app_secret: 'YOUR APP SECRET',
# - { name: 'gitlab', app_id: 'YOUR_APP_ID',
# app_secret: 'YOUR_APP_SECRET',
# args: { scope: 'api' } }
# - { name: 'bitbucket', app_id: 'YOUR_APP_ID',
# app_secret: 'YOUR_APP_SECRET'}
......
......@@ -68,6 +68,11 @@
get :jobs
end
resource :bitbucket, only: [:create, :new], controller: :bitbucket do
get :status
get :callback
get :jobs
end
resource :gitorious, only: [:create, :new], controller: :gitorious do
get :status
get :callback
......
class AddBitbucketAccessTokenAndSecretToUser < ActiveRecord::Migration
def change
add_column :users, :bitbucket_access_token, :string
add_column :users, :bitbucket_access_token_secret, :string
end
end
......@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20150213121042) do
ActiveRecord::Schema.define(version: 20150217123345) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -410,12 +410,12 @@
end
create_table "users", force: true do |t|
t.string "email", default: "", null: false
t.string "encrypted_password", default: "", null: false
t.string "email", default: "", null: false
t.string "encrypted_password", default: "", null: false
t.string "reset_password_token"
t.datetime "reset_password_sent_at"
t.datetime "remember_created_at"
t.integer "sign_in_count", default: 0
t.integer "sign_in_count", default: 0
t.datetime "current_sign_in_at"
t.datetime "last_sign_in_at"
t.string "current_sign_in_ip"
......@@ -423,22 +423,22 @@
t.datetime "created_at"
t.datetime "updated_at"
t.string "name"
t.boolean "admin", default: false, null: false
t.integer "projects_limit", default: 10
t.string "skype", default: "", null: false
t.string "linkedin", default: "", null: false
t.string "twitter", default: "", null: false
t.boolean "admin", default: false, null: false
t.integer "projects_limit", default: 10
t.string "skype", default: "", null: false
t.string "linkedin", default: "", null: false
t.string "twitter", default: "", null: false
t.string "authentication_token"
t.integer "theme_id", default: 1, null: false
t.integer "theme_id", default: 1, null: false
t.string "bio"
t.integer "failed_attempts", default: 0
t.integer "failed_attempts", default: 0
t.datetime "locked_at"
t.string "username"
t.boolean "can_create_group", default: true, null: false
t.boolean "can_create_team", default: true, null: false
t.boolean "can_create_group", default: true, null: false
t.boolean "can_create_team", default: true, null: false
t.string "state"
t.integer "color_scheme_id", default: 1, null: false
t.integer "notification_level", default: 1, null: false
t.integer "color_scheme_id", default: 1, null: false
t.integer "notification_level", default: 1, null: false
t.datetime "password_expires_at"
t.integer "created_by_id"
t.datetime "last_credential_check_at"
......@@ -447,13 +447,15 @@
t.datetime "confirmed_at"
t.datetime "confirmation_sent_at"
t.string "unconfirmed_email"
t.boolean "hide_no_ssh_key", default: false
t.string "website_url", default: "", null: false
t.boolean "hide_no_ssh_key", default: false
t.string "website_url", default: "", null: false
t.string "github_access_token"
t.string "gitlab_access_token"
t.string "notification_email"
t.boolean "hide_no_password", default: false
t.boolean "password_automatically_set", default: false
t.boolean "hide_no_password", default: false
t.boolean "password_automatically_set", default: false
t.string "bitbucket_access_token"
t.string "bitbucket_access_token_secret"
end
add_index "users", ["admin"], name: "index_users_on_admin", using: :btree
......
TODO
\ No newline at end of file
module Gitlab
module BitbucketImport
class Client
attr_reader :consumer, :api
def initialize(access_token = nil, access_token_secret = nil)
@consumer = ::OAuth::Consumer.new(
config.app_id,
config.app_secret,
bitbucket_options
)
if access_token && access_token_secret
@api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret)
end
end
def request_token(redirect_uri)
request_token = consumer.get_request_token(oauth_callback: redirect_uri)
{
oauth_token: request_token.token,
oauth_token_secret: request_token.secret,
oauth_callback_confirmed: request_token.callback_confirmed?.to_s
}
end
def authorize_url(request_token, redirect_uri)
request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
if request_token.callback_confirmed?
request_token.authorize_url
else
request_token.authorize_url(oauth_callback: redirect_uri)
end
end
def get_token(request_token, oauth_verifier, redirect_uri)
request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash)
if request_token.callback_confirmed?
request_token.get_access_token(oauth_verifier: oauth_verifier)
else
request_token.get_access_token(oauth_callback: redirect_uri)
end
end
def user
JSON.parse(api.get("/api/1.0/user").body)
end
def issues(project_identifier)
JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/issues").body)
end
def issue_comments(project_identifier, issue_id)
JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body)
end
def project(project_identifier)
JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}").body)
end
def deploy_key(project_identifier)
JSON.parse(api.get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find { |key| key["label"] =~ /GitLab/ }
end
def add_deploy_key(project_identifier, key)
JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body)
end
def projects
JSON.parse(api.get("/api/1.0/user/repositories").body).
select { |repo| repo["scm"] == "git" }
end
private
def config
Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket"}
end
def bitbucket_options
OmniAuth::Strategies::Bitbucket.default_options[:client_options]
end
end
end
end
module Gitlab
module BitbucketImport
class Importer
attr_reader :project, :client
def initialize(project)
@project = project
@client = Client.new(project.creator.bitbucket_access_token, project.creator.bitbucket_access_token_secret)
@formatter = Gitlab::ImportFormatter.new
end
def execute
project_identifier = project.import_source
return true unless client.project(project_identifier)["has_issues"]
#Issues && Comments
issues = client.issues(project_identifier)
issues["issues"].each do |issue|
body = @formatter.author_line(issue["reported_by"]["username"], issue["content"])
comments = client.issue_comments(project_identifier, issue["local_id"])
if comments.any?
body += @formatter.comments_header
end
comments.each do |comment|
body += @formatter.comment(comment["author_info"]["username"], comment["utc_created_on"], comment["content"])
end
project.issues.create!(
description: body,
title: issue["title"],
state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened',
author_id: gl_user_id(project, issue["reported_by"]["username"])
)
end
true
end
private
def gl_user_id(project, bitbucket_id)
user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s)
(user && user.id) || project.creator_id
end
end
end
end
module Gitlab
module BitbucketImport
class KeyAdder
attr_reader :repo, :current_user, :client
def initialize(repo, current_user)
@repo, @current_user = repo, current_user
@client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret)
end
def execute
project_identifier = "#{repo["owner"]}/#{repo["slug"]}"
return true if client.deploy_key(project_identifier)
# TODO: Point to actual public key.
client.add_deploy_key(project_identifier, File.read("/Users/douwemaan/.ssh/id_rsa.pub"))
true
end
end
end
end
module Gitlab
module BitbucketImport
class ProjectCreator
attr_reader :repo, :namespace, :current_user
def initialize(repo, namespace, current_user)
@repo = repo
@namespace = namespace
@current_user = current_user
end
def execute
@project = Project.new(
name: repo["name"],
path: repo["slug"],
description: repo["description"],
namespace: namespace,
creator: current_user,
visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,