SSH public-key authentication for push mirroring

parent b1b4c944
export default {
PASSWORD: 'password',
SSH: 'ssh_public_key',
};
......@@ -3,10 +3,12 @@ import _ from 'underscore';
import { __ } from '~/locale';
import Flash from '~/flash';
import axios from '~/lib/utils/axios_utils';
import SSHMirror from './ssh_mirror';
export default class MirrorRepos {
constructor(container) {
this.$container = $(container);
this.$password = null;
this.$form = $('.js-mirror-form', this.$container);
this.$urlInput = $('.js-mirror-url', this.$form);
this.$protectedBranchesInput = $('.js-mirror-protected', this.$form);
......@@ -26,6 +28,18 @@ export default class MirrorRepos {
this.$authMethod.on('change', () => this.togglePassword());
this.$password.on('input.updateUrl', () => this.debouncedUpdateUrl());
this.initMirrorSSH();
}
initMirrorSSH() {
if (this.$password) {
this.$password.off('input.updateUrl');
}
this.$password = undefined;
this.sshMirror = new SSHMirror('.js-mirror-form');
this.sshMirror.init();
}
updateUrl() {
......
This diff is collapsed.
import initForm from '../form';
import MirrorRepos from './mirror_repos';
import MirrorRepos from '~/mirrors/mirror_repos';
document.addEventListener('DOMContentLoaded', () => {
initForm();
......
......@@ -1223,3 +1223,27 @@ pre.light-well {
opacity: 1;
}
}
.project-mirror-settings {
.btn-show-advanced {
min-width: 135px;
.label-show {
display: none;
}
.label-hide {
display: inline;
}
&.show-advanced {
.label-show {
display: inline;
}
.label-hide {
display: none;
}
}
}
}
......@@ -77,6 +77,10 @@ class Projects::MirrorsController < Projects::ApplicationController
id
enabled
only_protected_branches
auth_method
password
ssh_known_hosts
regenerate_ssh_private_key
]
]
end
......
......@@ -2,6 +2,9 @@
module MirrorHelper
def mirrors_form_data_attributes
{ project_mirror_endpoint: project_mirror_path(@project) }
{
project_mirror_ssh_endpoint: ssh_host_keys_project_mirror_path(@project, :json),
project_mirror_endpoint: project_mirror_path(@project, :json)
}
end
end
# frozen_string_literal: true
# Mirroring may use password or SSH public-key authentication. This concern
# implements support for persisting the necessary data in a `credentials`
# serialized attribute. It also needs an `url` method to be defined
module MirrorAuthentication
SSH_PRIVATE_KEY_OPTS = {
type: 'RSA',
bits: 4096
}.freeze
extend ActiveSupport::Concern
included do
validates :auth_method, inclusion: { in: %w[password ssh_public_key] }, allow_blank: true
# We should generate a key even if there's no SSH URL present
before_validation :generate_ssh_private_key!, if: -> {
regenerate_ssh_private_key || ( auth_method == 'ssh_public_key' && ssh_private_key.blank? )
}
credentials_field :auth_method, reader: false
credentials_field :ssh_known_hosts
credentials_field :ssh_known_hosts_verified_at
credentials_field :ssh_known_hosts_verified_by_id
credentials_field :ssh_private_key
credentials_field :user
credentials_field :password
end
class_methods do
def credentials_field(name, reader: true)
if reader
define_method(name) do
credentials[name] if credentials.present?
end
end
define_method("#{name}=") do |value|
self.credentials ||= {}
# Removal of the password, username, etc, generally causes an update of
# the value to the empty string. Detect and gracefully handle this case.
if value.present?
self.credentials[name] = value
else
self.credentials.delete(name)
end
end
end
end
attr_accessor :regenerate_ssh_private_key
def ssh_key_auth?
ssh_mirror_url? && auth_method == 'ssh_public_key'
end
def password_auth?
auth_method == 'password'
end
def ssh_mirror_url?
url&.start_with?('ssh://')
end
def ssh_known_hosts_verified_by
@ssh_known_hosts_verified_by ||= ::User.find_by(id: ssh_known_hosts_verified_by_id)
end
def ssh_known_hosts_fingerprints
::SshHostKey.fingerprint_host_keys(ssh_known_hosts)
end
def auth_method
auth_method = credentials.fetch(:auth_method, nil) if credentials.present?
auth_method.presence || 'password'
end
def ssh_public_key
return nil if ssh_private_key.blank?
comment = "git@#{::Gitlab.config.gitlab.host}"
::SSHKey.new(ssh_private_key, comment: comment).ssh_public_key
end
def generate_ssh_private_key!
self.ssh_private_key = ::SSHKey.generate(SSH_PRIVATE_KEY_OPTS).private_key
end
end
......@@ -2,6 +2,7 @@
class RemoteMirror < ActiveRecord::Base
include AfterCommitQueue
include MirrorAuthentication
PROTECTED_BACKOFF_DELAY = 1.minute
UNPROTECTED_BACKOFF_DELAY = 5.minutes
......@@ -28,6 +29,8 @@ class RemoteMirror < ActiveRecord::Base
after_commit :remove_remote, on: :destroy
before_validation :store_credentials
scope :enabled, -> { where(enabled: true) }
scope :started, -> { with_update_status(:started) }
scope :stuck, -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }
......@@ -84,7 +87,21 @@ class RemoteMirror < ActiveRecord::Base
end
def update_repository(options)
raw.update(options)
if ssh_mirror_url?
if ssh_key_auth? && ssh_private_key.present?
options[:ssh_key] = ssh_private_key
end
if ssh_known_hosts.present?
options[:known_hosts] = ssh_known_hosts
end
end
Gitlab::Git::RemoteMirror.new(
project.repository.raw,
remote_name,
**options
).update
end
def sync?
......@@ -128,7 +145,8 @@ class RemoteMirror < ActiveRecord::Base
super(value) && return unless Gitlab::UrlSanitizer.valid?(value)
mirror_url = Gitlab::UrlSanitizer.new(value)
self.credentials = mirror_url.credentials
self.credentials ||= {}
self.credentials = self.credentials.merge(mirror_url.credentials)
super(mirror_url.sanitized_url)
end
......@@ -152,17 +170,28 @@ class RemoteMirror < ActiveRecord::Base
def ensure_remote!
return unless project
return unless remote_name && url
return unless remote_name && remote_url
# If this fails or the remote already exists, we won't know due to
# https://gitlab.com/gitlab-org/gitaly/issues/1317
project.repository.add_remote(remote_name, url)
project.repository.add_remote(remote_name, remote_url)
end
private
def raw
@raw ||= Gitlab::Git::RemoteMirror.new(project.repository.raw, remote_name)
def store_credentials
# This is a necessary workaround for attr_encrypted, which doesn't otherwise
# notice that the credentials have changed
self.credentials = self.credentials
end
# The remote URL omits any password if SSH public-key authentication is in use
def remote_url
return url unless ssh_key_auth? && password.present?
Gitlab::UrlSanitizer.new(read_attribute(:url), credentials: { user: user }).full_url
rescue
super
end
def fallback_remote_name
......@@ -214,7 +243,7 @@ class RemoteMirror < ActiveRecord::Base
project.repository.async_remove_remote(prev_remote_name)
end
project.repository.add_remote(remote_name, url)
project.repository.add_remote(remote_name, remote_url)
end
def remove_remote
......@@ -224,6 +253,6 @@ class RemoteMirror < ActiveRecord::Base
end
def mirror_url_changed?
url_changed? || encrypted_credentials_changed?
url_changed? || credentials_changed?
end
end
......@@ -3,11 +3,7 @@
class ProjectMirrorEntity < Grape::Entity
expose :id
expose :remote_mirrors_attributes do |project|
next [] unless project.remote_mirrors.present?
project.remote_mirrors.map do |remote|
remote.as_json(only: %i[id url enabled])
end
expose :remote_mirrors_attributes, using: RemoteMirrorEntity do |project|
project.remote_mirrors
end
end
# frozen_string_literal: true
class RemoteMirrorEntity < Grape::Entity
expose :id
expose :url
expose :enabled
expose :auth_method
expose :ssh_known_hosts
expose :ssh_public_key
expose :ssh_known_hosts_fingerprints do |remote_mirror|
remote_mirror.ssh_known_hosts_fingerprints.as_json
end
end
......@@ -11,7 +11,7 @@ module Projects
begin
remote_mirror.ensure_remote!
repository.fetch_remote(remote_mirror.remote_name, no_tags: true)
repository.fetch_remote(remote_mirror.remote_name, ssh_auth: remote_mirror, no_tags: true)
opts = {}
if remote_mirror.only_protected_branches?
......
- mirror = f.object
- is_push = local_assigns.fetch(:is_push, false)
- auth_options = [[_('Password'), 'password'], [_('SSH public key'), 'ssh_public_key']]
- regen_data = { auth_method: 'ssh_public_key', regenerate_ssh_private_key: true }
- ssh_public_key_present = mirror.ssh_public_key.present?
.form-group
= f.label :auth_method, _('Authentication method'), class: 'label-bold'
= f.select :auth_method,
options_for_select(auth_options, mirror.auth_method),
{}, { class: "form-control js-mirror-auth-type" }
.form-group
.collapse.js-well-changing-auth
.changing-auth-method= icon('spinner spin lg')
.well-password-auth.collapse.js-well-password-auth
= f.label :password, _("Password"), class: "label-bold"
= f.password_field :password, value: mirror.password, class: 'form-control'
- unless is_push
.well-ssh-auth.collapse.js-well-ssh-auth
%p.js-ssh-public-key-present{ class: ('collapse' unless ssh_public_key_present) }
= _('Here is the public SSH key that needs to be added to the remote server. For more information, please refer to the documentation.')
%p.js-ssh-public-key-pending{ class: ('collapse' if ssh_public_key_present) }
= _('An SSH key will be automatically generated when the form is submitted. For more information, please refer to the documentation.')
.clearfix.js-ssh-public-key-wrap{ class: ('collapse' unless ssh_public_key_present) }
%code.prepend-top-10.ssh-public-key
= mirror.ssh_public_key
= clipboard_button(text: mirror.ssh_public_key, title: _("Copy SSH public key to clipboard"), class: 'prepend-top-10 btn-copy-ssh-public-key')
= button_tag type: 'button',
data: { endpoint: project_mirror_path(@project), project_data: { import_data_attributes: regen_data } },
class: "btn btn-inverted btn-warning prepend-top-10 js-btn-regenerate-ssh-key#{ ' collapse' unless ssh_public_key_present }" do
= icon('spinner spin', class: 'js-spinner d-none')
= _('Regenerate key')
= render 'projects/mirrors/regenerate_public_ssh_key_confirm_modal'
......@@ -59,5 +59,7 @@
.badge.mirror-error-badge{ data: { toggle: 'tooltip', html: 'true' }, title: html_escape(mirror.last_error.try(:strip)) }= _('Error')
%td.mirror-action-buttons
.btn-group.mirror-actions-group.pull-right{ role: 'group' }
- if mirror.ssh_key_auth?
= clipboard_button(text: mirror.ssh_public_key, class: 'btn btn-default', title: _('Copy SSH public key'))
= render 'shared/remote_mirror_update_button', remote_mirror: mirror
%button.js-delete-mirror.btn.btn-danger{ type: 'button', data: { mirror_id: mirror.id, toggle: 'tooltip', container: 'body' }, title: _('Remove') }= icon('trash-o')
- protocols = Gitlab::UrlSanitizer::ALLOWED_SCHEMES.join('|')
.form-group
= label_tag :mirror_direction, _('Mirror direction'), class: 'label-light'
= select_tag :mirror_direction, options_for_select([[_('Push'), 'push']]), class: 'form-control js-mirror-direction', disabled: true
= f.fields_for :remote_mirrors, @project.remote_mirrors.build do |rm_f|
= rm_f.hidden_field :enabled, value: '1'
= rm_f.hidden_field :url, class: 'js-mirror-url-hidden', required: true, pattern: "(#{protocols}):\/\/.+"
= rm_f.hidden_field :only_protected_branches, class: 'js-mirror-protected-hidden'
.form-group
= label_tag :auth_method, _('Authentication method'), class: 'label-bold'
= select_tag :auth_method, options_for_select([[_('None'), 'none'], [_('Password'), 'password']], 'none'), { class: "form-control js-auth-method" }
.form-group.js-password-group.collapse
= label_tag :password, _('Password'), class: 'label-bold'
= text_field_tag :password, '', class: 'form-control js-password'
= render partial: "projects/mirrors/mirror_repos_push", locals: { f: f }
- protocols = Gitlab::UrlSanitizer::ALLOWED_SCHEMES.join('|')
= f.fields_for :remote_mirrors, @project.remote_mirrors.build do |rm_f|
= rm_f.hidden_field :enabled, value: '1'
= rm_f.hidden_field :url, class: 'js-mirror-url-hidden', required: true, pattern: "(#{protocols}):\/\/.+"
= rm_f.hidden_field :only_protected_branches, class: 'js-mirror-protected-hidden'
= render partial: 'projects/mirrors/ssh_host_keys', locals: { f: rm_f }
= render partial: 'projects/mirrors/authentication_method', locals: { f: rm_f, is_push: true }
.modal.js-regenerate-public-ssh-key-confirm-modal{ tabindex: -1 }
.modal-dialog
.modal-content
.modal-header
%h3.modal-title.page-title
Regenerate public SSH key?
%button.close.js-cancel{ type: 'button', 'data-dismiss': 'modal', 'aria-label' => _('Close') }
%span{ 'aria-hidden': true } &times;
.modal-body
%p= _('Are you sure you want to regenerate the public key? You will have to update the public key on the remote server before mirroring will work again.')
.form-actions.modal-footer
= button_tag _('Cancel'), type: 'button', class: 'btn js-cancel'
= button_tag _('Regenerate key'), type: 'button', class: 'btn btn-inverted btn-warning js-confirm'
= render 'projects/mirrors/mirror_repos'
- mirror = f.object
- verified_by = mirror.ssh_known_hosts_verified_by
- verified_at = mirror.ssh_known_hosts_verified_at
.form-group.js-ssh-host-keys-section{ class: ('collapse' unless mirror.ssh_mirror_url?) }
%button.btn.btn-inverted.btn-success.inline.js-detect-host-keys.append-right-10{ type: 'button' }
= icon('spinner spin', class: 'js-spinner d-none')
= _('Detect host keys')
.fingerprint-ssh-info.js-fingerprint-ssh-info.prepend-top-10.append-bottom-10{ class: ('collapse' unless mirror.ssh_mirror_url?) }
%label.label-bold
= _('Fingerprints')
.fingerprints-list.js-fingerprints-list
- mirror.ssh_known_hosts_fingerprints.each do |fp|
%code= fp.fingerprint
- if verified_at
.form-text.text-muted.js-fingerprint-verification
%i.fa.fa-check.fingerprint-verified
Verified by
- if verified_by
= link_to verified_by.name, user_path(verified_by)
- else
= _('a deleted user')
#{time_ago_in_words(verified_at)} ago
.js-ssh-hosts-advanced.inline
%button.btn.btn-default.btn-show-advanced.show-advanced{ type: 'button' }
%span.label-show
= _('Input host keys manually')
%span.label-hide
= _('Hide host keys manual input')
.js-ssh-known-hosts.collapse.prepend-top-default
= f.label :ssh_known_hosts, _('SSH host keys'), class: 'label-bold'
= f.text_area :ssh_known_hosts, class: 'form-control known-hosts js-known-hosts', rows: '10'
......@@ -3,7 +3,7 @@
- @content_class = "limit-container-width" unless fluid_layout
= render "projects/default_branch/show"
= render "projects/mirrors/show"
= render "projects/mirrors/mirror_repos"
-# Protected branches & tags use a lot of nested partials.
-# The shared parts of the views can be found in the `shared` directory.
......
---
title: Allow SSH public-key authentication for push mirroring
merge_request: 22982
author:
type: added
......@@ -135,23 +135,25 @@ If the mirror updates successfully, it will be enqueued once again with a small
If the mirror fails (for example, a branch diverged from upstream), the project's backoff period is
increased each time it fails, up to a maximum amount of time.
### SSH authentication **[STARTER]**
### SSH authentication
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2551) in [GitLab Starter](https://about.gitlab.com/pricing/) 9.5.
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/2551) for Push mirroring in [GitLab Starter](https://about.gitlab.com/pricing/) 9.5.
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/22982) for Pull mirroring in [GitLab Core](https://about.gitlab.com/pricing/) 11.6
SSH authentication is mutual:
- You have to prove to the server that you're allowed to access the repository.
- The server also has to prove to *you* that it's who it claims to be.
You provide your credentials as a password or public key. The server that the source repository
resides on provides its credentials as a "host key", the fingerprint of which needs to be verified manually.
You provide your credentials as a password or public key. The server that the
other repository resides on provides its credentials as a "host key", the
fingerprint of which needs to be verified manually.
If you're mirroring over SSH (that is, using an `ssh://` URL), you can authenticate using:
- Password-based authentication, just as over HTTPS.
- Public key authentication. This is often more secure than password authentication, especially when
the source repository supports [Deploy Keys](../ssh/README.md#deploy-keys).
- Public key authentication. This is often more secure than password authentication,
especially when the other repository supports [Deploy Keys](../ssh/README.md#deploy-keys).
To get started:
......@@ -171,9 +173,9 @@ If you click the:
- **Detect host keys** button, GitLab will fetch the host keys from the server and display the fingerprints.
- **Input host keys manually** button, a field is displayed where you can paste in host keys.
You now need to verify that the fingerprints are those you expect. GitLab.com
and other code hosting sites publish their fingerprints in the open for you
to check:
Assuming you used the former, you now need to verify that the fingerprints are
those you expect. GitLab.com and other code hosting sites publish their
fingerprints in the open for you to check:
- [AWS CodeCommit](http://docs.aws.amazon.com/codecommit/latest/userguide/regions.html#regions-fingerprints)
- [Bitbucket](https://confluence.atlassian.com/bitbucket/use-the-ssh-protocol-with-bitbucket-cloud-221449711.html#UsetheSSHprotocolwithBitbucketCloud-KnownhostorBitbucket%27spublickeyfingerprints)
......@@ -184,7 +186,8 @@ to check:
- [SourceForge](https://sourceforge.net/p/forge/documentation/SSH%20Key%20Fingerprints/)
Other providers will vary. If you're running self-managed GitLab, or otherwise
have access to the source server, you can securely gather the key fingerprints:
have access to the server for the other repository, you can securely gather the
key fingerprints:
```sh
$ cat /etc/ssh/ssh_host*pub | ssh-keygen -E md5 -l -f -
......@@ -196,25 +199,27 @@ $ cat /etc/ssh/ssh_host*pub | ssh-keygen -E md5 -l -f -
NOTE: **Note:**
You may need to exclude `-E md5` for some older versions of SSH.
When pulling changes from the source repository, GitLab will now check that at least one of the stored
host keys matches before connecting. This can prevent malicious code from being injected into your
mirror, or your password being stolen.
When mirroring the repository, GitLab will now check that at least one of the
stored host keys matches before connecting. This can prevent malicious code from
being injected into your mirror, or your password being stolen.
### SSH public key authentication
To use SSH public key authentication, you'll also need to choose that option from the **Authentication method**
dropdown. GitLab will generate a 4096-bit RSA key and display the public component of that key to you.
To use SSH public key authentication, you'll also need to choose that option
from the **Authentication method** dropdown. GitLab will generate a 4096-bit RSA
key and display the public component of that key to you.
You then need to add the public SSH key to the source repository configuration. If:
You then need to add the public SSH key to the other repository's configuration:
- The source is hosted on GitLab, you should add the public SSH key as a [Deploy Key](../ssh/README.md#deploy-keys).
- The source is hosted elsewhere, you may need to add the key to your user's `authorized_keys` file.
Paste the entire public SSH key into the file on its own line and save it.
- If the other repository is hosted on GitLab, you should add the public SSH key
as a [Deploy Key](../ssh/README.md#deploy-keys).
- If the other repository is hosted elsewhere, you may need to add the key to
your user's `authorized_keys` file. Paste the entire public SSH key into the
file on its own line and save it.
Once the public key is set up on the source repository, click the **Mirror repository** button and
your mirror will begin working.
If you need to change the key at any time, you can click the **Regenerate key** button to do so. You'll have to update the source repository with the new key to keep the mirror running.
If you need to change the key at any time, you can remove and re-add the mirror
to generate a new key. You'll have to update the other repository with the new
key to keep the mirror running.
### Overwrite diverged branches **[STARTER]**
......
......@@ -5,14 +5,24 @@ module Gitlab
class RemoteMirror
include Gitlab::Git::WrapsGitalyErrors
def initialize(repository, ref_name)
attr_reader :repository, :ref_name, :only_branches_matching, :ssh_key, :known_hosts
def initialize(repository, ref_name, only_branches_matching: [], ssh_key: nil, known_hosts: nil)
@repository = repository
@ref_name = ref_name
@only_branches_matching = only_branches_matching
@ssh_key = ssh_key
@known_hosts = known_hosts
end
def update(only_branches_matching: [])
def update
wrapped_gitaly_errors do
@repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching)
repository.gitaly_remote_client.update_remote_mirror(
ref_name,
only_branches_matching,
ssh_key: ssh_key,
known_hosts: known_hosts
)
end
end
end
......
......@@ -68,13 +68,18 @@ module Gitlab
encode_utf8(response.ref)
end
def update_remote_mirror(ref_name, only_branches_matching)
def update_remote_mirror(ref_name, only_branches_matching, ssh_key: nil, known_hosts: nil)
req_enum = Enumerator.new do |y|
y.yield Gitaly::UpdateRemoteMirrorRequest.new(
first_request = Gitaly::UpdateRemoteMirrorRequest.new(
repository: @gitaly_repo,
ref_name: ref_name
)
first_request.ssh_key = ssh_key if ssh_key.present?
first_request.known_hosts = known_hosts if known_hosts.present?
y.yield(first_request)
current_size = 0
slices = only_branches_matching.slice_before do |branch_name|
......
......@@ -69,7 +69,7 @@ module Gitlab
no_tags: no_tags, timeout: timeout, no_prune: !prune
)
if ssh_auth&.ssh_import?
if ssh_auth&.ssh_mirror_url?
if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present?