remote_mirror.rb 6.71 KB
Newer Older
1 2
# frozen_string_literal: true

3 4
class RemoteMirror < ActiveRecord::Base
  include AfterCommitQueue
5
  include MirrorAuthentication
6 7 8 9 10

  PROTECTED_BACKOFF_DELAY   = 1.minute
  UNPROTECTED_BACKOFF_DELAY = 5.minutes

  attr_encrypted :credentials,
Stan Hu's avatar
Stan Hu committed
11
                 key: Settings.attr_encrypted_db_key_base,
12 13 14 15 16 17 18 19
                 marshal: true,
                 encode: true,
                 mode: :per_attribute_iv_and_salt,
                 insecure_mode: true,
                 algorithm: 'aes-256-cbc'

  belongs_to :project, inverse_of: :remote_mirrors

20
  validates :url, presence: true, public_url: { protocols: %w(ssh git http https), allow_blank: true, enforce_user: true }
21 22 23 24 25 26 27 28 29

  before_save :set_new_remote_name, if: :mirror_url_changed?

  after_save :set_override_remote_mirror_available, unless: -> { Gitlab::CurrentSettings.current_application_settings.mirror_available }
  after_save :refresh_remote, if: :mirror_url_changed?
  after_update :reset_fields, if: :mirror_url_changed?

  after_commit :remove_remote, on: :destroy

30 31
  before_validation :store_credentials

32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
  scope :enabled, -> { where(enabled: true) }
  scope :started, -> { with_update_status(:started) }
  scope :stuck,   -> { started.where('last_update_at < ? OR (last_update_at IS NULL AND updated_at < ?)', 1.day.ago, 1.day.ago) }

  state_machine :update_status, initial: :none do
    event :update_start do
      transition [:none, :finished, :failed] => :started
    end

    event :update_finish do
      transition started: :finished
    end

    event :update_fail do
      transition started: :failed
    end

    state :started
    state :finished
    state :failed

    after_transition any => :started do |remote_mirror, _|
54
      Gitlab::Metrics.add_event(:remote_mirrors_running)
55 56 57 58 59

      remote_mirror.update(last_update_started_at: Time.now)
    end

    after_transition started: :finished do |remote_mirror, _|
60
      Gitlab::Metrics.add_event(:remote_mirrors_finished)
61 62

      timestamp = Time.now
Lin Jen-Shin's avatar
Lin Jen-Shin committed
63
      remote_mirror.update!(
64 65 66 67
        last_update_at: timestamp,
        last_successful_update_at: timestamp,
        last_error: nil,
        error_notification_sent: false
68 69 70
      )
    end

71
    after_transition started: :failed do |remote_mirror|
72
      Gitlab::Metrics.add_event(:remote_mirrors_failed)
73 74

      remote_mirror.update(last_update_at: Time.now)
75 76 77 78

      remote_mirror.run_after_commit do
        RemoteMirrorNotificationWorker.perform_async(remote_mirror.id)
      end
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
    end
  end

  def remote_name
    super || fallback_remote_name
  end

  def update_failed?
    update_status == 'failed'
  end

  def update_in_progress?
    update_status == 'started'
  end

  def update_repository(options)
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
    if ssh_mirror_url?
      if ssh_key_auth? && ssh_private_key.present?
        options[:ssh_key] = ssh_private_key
      end

      if ssh_known_hosts.present?
        options[:known_hosts] = ssh_known_hosts
      end
    end

    Gitlab::Git::RemoteMirror.new(
      project.repository.raw,
      remote_name,
      **options
    ).update
110 111
  end

112
  def sync?
113
    enabled?
114 115
  end

116
  def sync
117
    return unless sync?
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145

    if recently_scheduled?
      RepositoryUpdateRemoteMirrorWorker.perform_in(backoff_delay, self.id, Time.now)
    else
      RepositoryUpdateRemoteMirrorWorker.perform_async(self.id, Time.now)
    end
  end

  def enabled
    return false unless project && super
    return false unless project.remote_mirror_available?
    return false unless project.repository_exists?
    return false if project.pending_delete?

    true
  end
  alias_method :enabled?, :enabled

  def updated_since?(timestamp)
    last_update_started_at && last_update_started_at > timestamp && !update_failed?
  end

  def mark_for_delete_if_blank_url
    mark_for_destruction if url.blank?
  end

  def mark_as_failed(error_message)
    update_column(:last_error, Gitlab::UrlSanitizer.sanitize(error_message))
146
    update_fail
147 148 149 150 151 152
  end

  def url=(value)
    super(value) && return unless Gitlab::UrlSanitizer.valid?(value)

    mirror_url = Gitlab::UrlSanitizer.new(value)
153 154
    self.credentials ||= {}
    self.credentials = self.credentials.merge(mirror_url.credentials)
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175

    super(mirror_url.sanitized_url)
  end

  def url
    if super
      Gitlab::UrlSanitizer.new(super, credentials: credentials).full_url
    end
  rescue
    super
  end

  def safe_url
    return if url.nil?

    result = URI.parse(url)
    result.password = '*****' if result.password
    result.user = '*****' if result.user && result.user != "git" # tokens or other data may be saved as user
    result.to_s
  end

176 177
  def ensure_remote!
    return unless project
178
    return unless remote_name && remote_url
179 180 181

    # If this fails or the remote already exists, we won't know due to
    # https://gitlab.com/gitlab-org/gitaly/issues/1317
182
    project.repository.add_remote(remote_name, remote_url)
183 184
  end

185 186 187 188
  def after_sent_notification
    update_column(:error_notification_sent, true)
  end

189 190
  private

191 192 193 194 195 196 197 198 199 200 201 202 203
  def store_credentials
    # This is a necessary workaround for attr_encrypted, which doesn't otherwise
    # notice that the credentials have changed
    self.credentials = self.credentials
  end

  # The remote URL omits any password if SSH public-key authentication is in use
  def remote_url
    return url unless ssh_key_auth? && password.present?

    Gitlab::UrlSanitizer.new(read_attribute(:url), credentials: { user: user }).full_url
  rescue
    super
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
  end

  def fallback_remote_name
    return unless id

    "remote_mirror_#{id}"
  end

  def recently_scheduled?
    return false unless self.last_update_started_at

    self.last_update_started_at >= Time.now - backoff_delay
  end

  def backoff_delay
    if self.only_protected_branches
      PROTECTED_BACKOFF_DELAY
    else
      UNPROTECTED_BACKOFF_DELAY
    end
  end

  def reset_fields
    update_columns(
      last_error: nil,
      last_update_at: nil,
      last_successful_update_at: nil,
231 232
      update_status: 'finished',
      error_notification_sent: false
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
    )
  end

  def set_override_remote_mirror_available
    enabled = read_attribute(:enabled)

    project.update(remote_mirror_available_overridden: enabled)
  end

  def set_new_remote_name
    self.remote_name = "remote_mirror_#{SecureRandom.hex}"
  end

  def refresh_remote
    return unless project

    # Before adding a new remote we have to delete the data from
    # the previous remote name
    prev_remote_name = remote_name_was || fallback_remote_name
    run_after_commit do
      project.repository.async_remove_remote(prev_remote_name)
    end

256
    project.repository.add_remote(remote_name, remote_url)
257 258 259 260 261 262 263 264 265
  end

  def remove_remote
    return unless project # could be pending to delete so don't need to touch the git repository

    project.repository.async_remove_remote(remote_name)
  end

  def mirror_url_changed?
266
    url_changed? || credentials_changed?
267 268
  end
end