Add DNS verification to Pages custom domains

app/controllers/projects/pages_domains_controller.rb

@@ -3,7 +3,7 @@ class Projects::PagesDomainsController < Projects::ApplicationController
 
   before_action :require_pages_enabled!
   before_action :authorize_update_pages!, except: [:show]
-  before_action :domain, only: [:show, :destroy]
+  before_action :domain, only: [:show, :destroy, :verify]
 
   def show
   end
@@ -12,11 +12,23 @@ class Projects::PagesDomainsController < Projects::ApplicationController
     @domain = @project.pages_domains.new
   end
 
+  def verify
+    result = VerifyPagesDomainService.new(@domain).execute
+
+    if result[:status] == :success
+      flash[:notice] = 'Successfully verified domain ownership'
+    else
+      flash[:alert] = 'Failed to verify domain ownership'
+    end
+
+    redirect_to project_pages_domain_path(@project, @domain)
+  end
+
   def create
     @domain = @project.pages_domains.create(pages_domain_params)
 
     if @domain.valid?
-      redirect_to project_pages_path(@project)
+      redirect_to project_pages_domain_path(@project, @domain)
     else
       render 'new'
     end
@@ -46,6 +58,6 @@ class Projects::PagesDomainsController < Projects::ApplicationController
   end
 
   def domain
-    @domain ||= @project.pages_domains.find_by(domain: params[:id].to_s)
+    @domain ||= @project.pages_domains.find_by!(domain: params[:id].to_s)
   end
 end

app/helpers/application_settings_helper.rb

@@ -199,6 +199,7 @@ module ApplicationSettingsHelper
       :metrics_port,
       :metrics_sample_interval,
       :metrics_timeout,
+      :pages_domain_verification_enabled,
       :password_authentication_enabled_for_web,
       :password_authentication_enabled_for_git,
       :performance_bar_allowed_group_id,

app/mailers/emails/pages_domains.rb

+module Emails
+  module PagesDomains
+    def pages_domain_enabled_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("GitLab Pages domain '#{domain.domain}' has been enabled")
+      )
+    end
+
+    def pages_domain_disabled_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("GitLab Pages domain '#{domain.domain}' has been disabled")
+      )
+    end
+
+    def pages_domain_verification_succeeded_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("Verification succeeded for GitLab Pages domain '#{domain.domain}'")
+      )
+    end
+
+    def pages_domain_verification_failed_email(domain, recipient)
+      @domain = domain
+      @project = domain.project
+
+      mail(
+        to: recipient.notification_email,
+        subject: subject("ACTION REQUIRED: Verification failed for GitLab Pages domain '#{domain.domain}'")
+      )
+    end
+  end
+end

app/mailers/notify.rb

@@ -5,6 +5,7 @@ class Notify < BaseMailer
   include Emails::Issues
   include Emails::MergeRequests
   include Emails::Notes
+  include Emails::PagesDomains
   include Emails::Projects
   include Emails::Profile
   include Emails::Pipelines

app/models/pages_domain.rb

 class PagesDomain < ActiveRecord::Base
+  VERIFICATION_KEY = 'gitlab-pages-verification-code'.freeze
+  VERIFICATION_THRESHOLD = 3.days.freeze
+
   belongs_to :project
 
   validates :domain, hostname: { allow_numeric_hostname: true }
   validates :domain, uniqueness: { case_sensitive: false }
   validates :certificate, certificate: true, allow_nil: true, allow_blank: true
   validates :key, certificate_key: true, allow_nil: true, allow_blank: true
+  validates :verification_code, presence: true, allow_blank: false
 
   validate :validate_pages_domain
   validate :validate_matching_key, if: ->(domain) { domain.certificate.present? || domain.key.present? }
@@ -16,10 +20,32 @@ class PagesDomain < ActiveRecord::Base
     key: Gitlab::Application.secrets.db_key_base,
     algorithm: 'aes-256-cbc'
 
+  after_initialize :set_verification_code
   after_create :update_daemon
-  after_save :update_daemon
+  after_update :update_daemon, if: :pages_config_changed?
   after_destroy :update_daemon
 
+  scope :enabled, -> { where('enabled_until >= ?', Time.now ) }
+  scope :needs_verification, -> do
+    verified_at = arel_table[:verified_at]
+    enabled_until = arel_table[:enabled_until]
+    threshold = Time.now + VERIFICATION_THRESHOLD
+
+    where(verified_at.eq(nil).or(enabled_until.eq(nil).or(enabled_until.lt(threshold))))
+  end
+
+  def verified?
+    !!verified_at
+  end
+
+  def unverified?
+    !verified?
+  end
+
+  def enabled?
+    !Gitlab::CurrentSettings.pages_domain_verification_enabled? || enabled_until.present?
+  end
+
   def to_param
     domain
   end
@@ -84,12 +110,49 @@ class PagesDomain < ActiveRecord::Base
     @certificate_text ||= x509.try(:to_text)
   end
 
+  # Verification codes may be TXT records for domain or verification_domain, to
+  # support the use of CNAME records on domain.
+  def verification_domain
+    return unless domain.present?
+
+    "_#{VERIFICATION_KEY}.#{domain}"
+  end
+
+  def keyed_verification_code
+    return unless verification_code.present?
+
+    "#{VERIFICATION_KEY}=#{verification_code}"
+  end
+
   private
 
+  def set_verification_code
+    return if self.verification_code.present?
+
+    self.verification_code = SecureRandom.hex(16)
+  end
+
   def update_daemon
     ::Projects::UpdatePagesConfigurationService.new(project).execute
   end
 
+  def pages_config_changed?
+    project_id_changed? ||
+      domain_changed? ||
+      certificate_changed? ||
+      key_changed? ||
+      became_enabled? ||
+      became_disabled?
+  end
+
+  def became_enabled?
+    enabled_until.present? && !enabled_until_was.present?
+  end
+
+  def became_disabled?
+    !enabled_until.present? && enabled_until_was.present?
+  end
+
   def validate_matching_key
     unless has_matching_key?
       self.errors.add(:key, "doesn't match the certificate")

app/services/notification_service.rb

@@ -339,6 +339,30 @@ class NotificationService
     end
   end
 
+  def pages_domain_verification_succeeded(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_verification_succeeded_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_verification_failed(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_verification_failed_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_enabled(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_enabled_email(domain, user).deliver_later
+    end
+  end
+
+  def pages_domain_disabled(domain)
+    recipients_for_pages_domain(domain).each do |user|
+      mailer.pages_domain_disabled_email(domain, user).deliver_later
+    end
+  end
+
   protected
 
   def new_resource_email(target, method)
@@ -433,6 +457,14 @@ class NotificationService
 
   private
 
+  def recipients_for_pages_domain(domain)
+    project = domain.project
+
+    return [] unless project
+
+    notifiable_users(project.team.masters, :watch, target: project)
+  end
+
   def notifiable?(*args)
     NotificationRecipientService.notifiable?(*args)
   end

app/services/projects/update_pages_configuration_service.rb

@@ -23,7 +23,7 @@ module Projects
     end
 
     def pages_domains_config
-      project.pages_domains.map do |domain|
+      enabled_pages_domains.map do |domain|
         {
           domain: domain.domain,
           certificate: domain.certificate,
@@ -32,6 +32,14 @@ module Projects
       end
     end
 
+    def enabled_pages_domains
+      if Gitlab::CurrentSettings.pages_domain_verification_enabled?
+        project.pages_domains.enabled
+      else
+        project.pages_domains
+      end
+    end
+
     def reload_daemon
       # GitLab Pages daemon constantly watches for modification time of `pages.path`
       # It reloads configuration when `pages.path` is modified

app/services/verify_pages_domain_service.rb

+require 'resolv'
+
+class VerifyPagesDomainService < BaseService
+  # The maximum number of seconds to be spent on each DNS lookup
+  RESOLVER_TIMEOUT_SECONDS = 15
+
+  # How long verification lasts for
+  VERIFICATION_PERIOD = 7.days
+
+  attr_reader :domain
+
+  def initialize(domain)
+    @domain = domain
+  end
+
+  def execute
+    return error("No verification code set for #{domain.domain}") unless domain.verification_code.present?
+
+    if !verification_enabled? || dns_record_present?
+      verify_domain!
+    elsif expired?
+      disable_domain!
+    else
+      unverify_domain!
+    end
+  end
+
+  private
+
+  def verify_domain!
+    was_disabled = !domain.enabled?
+    was_unverified = domain.unverified?
+
+    # Prevent any pre-existing grace period from being truncated
+    reverify = [domain.enabled_until, VERIFICATION_PERIOD.from_now].compact.max
+
+    domain.update!(verified_at: Time.now, enabled_until: reverify)
+
+    if was_disabled
+      notify(:enabled)
+    elsif was_unverified
+      notify(:verification_succeeded)
+    end
+
+    success
+  end
+
+  def unverify_domain!
+    if domain.verified?
+      domain.update!(verified_at: nil)
+      notify(:verification_failed)
+    end
+
+    error("Couldn't verify #{domain.domain}")
+  end
+
+  def disable_domain!
+    domain.update!(verified_at: nil, enabled_until: nil)
+
+    notify(:disabled)
+
+    error("Couldn't verify #{domain.domain}. It is now disabled.")
+  end
+
+  # A domain is only expired until `disable!` has been called
+  def expired?
+    domain.enabled_until && domain.enabled_until < Time.now
+  end
+
+  def dns_record_present?
+    Resolv::DNS.open do |resolver|
+      resolver.timeouts = RESOLVER_TIMEOUT_SECONDS
+
+      check(domain.domain, resolver) || check(domain.verification_domain, resolver)
+    end
+  end
+
+  def check(domain_name, resolver)
+    records = parse(txt_records(domain_name, resolver))
+
+    records.any? do |record|
+      record == domain.keyed_verification_code || record == domain.verification_code
+    end
+  rescue => err
+    log_error("Failed to check TXT records on #{domain_name} for #{domain.domain}: #{err}")
+    false
+  end
+
+  def txt_records(domain_name, resolver)
+    resolver.getresources(domain_name, Resolv::DNS::Resource::IN::TXT)
+  end
+
+  def parse(records)
+    records.flat_map(&:strings).flat_map(&:split)
+  end
+
+  def verification_enabled?
+    Gitlab::CurrentSettings.pages_domain_verification_enabled?
+  end
+
+  def notify(type)
+    return unless verification_enabled?
+
+    notification_service.public_send("pages_domain_#{type}", domain) # rubocop:disable GitlabSecurity/PublicSend
+  end
+end

app/views/admin/application_settings/_form.html.haml

@@ -237,6 +237,17 @@
       .col-sm-10
         = f.number_field :max_pages_size, class: 'form-control'
         .help-block 0 for unlimited
+    .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :pages_domain_verification_enabled do
+            = f.check_box :pages_domain_verification_enabled
+            Require users to prove ownership of custom domains
+        .help-block
+          Domain verification is an essential security measure for public GitLab
+          sites. Users are required to demonstrate they control a domain before
+          it is enabled
+          = link_to icon('question-circle'), help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
 
   %fieldset
     %legend Continuous Integration and Deployment

app/views/notify/pages_domain_disabled_email.html.haml

+%p
+  Following a verification check, your GitLab Pages custom domain has been
+  %strong disabled.
+  This means that your content is no longer visible at #{link_to @domain.url, @domain.url}
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  If this domain has been disabled in error, please follow
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  to verify and re-enable your domain.
+%p
+  If you no longer wish to use this domain with GitLab Pages, please remove it
+  from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_disabled_email.text.haml

+Following a verification check, your GitLab Pages custom domain has been
+**disabled**. This means that your content is no longer visible at #{@domain.url}
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+If this domain has been disabled in error, please follow these instructions
+to verify and re-enable your domain:
+
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+
+If you no longer wish to use this domain with GitLab Pages, please remove it
+from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_enabled_email.html.haml

+%p
+  Following a verification check, your GitLab Pages custom domain has been
+  enabled. You should now be able to view your content at #{link_to @domain.url, @domain.url}
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.

app/views/notify/pages_domain_enabled_email.text.haml

+Following a verification check, your GitLab Pages custom domain has been
+enabled. You should now be able to view your content at #{@domain.url}
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.

app/views/notify/pages_domain_verification_failed_email.html.haml

+%p
+  Verification has failed for one of your GitLab Pages custom domains!
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  Unless you take action, it will be disabled on
+  %strong= @domain.enabled_until.strftime('%F %T.')
+  Until then, you can view your content at #{link_to @domain.url, @domain.url}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.
+%p
+  If you no longer wish to use this domain with GitLab Pages, please remove it
+  from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_verification_failed_email.text.haml

+Verification has failed for one of your GitLab Pages custom domains!
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+Unless you take action, it will be disabled on *#{@domain.enabled_until.strftime('%F %T')}*.
+Until then, you can view your content at #{@domain.url}
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.
+
+If you no longer wish to use this domain with GitLab Pages, please remove it
+from your GitLab project and delete any related DNS records.

app/views/notify/pages_domain_verification_succeeded_email.html.haml

+%p
+  One of your GitLab Pages custom domains has been successfully verified!
+%p
+  Project: #{link_to @project.human_name, project_url(@project)}
+%p
+  Domain: #{link_to @domain.domain, project_pages_domain_url(@project, @domain)}
+%p
+  This is a notification. No action is required on your part. You can view your
+  content at #{link_to @domain.url, @domain.url}
+%p
+  Please visit
+  = link_to 'these instructions', help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+  for more information about custom domain verification.

app/views/notify/pages_domain_verification_succeeded_email.text.haml

+One of your GitLab Pages custom domains has been successfully verified!
+
+Project: #{@project.human_name} (#{project_url(@project)})
+Domain:  #{@domain.domain} (#{project_pages_domain_url(@project, @domain)})
+
+No action is required on your part. You can view your content at #{@domain.url}
+
+Please visit
+= help_page_url('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+for more information about custom domain verification.

app/views/projects/pages/_list.html.haml

@@ -3,15 +3,26 @@
     .panel-heading
       Domains (#{@domains.count})
     %ul.well-list
+      - verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled?
       - @domains.each do |domain|
         %li
           .pull-right
             = link_to 'Details', project_pages_domain_path(@project, domain), class: "btn btn-sm btn-grouped"
             = link_to 'Remove', project_pages_domain_path(@project, domain), data: { confirm: 'Are you sure?'}, method: :delete, class: "btn btn-remove btn-sm btn-grouped"
           .clearfix
-            %span= link_to domain.domain, domain.url
+            - if verification_enabled
+              - tooltip, status = domain.unverified? ? ['Unverified', 'failed'] : ['Verified', 'success']
+              = link_to domain.url, title: tooltip, class: 'has-tooltip' do
+                = sprite_icon("status_#{status}", size: 16, css_class: "has-tooltip ci-status-icon ci-status-icon-#{status}")
+                = domain.domain
+            - else
+              = link_to domain.domain, domain.url
           %p
             - if domain.subject
               %span.label.label-gray Certificate: #{domain.subject}
             - if domain.expired?
               %span.label.label-danger Expired
+        - if verification_enabled && domain.unverified?
+          %li.warning-row
+            #{domain.domain} is not verified. To learn how to verify ownership, visit your
+            = link_to 'domain details', project_pages_domain_path(@project, domain)

app/views/projects/pages_domains/show.html.haml

 - page_title "#{@domain.domain}", 'Pages Domains'
+- verification_enabled = Gitlab::CurrentSettings.pages_domain_verification_enabled?
+- if verification_enabled && @domain.unverified?
+  %p.alert.alert-warning
+    %strong
+      This domain is not verified. You will need to verify ownership before
+      access is enabled.
 
 %h3.page-title
   Pages Domain
@@ -15,9 +21,26 @@
         DNS
       %td
         %p
-          To access the domain create a new DNS record:
+          To access this domain create a new DNS record:
         %pre
           #{@domain.domain} CNAME #{@domain.project.pages_subdomain}.#{Settings.pages.host}.
+    - if verification_enabled
+      %tr
+        %td
+          Verification status
+        %td
+          %p
+            - help_link = help_page_path('user/project/pages/getting_started_part_three.md', anchor: 'dns-txt-record')
+            To #{link_to 'verify ownership', help_link} of your domain, create
+            this DNS record:
+          %pre
+            #{@domain.verification_domain} TXT #{@domain.keyed_verification_code}
+          %p
+            - if @domain.verified?
+              #{@domain.domain} has been successfully verified.
+            - else
+              = button_to 'Verify ownership', verify_project_pages_domain_path(@project, @domain), class: 'btn btn-save btn-sm'
+
     %tr
       %td
         Certificate

app/workers/all_queues.yml

@@ -3,6 +3,7 @@
 - cronjob:expire_build_artifacts
 - cronjob:gitlab_usage_ping
 - cronjob:import_export_project_cleanup
+- cronjob:pages_domain_verification_cron
 - cronjob:pipeline_schedule
 - cronjob:prune_old_events
 - cronjob:remove_expired_group_links
@@ -82,6 +83,7 @@
 - new_merge_request
 - new_note
 - pages
+- pages_domain_verification
 - post_receive
 - process_commit
 - project_cache

app/workers/pages_domain_verification_cron_worker.rb

+class PagesDomainVerificationCronWorker
+  include ApplicationWorker
+  include CronjobQueue
+
+  def perform
+    PagesDomain.needs_verification.find_each do |domain|
+      PagesDomainVerificationWorker.perform_async(domain.id)
+    end
+  end
+end

app/workers/pages_domain_verification_worker.rb

+class PagesDomainVerificationWorker
+  include ApplicationWorker
+
+  def perform(domain_id)
+    domain = PagesDomain.find_by(id: domain_id)
+
+    return unless domain
+
+    VerifyPagesDomainService.new(domain).execute
+  end
+end

changelogs/unreleased/29497-pages-custom-domain-dns-verification.yml

+---
+title: Add verification for GitLab Pages custom domains
+merge_request:
+author:
+type: security

config/gitlab.yml.example

@@ -214,6 +214,10 @@ production: &base
     repository_archive_cache_worker:
       cron: "0 * * * *"
 
+    # Verify custom GitLab Pages domains
+    pages_domain_verification_cron_worker:
+      cron: "*/15 * * * *"
+
   registry:
     # enabled: true
     # host: registry.example.com

config/initializers/1_settings.rb

@@ -427,6 +427,10 @@ Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *'
 Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker'
 
+Settings.cron_jobs['pages_domain_verification_cron_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['pages_domain_verification_cron_worker']['cron'] ||= '*/15 * * * *'
+Settings.cron_jobs['pages_domain_verification_cron_worker']['job_class'] = 'PagesDomainVerificationCronWorker'
+
 #
 # GitLab Shell
 #

config/routes/project.rb

@@ -55,7 +55,11 @@ constraints(ProjectUrlConstrainer.new) do
       end
 
       resource :pages, only: [:show, :destroy] do
-        resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} }
+        resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} } do
+          member do
+            post :verify
+          end
+        end
       end
 
       resources :snippets, concerns: :awardable, constraints: { id: /\d+/ } do

config/sidekiq_queues.yml

@@ -67,3 +67,4 @@
   - [gcp_cluster, 1]
   - [project_migrate_hashed_storage, 1]
   - [storage_migrator, 1]
+  - [pages_domain_verification, 1]

db/migrate/20180216120000_add_pages_domain_verification.rb

+class AddPagesDomainVerification < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :pages_domains, :verified_at, :datetime_with_timezone
+    add_column :pages_domains, :verification_code, :string
+  end
+end

db/migrate/20180216120010_add_pages_domain_verified_at_index.rb

+class AddPagesDomainVerifiedAtIndex < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :pages_domains, :verified_at
+  end
+
+  def down
+    remove_concurrent_index :pages_domains, :verified_at
+  end
+end

db/migrate/20180216120020_allow_domain_verification_to_be_disabled.rb

+class AllowDomainVerificationToBeDisabled < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    add_column :application_settings, :pages_domain_verification_enabled, :boolean, default: true, null: false
+  end
+end

db/migrate/20180216120030_add_pages_domain_enabled_until.rb

+class AddPagesDomainEnabledUntil < ActiveRecord::Migration
+  DOWNTIME = false