Commit d0e3e823 authored by Kamil Trzcinski's avatar Kamil Trzcinski

Implement Build Artifacts

- Offloads uploading to GitLab Workhorse
- Use /authorize request for fast uploading
- Added backup recipes for artifacts
- Support download acceleration using X-Sendfile
parent 354b69dd
......@@ -37,6 +37,9 @@ nohup.out
public/assets/
public/uploads.*
public/uploads/
shared/artifacts/
shared/tmp/artifacts-uploads/
shared/tmp/artifacts-cache/
rails_best_practices_output.html
/tags
tmp/
......
......@@ -95,6 +95,7 @@ v 8.1.0 (unreleased)
- Show CI status on Your projects page and Starred projects page
- Remove "Continuous Integration" page from dashboard
- Add notes and SSL verification entries to hook APIs (Ben Boeckel)
- Added build artifacts
- Fix grammar in admin area "labels" .nothing-here-block when no labels exist.
- Move CI runners page to project settings area
- Move CI variables page to project settings area
......
......@@ -54,7 +54,7 @@ gem 'gollum-lib', '~> 4.0.2'
gem "github-linguist", "~> 4.7.0", require: "linguist"
# API
gem 'grape', '~> 0.6.1'
gem 'grape', '~> 0.13.0'
gem 'grape-entity', '~> 0.4.2'
gem 'rack-cors', '~> 0.4.0', require: 'rack/cors'
......
......@@ -306,10 +306,10 @@ GEM
gon (5.0.4)
actionpack (>= 2.3.0)
json
grape (0.6.1)
grape (0.13.0)
activesupport
builder
hashie (>= 1.2.0)
hashie (>= 2.1.0)
multi_json (>= 1.3.2)
multi_xml (>= 0.5.2)
rack (>= 1.3.0)
......@@ -829,7 +829,7 @@ DEPENDENCIES
gitlab_omniauth-ldap (~> 1.2.1)
gollum-lib (~> 4.0.2)
gon (~> 5.0.0)
grape (~> 0.6.1)
grape (~> 0.13.0)
grape-entity (~> 0.4.2)
haml-rails (~> 0.9.0)
hipchat (~> 1.5.0)
......
......@@ -58,6 +58,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:admin_notification_email,
:user_oauth_applications,
:shared_runners_enabled,
:max_artifacts_size,
restricted_visibility_levels: [],
import_sources: []
)
......
......@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController
before_action :build, except: [:index, :cancel_all]
before_action :authorize_manage_builds!, except: [:index, :show, :status]
before_action :authorize_download_build_artifacts!, only: [:download]
layout "project"
......@@ -51,6 +52,18 @@ class Projects::BuildsController < Projects::ApplicationController
redirect_to build_path(build)
end
def download
unless artifacts_file.file_storage?
return redirect_to artifacts_file.url
end
unless artifacts_file.exists?
return not_found!
end
send_file artifacts_file.path, disposition: 'attachment'
end
def status
render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha)
end
......@@ -67,6 +80,10 @@ class Projects::BuildsController < Projects::ApplicationController
@build ||= ci_project.builds.unscoped.find_by!(id: params[:id])
end
def artifacts_file
build.artifacts_file
end
def build_path(build)
namespace_project_build_path(build.gl_project.namespace, build.gl_project, build)
end
......@@ -76,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController
return page_404
end
end
def authorize_download_build_artifacts!
unless can?(current_user, :download_build_artifacts, @project)
if current_user.nil?
return authenticate_user!
else
return render_404
end
end
end
end
......@@ -154,6 +154,7 @@ class Ability
:create_merge_request,
:create_wiki,
:manage_builds,
:download_build_artifacts,
:push_code
]
end
......
......@@ -89,6 +89,7 @@ class ApplicationSetting < ActiveRecord::Base
restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Settings.gitlab_ci['max_artifacts_size'],
)
end
......
......@@ -39,6 +39,8 @@ module Ci
scope :ignore_failures, ->() { where(allow_failure: false) }
scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
mount_uploader :artifacts_file, ArtifactUploader
acts_as_taggable
# To prevent db load megabytes of data from trace
......@@ -217,6 +219,14 @@ module Ci
"#{dir_to_trace}/#{id}.log"
end
def token
project.token
end
def valid_token? token
project.valid_token? token
end
def target_url
Gitlab::Application.routes.url_helpers.
namespace_project_build_url(gl_project.namespace, gl_project, self)
......@@ -248,6 +258,13 @@ module Ci
pending? && !any_runners_online?
end
def download_url
if artifacts_file.exists?
Gitlab::Application.routes.url_helpers.
download_namespace_project_build_path(gl_project.namespace, gl_project, self)
end
end
private
def yaml_variables
......
......@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base
def show_warning?
false
end
def download_url
nil
end
end
# encoding: utf-8
class ArtifactUploader < CarrierWave::Uploader::Base
storage :file
attr_accessor :build, :field
def self.artifacts_path
File.expand_path('shared/artifacts/', Rails.root)
end
def self.artifacts_upload_path
File.expand_path('shared/tmp/artifacts-uploads/', Rails.root)
end
def self.artifacts_cache_path
File.expand_path('shared/tmp/artifacts-cache/', Rails.root)
end
def initialize(build, field)
@build, @field = build, field
end
def artifacts_path
File.join(build.created_at.utc.strftime('%Y_%m'), build.project.id.to_s, build.id.to_s)
end
def store_dir
File.join(ArtifactUploader.artifacts_path, artifacts_path)
end
def cache_dir
File.join(ArtifactUploader.artifacts_cache_path, artifacts_path)
end
def file_storage?
self.class.storage == CarrierWave::Storage::File
end
def exists?
file.try(:exists?)
end
def move_to_cache
true
end
def move_to_store
true
end
end
......@@ -139,5 +139,10 @@
= f.check_box :shared_runners_enabled
Enable shared runners for a new projects
.form-group
= f.label :max_artifacts_size, 'Maximum artifacts size (MB)', class: 'control-label col-sm-2'
.col-sm-10
= f.number_field :max_artifacts_size, class: 'form-control'
.form-actions
= f.submit 'Save', class: 'btn btn-primary'
......@@ -87,6 +87,9 @@
Test coverage
%h1 #{@build.coverage}%
- if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url
.build-widget.center
= link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary'
.build-widget
%h4.title
......
......@@ -61,6 +61,9 @@
%td
.pull-right
- if current_user && can?(current_user, :download_build_artifacts, @project) && commit_status.download_url
= link_to commit_status.download_url, title: 'Download artifacts' do
%i.fa.fa-download
- if current_user && can?(current_user, :manage_builds, commit_status.gl_project)
- if commit_status.active?
- if commit_status.cancel_url
......
......@@ -186,6 +186,7 @@ Settings.gitlab_ci['all_broken_builds'] = true if Settings.gitlab_ci['all_br
Settings.gitlab_ci['add_pusher'] = false if Settings.gitlab_ci['add_pusher'].nil?
Settings.gitlab_ci['url'] ||= Settings.send(:build_gitlab_ci_url)
Settings.gitlab_ci['builds_path'] = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root)
Settings.gitlab_ci['max_artifacts_size'] ||= 100
#
# Reply by email
......
......@@ -611,6 +611,7 @@ Gitlab::Application.routes.draw do
member do
get :status
post :cancel
get :download
post :retry
end
end
......
class AddArtifactsFileToBuilds < ActiveRecord::Migration
def change
add_column :ci_builds, :artifacts_file, :text
end
end
class AddMaxArtifactsSizeToApplicationSettings < ActiveRecord::Migration
def change
add_column :application_settings, :max_artifacts_size, :integer, default: 100, null: false
end
end
......@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20151105094515) do
ActiveRecord::Schema.define(version: 20151109100728) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
......@@ -48,6 +48,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
t.text "help_page_text"
t.string "admin_notification_email"
t.boolean "shared_runners_enabled", default: true, null: false
t.integer "max_artifacts_size", default: 100, null: false
end
create_table "audit_events", force: true do |t|
......@@ -108,6 +109,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
t.string "type"
t.string "target_url"
t.string "description"
t.text "artifacts_file"
end
add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
......
......@@ -141,6 +141,7 @@ job_name:
| tags | optional | Defines a list of tags which are used to select runner |
| allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status |
| when | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` |
| artifacts | optional | Define list build artifacts |
### script
`script` is a shell script which is executed by runner. The shell script is prepended with `before_script`.
......@@ -258,6 +259,20 @@ The above script will:
1. Execute `cleanup_build` only when the `build` failed,
2. Always execute `cleanup` as the last step in pipeline.
### artifacts
`artifacts` is used to specify list of files and directories which should be attached to build after success.
```
artifacts:
- binaries/
- .config
```
The above definition will archive all files in `binaries/` and `.config`.
The artifacts will be send after the build success to GitLab and will be accessible in GitLab interface to download.
This feature requires GitLab Runner v 0.7.0.
## Validate the .gitlab-ci.yml
Each instance of GitLab CI has an embedded debug tool called Lint.
You can find the link to the Lint in the project's settings page or use short url `/lint`.
......
......@@ -246,6 +246,11 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da
# Change the permissions of the directory where CI build traces are stored
sudo chmod -R u+rwX builds/
# Change the permissions of the directory where CI artifacts are stored
sudo chmod -R u+rwX shared/artifacts/
sudo chmod -R u+rwX shared/tmp/artifacts-uploads/
sudo chmod -R u+rwX shared/tmp/artifacts-cache/
# Copy the example Unicorn config
sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb
......
......@@ -29,7 +29,8 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
```
Also you can choose what should be backed up by adding environment variable SKIP. Available options: db,
uploads (attachments), repositories, builds(CI build output logs). Use a comma to specify several options at the same time.
uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts).
Use a comma to specify several options at the same time.
```
sudo gitlab-rake gitlab:backup:create SKIP=db,uploads
......
......@@ -133,6 +133,12 @@ module API
authorize! :admin_project, user_project
end
def require_gitlab_workhorse!
unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present?
forbidden!('Request should be executed via GitLab Workhorse')
end
end
def can?(object, action, subject)
abilities.allowed?(object, action, subject)
end
......@@ -234,6 +240,10 @@ module API
render_api_error!(message || '409 Conflict', 409)
end
def file_to_large!
render_api_error!('413 Request Entity Too Large', 413)
end
def render_validation_error!(model)
if model.errors.any?
render_api_error!(model.errors.messages || '400 Bad Request', 400)
......@@ -282,6 +292,40 @@ module API
end
end
# file helpers
def uploaded_file!(uploads_path)
required_attributes! [:file]
# sanitize file paths
# this requires for all paths to exist
uploads_path = File.realpath(uploads_path)
file_path = File.realpath(params[:file])
bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
UploadedFile.new(
file_path,
params[:filename],
params[:filetype] || 'application/octet-stream',
)
end
def present_file!(path, filename, content_type = 'application/octet-stream')
filename ||= File.basename(path)
header['Content-Disposition'] = "attachment; filename=#{filename}"
header['Content-Transfer-Encoding'] = 'binary'
content_type content_type
# Support download acceleration
case headers['X-Sendfile-Type']
when 'X-Sendfile'
header['X-Sendfile'] = path
body
else
file FileStreamer.new(path)
end
end
private
def add_pagination_headers(paginated, per_page)
......
require 'backup/files'
module Backup
class Artifacts < Files
def initialize
super('artifacts', ArtifactUploader.artifacts_path)
end
def create_files_dir
Dir.mkdir(app_files_dir, 0700)
end
end
end
......@@ -150,7 +150,7 @@ module Backup
private
def backup_contents
folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"]
folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"]
end
def folders_to_backup
......
......@@ -27,6 +27,7 @@ module Ci
helpers Helpers
helpers ::API::Helpers
helpers Gitlab::CurrentSettings
mount Builds
mount Commits
......
......@@ -47,6 +47,108 @@ module Ci
build.drop
end
end
# Authorize artifacts uploading for build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# size (optional) - the size of uploaded file
# Example Request:
# POST /builds/:id/artifacts/authorize
post ":id/artifacts/authorize" do
require_gitlab_workhorse!
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
forbidden!('build is not running') unless build.running?
if params[:filesize]
file_size = params[:filesize].to_i
file_to_large! unless file_size < max_artifacts_size
end
status 200
{ temp_path: ArtifactUploader.artifacts_upload_path }
end
# Upload artifacts to build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# Headers:
# Content-Type - File content type
# Content-Disposition - File media type and real name
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Body:
# The file content
#
# Parameters (set by GitLab Workhorse):
# file - path to locally stored body (generated by Workhorse)
# filename - real filename as send in Content-Disposition
# filetype - real content type as send in Content-Type
# filesize - real file size as send in Content-Length
# Example Request:
# POST /builds/:id/artifacts
post ":id/artifacts" do
require_gitlab_workhorse!
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
forbidden!('build is not running') unless build.running?
file = uploaded_file!(ArtifactUploader.artifacts_upload_path)
file_to_large! unless file.size < max_artifacts_size
if build.update_attributes(artifacts_file: file)
present build, with: Entities::Build
else
render_validation_error!(build)
end
end
# Download the artifacts file from build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Example Request:
# GET /builds/:id/artifacts
get ":id/artifacts" do
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
artifacts_file = build.artifacts_file
unless artifacts_file.file_storage?
return redirect_to build.artifacts_file.url
end
unless artifacts_file.exists?
not_found!
end
present_file!(artifacts_file.path, artifacts_file.filename)
end
# Remove the artifacts file from build
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Example Request:
# DELETE /builds/:id/artifacts
delete ":id/artifacts" do
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
build.remove_artifacts_file!
end
end
end
end
......
......@@ -11,10 +11,16 @@ module Ci
expose :builds
end
class ArtifactFile < Grape::Entity
expose :filename, :size
end
class Build < Grape::Entity
expose :id, :commands, :ref, :sha, :status, :project_id, :repo_url,
:before_sha, :allow_git_fetch, :project_name
expose :name, :token, :stage
expose :options do |model|
model.options
end
......@@ -24,6 +30,7 @@ module Ci
end
expose :variables
expose :artifacts_file, using: ArtifactFile
end
class Runner < Grape::Entity
......
module Ci
module API
module Helpers
BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN"
BUILD_TOKEN_PARAM = :token
UPDATE_RUNNER_EVERY = 60
def authenticate_runners!
......@@ -15,6 +17,11 @@ module Ci
forbidden! unless project.valid_token?(params[:project_token])
end
def authenticate_build_token!(build)
token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s
forbidden! unless token && build.valid_token?(token)
end
def update_runner_last_contact
# Use a random threshold to prevent beating DB updates
contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY)
......@@ -32,6 +39,10 @@ module Ci
info = attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"])
current_runner.update(info)
end
def max_artifacts_size
current_application_settings.max_artifacts_size.megabytes.to_i
end
end
end
end
......@@ -5,7 +5,7 @@ module Ci
DEFAULT_STAGES = %w(build test deploy)
DEFAULT_STAGE = 'test'
ALLOWED_YAML_KEYS = [:before_script, :image, :services, :types, :stages, :variables]
ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when]
ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts]
attr_reader :before_script, :image, :services, :variables, :path
......@@ -77,7 +77,8 @@ module Ci
when: job[:when] || 'on_success',
options: {
image: job[:image] || @image,
services: job[:services] || @services
services: job[:services] || @services,
artifacts: job[:artifacts]
}.compact
}
end
......@@ -159,6 +160,10 @@ module Ci
raise ValidationError, "#{name} job: except parameter should be an array of strings"
end
if job[:artifacts] && !validate_array_of_strings(job[:artifacts])
raise ValidationError, "#{name}: artifacts parameter should be an array of strings"
end
if job[:allow_failure] && !job[:allow_failure].in?([true, false])
raise ValidationError, "#{name} job: allow_failure parameter should be an boolean"
end
......
class FileStreamer #:nodoc:
attr_reader :to_path
def initialize(path)
@to_path = path
end
# Stream the file's contents if Rack::Sendfile isn't present.
def each
File.open(to_path, 'rb') do |file|
while chunk = file.read(16384)
yield chunk
end
end
end
end
......@@ -25,6 +25,7 @@ module Gitlab
session_expire_delay: Settings.gitlab['session_expire_delay'],
import_sources: Settings.gitlab['import_sources'],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Ci::Settings.gitlab_ci['max_artifacts_size'],
)
end
......
......@@ -131,6 +131,22 @@ server {
return 418;
}
# Build artifacts should be submitted to this location
location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
error_page 418 = @gitlab-git-http-server;
return 418;
}
# Build artifacts should be submitted to this location
location ~ /ci/api/v1/builds/[0-9]+/artifacts {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
error_page 418 = @gitlab-git-http-server;
return 418;
}
location @gitlab-workhorse {
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
......
......@@ -178,6 +178,22 @@ server {
return 418;
}
# Build artifacts should be submitted to this location
location ~ ^/[\w\.-]+/[\w\.-]+/builds/download {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
error_page 418 = @gitlab-git-http-server;
return 418;
}
# Build artifacts should be submitted to this location
location ~ /ci/api/v1/builds/[0-9]+/artifacts {
client_max_body_size 0;
# 'Error' 418 is a hack to re-use the @gitlab-git-http-server block
error_page 418 = @gitlab-git-http-server;
return 418;
}
location @gitlab-workhorse {
## If you use HTTPS make sure you disable gzip compression
## to be safe against BREACH attack.
......
......@@ -12,6 +12,7 @@ namespace :gitlab do
Rake::Task["gitlab:backup:repo:create"].invoke