Commit d0e3e823 authored by Kamil Trzcinski's avatar Kamil Trzcinski

Implement Build Artifacts

- Offloads uploading to GitLab Workhorse
- Use /authorize request for fast uploading
- Added backup recipes for artifacts
- Support download acceleration using X-Sendfile
parent 354b69dd
...@@ -37,6 +37,9 @@ nohup.out ...@@ -37,6 +37,9 @@ nohup.out
public/assets/ public/assets/
public/uploads.* public/uploads.*
public/uploads/ public/uploads/
shared/artifacts/
shared/tmp/artifacts-uploads/
shared/tmp/artifacts-cache/
rails_best_practices_output.html rails_best_practices_output.html
/tags /tags
tmp/ tmp/
......
...@@ -95,6 +95,7 @@ v 8.1.0 (unreleased) ...@@ -95,6 +95,7 @@ v 8.1.0 (unreleased)
- Show CI status on Your projects page and Starred projects page - Show CI status on Your projects page and Starred projects page
- Remove "Continuous Integration" page from dashboard - Remove "Continuous Integration" page from dashboard
- Add notes and SSL verification entries to hook APIs (Ben Boeckel) - Add notes and SSL verification entries to hook APIs (Ben Boeckel)
- Added build artifacts
- Fix grammar in admin area "labels" .nothing-here-block when no labels exist. - Fix grammar in admin area "labels" .nothing-here-block when no labels exist.
- Move CI runners page to project settings area - Move CI runners page to project settings area
- Move CI variables page to project settings area - Move CI variables page to project settings area
......
...@@ -54,7 +54,7 @@ gem 'gollum-lib', '~> 4.0.2' ...@@ -54,7 +54,7 @@ gem 'gollum-lib', '~> 4.0.2'
gem "github-linguist", "~> 4.7.0", require: "linguist" gem "github-linguist", "~> 4.7.0", require: "linguist"
# API # API
gem 'grape', '~> 0.6.1' gem 'grape', '~> 0.13.0'
gem 'grape-entity', '~> 0.4.2' gem 'grape-entity', '~> 0.4.2'
gem 'rack-cors', '~> 0.4.0', require: 'rack/cors' gem 'rack-cors', '~> 0.4.0', require: 'rack/cors'
......
...@@ -306,10 +306,10 @@ GEM ...@@ -306,10 +306,10 @@ GEM
gon (5.0.4) gon (5.0.4)
actionpack (>= 2.3.0) actionpack (>= 2.3.0)
json json
grape (0.6.1) grape (0.13.0)
activesupport activesupport
builder builder
hashie (>= 1.2.0) hashie (>= 2.1.0)
multi_json (>= 1.3.2) multi_json (>= 1.3.2)
multi_xml (>= 0.5.2) multi_xml (>= 0.5.2)
rack (>= 1.3.0) rack (>= 1.3.0)
...@@ -829,7 +829,7 @@ DEPENDENCIES ...@@ -829,7 +829,7 @@ DEPENDENCIES
gitlab_omniauth-ldap (~> 1.2.1) gitlab_omniauth-ldap (~> 1.2.1)
gollum-lib (~> 4.0.2) gollum-lib (~> 4.0.2)
gon (~> 5.0.0) gon (~> 5.0.0)
grape (~> 0.6.1) grape (~> 0.13.0)
grape-entity (~> 0.4.2) grape-entity (~> 0.4.2)
haml-rails (~> 0.9.0) haml-rails (~> 0.9.0)
hipchat (~> 1.5.0) hipchat (~> 1.5.0)
......
...@@ -58,6 +58,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -58,6 +58,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:admin_notification_email, :admin_notification_email,
:user_oauth_applications, :user_oauth_applications,
:shared_runners_enabled, :shared_runners_enabled,
:max_artifacts_size,
restricted_visibility_levels: [], restricted_visibility_levels: [],
import_sources: [] import_sources: []
) )
......
...@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -3,6 +3,7 @@ class Projects::BuildsController < Projects::ApplicationController
before_action :build, except: [:index, :cancel_all] before_action :build, except: [:index, :cancel_all]
before_action :authorize_manage_builds!, except: [:index, :show, :status] before_action :authorize_manage_builds!, except: [:index, :show, :status]
before_action :authorize_download_build_artifacts!, only: [:download]
layout "project" layout "project"
...@@ -51,6 +52,18 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -51,6 +52,18 @@ class Projects::BuildsController < Projects::ApplicationController
redirect_to build_path(build) redirect_to build_path(build)
end end
def download
unless artifacts_file.file_storage?
return redirect_to artifacts_file.url
end
unless artifacts_file.exists?
return not_found!
end
send_file artifacts_file.path, disposition: 'attachment'
end
def status def status
render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha) render json: @build.to_json(only: [:status, :id, :sha, :coverage], methods: :sha)
end end
...@@ -67,6 +80,10 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -67,6 +80,10 @@ class Projects::BuildsController < Projects::ApplicationController
@build ||= ci_project.builds.unscoped.find_by!(id: params[:id]) @build ||= ci_project.builds.unscoped.find_by!(id: params[:id])
end end
def artifacts_file
build.artifacts_file
end
def build_path(build) def build_path(build)
namespace_project_build_path(build.gl_project.namespace, build.gl_project, build) namespace_project_build_path(build.gl_project.namespace, build.gl_project, build)
end end
...@@ -76,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController ...@@ -76,4 +93,14 @@ class Projects::BuildsController < Projects::ApplicationController
return page_404 return page_404
end end
end end
def authorize_download_build_artifacts!
unless can?(current_user, :download_build_artifacts, @project)
if current_user.nil?
return authenticate_user!
else
return render_404
end
end
end
end end
...@@ -154,6 +154,7 @@ class Ability ...@@ -154,6 +154,7 @@ class Ability
:create_merge_request, :create_merge_request,
:create_wiki, :create_wiki,
:manage_builds, :manage_builds,
:download_build_artifacts,
:push_code :push_code
] ]
end end
......
...@@ -89,6 +89,7 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -89,6 +89,7 @@ class ApplicationSetting < ActiveRecord::Base
restricted_signup_domains: Settings.gitlab['restricted_signup_domains'], restricted_signup_domains: Settings.gitlab['restricted_signup_domains'],
import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'], import_sources: ['github','bitbucket','gitlab','gitorious','google_code','fogbugz','git'],
shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'],
max_artifacts_size: Settings.gitlab_ci['max_artifacts_size'],
) )
end end
......
...@@ -39,6 +39,8 @@ module Ci ...@@ -39,6 +39,8 @@ module Ci
scope :ignore_failures, ->() { where(allow_failure: false) } scope :ignore_failures, ->() { where(allow_failure: false) }
scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) } scope :similar, ->(build) { where(ref: build.ref, tag: build.tag, trigger_request_id: build.trigger_request_id) }
mount_uploader :artifacts_file, ArtifactUploader
acts_as_taggable acts_as_taggable
# To prevent db load megabytes of data from trace # To prevent db load megabytes of data from trace
...@@ -217,6 +219,14 @@ module Ci ...@@ -217,6 +219,14 @@ module Ci
"#{dir_to_trace}/#{id}.log" "#{dir_to_trace}/#{id}.log"
end end
def token
project.token
end
def valid_token? token
project.valid_token? token
end
def target_url def target_url
Gitlab::Application.routes.url_helpers. Gitlab::Application.routes.url_helpers.
namespace_project_build_url(gl_project.namespace, gl_project, self) namespace_project_build_url(gl_project.namespace, gl_project, self)
...@@ -248,6 +258,13 @@ module Ci ...@@ -248,6 +258,13 @@ module Ci
pending? && !any_runners_online? pending? && !any_runners_online?
end end
def download_url
if artifacts_file.exists?
Gitlab::Application.routes.url_helpers.
download_namespace_project_build_path(gl_project.namespace, gl_project, self)
end
end
private private
def yaml_variables def yaml_variables
......
...@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base ...@@ -92,4 +92,8 @@ class CommitStatus < ActiveRecord::Base
def show_warning? def show_warning?
false false
end end
def download_url
nil
end
end end
# encoding: utf-8
class ArtifactUploader < CarrierWave::Uploader::Base
storage :file
attr_accessor :build, :field
def self.artifacts_path
File.expand_path('shared/artifacts/', Rails.root)
end
def self.artifacts_upload_path
File.expand_path('shared/tmp/artifacts-uploads/', Rails.root)
end
def self.artifacts_cache_path
File.expand_path('shared/tmp/artifacts-cache/', Rails.root)
end
def initialize(build, field)
@build, @field = build, field
end
def artifacts_path
File.join(build.created_at.utc.strftime('%Y_%m'), build.project.id.to_s, build.id.to_s)
end
def store_dir
File.join(ArtifactUploader.artifacts_path, artifacts_path)
end
def cache_dir
File.join(ArtifactUploader.artifacts_cache_path, artifacts_path)
end
def file_storage?
self.class.storage == CarrierWave::Storage::File
end
def exists?
file.try(:exists?)
end
def move_to_cache
true
end
def move_to_store
true
end
end
...@@ -139,5 +139,10 @@ ...@@ -139,5 +139,10 @@
= f.check_box :shared_runners_enabled = f.check_box :shared_runners_enabled
Enable shared runners for a new projects Enable shared runners for a new projects
.form-group
= f.label :max_artifacts_size, 'Maximum artifacts size (MB)', class: 'control-label col-sm-2'
.col-sm-10
= f.number_field :max_artifacts_size, class: 'form-control'
.form-actions .form-actions
= f.submit 'Save', class: 'btn btn-primary' = f.submit 'Save', class: 'btn btn-primary'
...@@ -87,6 +87,9 @@ ...@@ -87,6 +87,9 @@
Test coverage Test coverage
%h1 #{@build.coverage}% %h1 #{@build.coverage}%
- if current_user && can?(current_user, :download_build_artifacts, @project) && @build.download_url
.build-widget.center
= link_to "Download artifacts", @build.download_url, class: 'btn btn-sm btn-primary'
.build-widget .build-widget
%h4.title %h4.title
......
...@@ -61,6 +61,9 @@ ...@@ -61,6 +61,9 @@
%td %td
.pull-right .pull-right
- if current_user && can?(current_user, :download_build_artifacts, @project) && commit_status.download_url
= link_to commit_status.download_url, title: 'Download artifacts' do
%i.fa.fa-download
- if current_user && can?(current_user, :manage_builds, commit_status.gl_project) - if current_user && can?(current_user, :manage_builds, commit_status.gl_project)
- if commit_status.active? - if commit_status.active?
- if commit_status.cancel_url - if commit_status.cancel_url
......
...@@ -186,6 +186,7 @@ Settings.gitlab_ci['all_broken_builds'] = true if Settings.gitlab_ci['all_br ...@@ -186,6 +186,7 @@ Settings.gitlab_ci['all_broken_builds'] = true if Settings.gitlab_ci['all_br
Settings.gitlab_ci['add_pusher'] = false if Settings.gitlab_ci['add_pusher'].nil? Settings.gitlab_ci['add_pusher'] = false if Settings.gitlab_ci['add_pusher'].nil?
Settings.gitlab_ci['url'] ||= Settings.send(:build_gitlab_ci_url) Settings.gitlab_ci['url'] ||= Settings.send(:build_gitlab_ci_url)
Settings.gitlab_ci['builds_path'] = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root) Settings.gitlab_ci['builds_path'] = File.expand_path(Settings.gitlab_ci['builds_path'] || "builds/", Rails.root)
Settings.gitlab_ci['max_artifacts_size'] ||= 100
# #
# Reply by email # Reply by email
......
...@@ -611,6 +611,7 @@ Gitlab::Application.routes.draw do ...@@ -611,6 +611,7 @@ Gitlab::Application.routes.draw do
member do member do
get :status get :status
post :cancel post :cancel
get :download
post :retry post :retry
end end
end end
......
class AddArtifactsFileToBuilds < ActiveRecord::Migration
def change
add_column :ci_builds, :artifacts_file, :text
end
end
class AddMaxArtifactsSizeToApplicationSettings < ActiveRecord::Migration
def change
add_column :application_settings, :max_artifacts_size, :integer, default: 100, null: false
end
end
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20151105094515) do ActiveRecord::Schema.define(version: 20151109100728) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
...@@ -48,6 +48,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do ...@@ -48,6 +48,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
t.text "help_page_text" t.text "help_page_text"
t.string "admin_notification_email" t.string "admin_notification_email"
t.boolean "shared_runners_enabled", default: true, null: false t.boolean "shared_runners_enabled", default: true, null: false
t.integer "max_artifacts_size", default: 100, null: false
end end
create_table "audit_events", force: true do |t| create_table "audit_events", force: true do |t|
...@@ -108,6 +109,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do ...@@ -108,6 +109,7 @@ ActiveRecord::Schema.define(version: 20151105094515) do
t.string "type" t.string "type"
t.string "target_url" t.string "target_url"
t.string "description" t.string "description"
t.text "artifacts_file"
end end
add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree add_index "ci_builds", ["commit_id", "stage_idx", "created_at"], name: "index_ci_builds_on_commit_id_and_stage_idx_and_created_at", using: :btree
......
...@@ -141,6 +141,7 @@ job_name: ...@@ -141,6 +141,7 @@ job_name:
| tags | optional | Defines a list of tags which are used to select runner | | tags | optional | Defines a list of tags which are used to select runner |
| allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status | | allow_failure | optional | Allow build to fail. Failed build doesn't contribute to commit status |
| when | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` | | when | optional | Define when to run build. Can be `on_success`, `on_failure` or `always` |
| artifacts | optional | Define list build artifacts |
### script ### script
`script` is a shell script which is executed by runner. The shell script is prepended with `before_script`. `script` is a shell script which is executed by runner. The shell script is prepended with `before_script`.
...@@ -258,6 +259,20 @@ The above script will: ...@@ -258,6 +259,20 @@ The above script will:
1. Execute `cleanup_build` only when the `build` failed, 1. Execute `cleanup_build` only when the `build` failed,
2. Always execute `cleanup` as the last step in pipeline. 2. Always execute `cleanup` as the last step in pipeline.
### artifacts
`artifacts` is used to specify list of files and directories which should be attached to build after success.
```
artifacts:
- binaries/
- .config
```
The above definition will archive all files in `binaries/` and `.config`.
The artifacts will be send after the build success to GitLab and will be accessible in GitLab interface to download.
This feature requires GitLab Runner v 0.7.0.
## Validate the .gitlab-ci.yml ## Validate the .gitlab-ci.yml
Each instance of GitLab CI has an embedded debug tool called Lint. Each instance of GitLab CI has an embedded debug tool called Lint.
You can find the link to the Lint in the project's settings page or use short url `/lint`. You can find the link to the Lint in the project's settings page or use short url `/lint`.
......
...@@ -246,6 +246,11 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da ...@@ -246,6 +246,11 @@ We recommend using a PostgreSQL database. For MySQL check [MySQL setup guide](da
# Change the permissions of the directory where CI build traces are stored # Change the permissions of the directory where CI build traces are stored
sudo chmod -R u+rwX builds/ sudo chmod -R u+rwX builds/
# Change the permissions of the directory where CI artifacts are stored
sudo chmod -R u+rwX shared/artifacts/
sudo chmod -R u+rwX shared/tmp/artifacts-uploads/
sudo chmod -R u+rwX shared/tmp/artifacts-cache/
# Copy the example Unicorn config # Copy the example Unicorn config
sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb sudo -u git -H cp config/unicorn.rb.example config/unicorn.rb
......
...@@ -29,7 +29,8 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production ...@@ -29,7 +29,8 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production
``` ```
Also you can choose what should be backed up by adding environment variable SKIP. Available options: db, Also you can choose what should be backed up by adding environment variable SKIP. Available options: db,
uploads (attachments), repositories, builds(CI build output logs). Use a comma to specify several options at the same time. uploads (attachments), repositories, builds(CI build output logs), artifacts (CI build artifacts).
Use a comma to specify several options at the same time.
``` ```
sudo gitlab-rake gitlab:backup:create SKIP=db,uploads sudo gitlab-rake gitlab:backup:create SKIP=db,uploads
......
...@@ -133,6 +133,12 @@ module API ...@@ -133,6 +133,12 @@ module API
authorize! :admin_project, user_project authorize! :admin_project, user_project
end end
def require_gitlab_workhorse!
unless headers['Gitlab-Git-Http-Server'].present? || headers['GitLab-Git-HTTP-Server'].present?
forbidden!('Request should be executed via GitLab Workhorse')
end
end
def can?(object, action, subject) def can?(object, action, subject)
abilities.allowed?(object, action, subject) abilities.allowed?(object, action, subject)
end end
...@@ -234,6 +240,10 @@ module API ...@@ -234,6 +240,10 @@ module API
render_api_error!(message || '409 Conflict', 409) render_api_error!(message || '409 Conflict', 409)
end end
def file_to_large!
render_api_error!('413 Request Entity Too Large', 413)
end
def render_validation_error!(model) def render_validation_error!(model)
if model.errors.any? if model.errors.any?
render_api_error!(model.errors.messages || '400 Bad Request', 400) render_api_error!(model.errors.messages || '400 Bad Request', 400)
...@@ -282,6 +292,40 @@ module API ...@@ -282,6 +292,40 @@ module API
end end
end end
# file helpers
def uploaded_file!(uploads_path)
required_attributes! [:file]
# sanitize file paths
# this requires for all paths to exist
uploads_path = File.realpath(uploads_path)
file_path = File.realpath(params[:file])
bad_request!('Bad file path') unless file_path.start_with?(uploads_path)
UploadedFile.new(
file_path,
params[:filename],
params[:filetype] || 'application/octet-stream',
)
end
def present_file!(path, filename, content_type = 'application/octet-stream')
filename ||= File.basename(path)
header['Content-Disposition'] = "attachment; filename=#{filename}"
header['Content-Transfer-Encoding'] = 'binary'
content_type content_type
# Support download acceleration
case headers['X-Sendfile-Type']
when 'X-Sendfile'
header['X-Sendfile'] = path
body
else
file FileStreamer.new(path)
end
end
private private
def add_pagination_headers(paginated, per_page) def add_pagination_headers(paginated, per_page)
......
require 'backup/files'
module Backup
class Artifacts < Files
def initialize
super('artifacts', ArtifactUploader.artifacts_path)
end
def create_files_dir
Dir.mkdir(app_files_dir, 0700)
end
end
end
...@@ -150,7 +150,7 @@ module Backup ...@@ -150,7 +150,7 @@ module Backup
private private
def backup_contents def backup_contents
folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "artifacts.tar.gz", "backup_information.yml"]
end end
def folders_to_backup def folders_to_backup
......
...@@ -27,6 +27,7 @@ module Ci ...@@ -27,6 +27,7 @@ module Ci
helpers Helpers helpers Helpers
helpers ::API::Helpers helpers ::API::Helpers
helpers Gitlab::CurrentSettings
mount Builds mount Builds
mount Commits mount Commits
......
...@@ -47,6 +47,108 @@ module Ci ...@@ -47,6 +47,108 @@ module Ci
build.drop build.drop
end end
end end
# Authorize artifacts uploading for build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# size (optional) - the size of uploaded file
# Example Request:
# POST /builds/:id/artifacts/authorize
post ":id/artifacts/authorize" do
require_gitlab_workhorse!
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
forbidden!('build is not running') unless build.running?
if params[:filesize]
file_size = params[:filesize].to_i
file_to_large! unless file_size < max_artifacts_size
end
status 200
{ temp_path: ArtifactUploader.artifacts_upload_path }
end
# Upload artifacts to build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# Headers:
# Content-Type - File content type
# Content-Disposition - File media type and real name
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Body:
# The file content
#
# Parameters (set by GitLab Workhorse):
# file - path to locally stored body (generated by Workhorse)
# filename - real filename as send in Content-Disposition
# filetype - real content type as send in Content-Type
# filesize - real file size as send in Content-Length
# Example Request:
# POST /builds/:id/artifacts
post ":id/artifacts" do
require_gitlab_workhorse!
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
forbidden!('build is not running') unless build.running?
file = uploaded_file!(ArtifactUploader.artifacts_upload_path)
file_to_large! unless file.size < max_artifacts_size
if build.update_attributes(artifacts_file: file)
present build, with: Entities::Build
else
render_validation_error!(build)
end
end
# Download the artifacts file from build - Runners only
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Example Request:
# GET /builds/:id/artifacts
get ":id/artifacts" do
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
artifacts_file = build.artifacts_file
unless artifacts_file.file_storage?
return redirect_to build.artifacts_file.url
end
unless artifacts_file.exists?
not_found!
end
present_file!(artifacts_file.path, artifacts_file.filename)
end
# Remove the artifacts file from build
#
# Parameters:
# id (required) - The ID of a build
# token (required) - The build authorization token
# Headers:
# BUILD-TOKEN (required) - The build authorization token, the same as token
# Example Request:
# DELETE /builds/:id/artifacts
delete ":id/artifacts" do
build = Ci::Build.find_by_id(params[:id])
not_found! unless build
authenticate_build_token!(build)
build.remove_artifacts_file!
end
end end
end end
end end
......
...@@ -11,10 +11,16 @@ module Ci ...@@ -11,10 +11,16 @@ module Ci
expose :builds