Commit 0103d5be authored by Kamil Trzciński's avatar Kamil Trzciński

Add config_options|variables to BuildMetadata

These are data columns that store runtime configuration
of build needed to execute it on runner and within pipeline.

The definition of this data is that once used, and when no longer
needed (due to retry capability) they can be freely removed.

They use `jsonb` on PostgreSQL, and `text` on MySQL (due to lacking
support for json datatype on old enough version).
parent b647ad96
......@@ -8,10 +8,15 @@ module Ci
include ObjectStorage::BackgroundMove
include Presentable
include Importable
include IgnorableColumn
include Gitlab::Utils::StrongMemoize
include Deployable
include HasRef
BuildArchivedError = Class.new(StandardError)
ignore_column :commands
belongs_to :project, inverse_of: :builds
belongs_to :runner
belongs_to :trigger_request
......@@ -31,7 +36,7 @@ module Ci
has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
end
has_one :metadata, class_name: 'Ci::BuildMetadata'
has_one :metadata, class_name: 'Ci::BuildMetadata', autosave: true
has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build
accepts_nested_attributes_for :runner_session
......@@ -273,11 +278,14 @@ module Ci
# degenerated build is one that cannot be run by Runner
def degenerated?
self.options.nil?
self.options.blank?
end
def degenerate!
self.update!(options: nil, yaml_variables: nil, commands: nil)
Build.transaction do
self.update!(options: nil, yaml_variables: nil)
self.metadata&.destroy
end
end
def archived?
......@@ -624,11 +632,23 @@ module Ci
end
def when
read_attribute(:when) || build_attributes_from_config[:when] || 'on_success'
read_attribute(:when) || 'on_success'
end
def options
read_metadata_attribute(:options, :config_options, {})
end
def yaml_variables
read_attribute(:yaml_variables) || build_attributes_from_config[:yaml_variables] || []
read_metadata_attribute(:yaml_variables, :config_variables, [])
end
def options=(value)
write_metadata_attribute(:options, :config_options, value)
end
def yaml_variables=(value)
write_metadata_attribute(:yaml_variables, :config_variables, value)
end
def user_variables
......@@ -904,8 +924,11 @@ module Ci
# have the old integer only format. This method returns the retry option
# normalized as a hash in 11.5+ format.
def normalized_retry
value = options&.dig(:retry)
value.is_a?(Integer) ? { max: value } : value.to_h
strong_memoize(:normalized_retry) do
value = options&.dig(:retry)
value = value.is_a?(Integer) ? { max: value } : value.to_h
value.with_indifferent_access
end
end
def build_attributes_from_config
......@@ -929,5 +952,20 @@ module Ci
def project_destroyed?
project.pending_delete?
end
def read_metadata_attribute(legacy_key, metadata_key, default_value = nil)
read_attribute(legacy_key) || metadata&.read_attribute(metadata_key) || default_value
end
def write_metadata_attribute(legacy_key, metadata_key, value)
# save to metadata or this model depending on the state of feature flag
if Feature.enabled?(:ci_build_metadata_config)
ensure_metadata.write_attribute(metadata_key, value)
write_attribute(legacy_key, nil)
else
write_attribute(legacy_key, value)
metadata&.write_attribute(metadata_key, nil)
end
end
end
end
......@@ -13,8 +13,12 @@ module Ci
belongs_to :build, class_name: 'Ci::Build'
belongs_to :project
before_create :set_build_project
validates :build, presence: true
validates :project, presence: true
serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
serialize :config_variables, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
chronic_duration_attr_reader :timeout_human_readable, :timeout
......@@ -33,5 +37,11 @@ module Ci
update(timeout: timeout, timeout_source: timeout_source)
end
private
def set_build_project
self.project_id ||= self.build.project_id
end
end
end
......@@ -2,7 +2,7 @@
module Ci
class RetryBuildService < ::BaseService
CLONE_ACCESSORS = %i[pipeline project ref tag options commands name
CLONE_ACCESSORS = %i[pipeline project ref tag options name
allow_failure stage stage_id stage_idx trigger_request
yaml_variables when environment coverage_regex
description tag_list protected].freeze
......
......@@ -13,20 +13,23 @@
%tbody
- @stages.each do |stage|
- @builds.select { |build| build[:stage] == stage }.each do |build|
- job = @jobs[build[:name].to_sym]
%tr
%td #{stage.capitalize} Job - #{build[:name]}
%td
%pre= build[:commands]
%pre= job[:before_script].to_a.join('\n')
%pre= job[:script].to_a.join('\n')
%pre= job[:after_script].to_a.join('\n')
%br
%b Tag list:
= build[:tag_list].to_a.join(", ")
%br
%b Only policy:
= @jobs[build[:name].to_sym][:only].to_a.join(", ")
= job[:only].to_a.join(", ")
%br
%b Except policy:
= @jobs[build[:name].to_sym][:except].to_a.join(", ")
= job[:except].to_a.join(", ")
%br
%b Environment:
= build[:environment]
......
# frozen_string_literal: true
require 'active_record/connection_adapters/abstract_mysql_adapter'
require 'active_record/connection_adapters/mysql/schema_definitions'
# MySQL (5.6) and MariaDB (10.1) are currently supported versions within GitLab,
# Since they do not support native `json` datatype we force to emulate it as `text`
if Gitlab::Database.mysql?
module ActiveRecord
module ConnectionAdapters
class AbstractMysqlAdapter
JSON_DATASIZE = 1.megabyte
NATIVE_DATABASE_TYPES.merge!(
json: { name: "text", limit: JSON_DATASIZE },
jsonb: { name: "text", limit: JSON_DATASIZE }
)
end
module MySQL
module ColumnMethods
# We add `jsonb` helper, as `json` is already defined for `MySQL` since Rails 5
def jsonb(*args, **options)
args.each { |name| column(name, :json, options) }
end
end
end
end
end
end
......@@ -102,14 +102,15 @@ class Gitlab::Seeder::Pipelines
[]
end
def create_pipeline!(project, ref, commit)
project.ci_pipelines.create!(sha: commit.id, ref: ref, source: :push)
end
def build_create!(pipeline, opts = {})
attributes = job_attributes(pipeline, opts)
.merge(commands: '$ build command')
attributes[:options] ||= {}
attributes[:options][:script] = 'build command'
Ci::Build.create!(attributes).tap do |build|
# We need to set build trace and artifacts after saving a build
......
# frozen_string_literal: true
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddOptionsToBuildMetadata < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :ci_builds_metadata, :config_options, :jsonb
add_column :ci_builds_metadata, :config_variables, :jsonb
end
end
......@@ -374,6 +374,8 @@ ActiveRecord::Schema.define(version: 20190103140724) do
t.integer "project_id", null: false
t.integer "timeout"
t.integer "timeout_source", default: 1, null: false
t.jsonb "config_options"
t.jsonb "config_variables"
t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id", unique: true, using: :btree
t.index ["project_id"], name: "index_ci_builds_metadata_on_project_id", using: :btree
end
......
......@@ -325,6 +325,31 @@ This ensures all timestamps have a time zone specified. This in turn means exist
suddenly use a different timezone when the system's timezone changes. It also makes it very clear which
timezone was used in the first place.
## Storing JSON in database
The Rails 5 natively supports `JSONB` (binary JSON) column type.
Example migration adding this column:
```ruby
class AddOptionsToBuildMetadata < ActiveRecord::Migration[5.0]
DOWNTIME = false
def change
add_column :ci_builds_metadata, :config_options, :jsonb
end
end
```
On MySQL the `JSON` and `JSONB` is translated to `TEXT 1MB`, as `JSONB` is PostgreSQL only feature.
For above reason you have to use a serializer to provide a translation layer
in order to support PostgreSQL and MySQL seamlessly:
```ruby
class BuildMetadata
serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
end
```
## Testing
......
......@@ -15,7 +15,6 @@ module Gitlab
def from_commands(job)
self.new(:script).tap do |step|
step.script = job.options[:before_script].to_a + job.options[:script].to_a
step.script = job.commands.split("\n") if step.script.empty?
step.timeout = job.metadata_timeout
step.when = WHEN_ON_SUCCESS
end
......
......@@ -95,7 +95,7 @@ module Gitlab
helpers :before_script, :script, :stage, :type, :after_script,
:cache, :image, :services, :only, :except, :variables,
:artifacts, :commands, :environment, :coverage, :retry,
:artifacts, :environment, :coverage, :retry,
:parallel
attributes :script, :tags, :allow_failure, :when, :dependencies,
......@@ -121,10 +121,6 @@ module Gitlab
@config.merge(to_hash.compact)
end
def commands
(before_script_value.to_a + script_value.to_a).join("\n")
end
def manual_action?
self.when == 'manual'
end
......@@ -156,7 +152,6 @@ module Gitlab
{ name: name,
before_script: before_script_value,
script: script_value,
commands: commands,
image: image_value,
services: services_value,
stage: stage_value,
......
......@@ -33,7 +33,6 @@ module Gitlab
{ stage_idx: @stages.index(job[:stage]),
stage: job[:stage],
commands: job[:commands],
tag_list: job[:tags] || [],
name: job[:name].to_s,
allow_failure: job[:ignore],
......
......@@ -148,6 +148,7 @@ excluded_attributes:
- :when
- :artifacts_file
- :artifacts_metadata
- :commands
push_event_payload:
- :event_id
project_badges:
......
......@@ -150,6 +150,7 @@ module Gitlab
if BUILD_MODELS.include?(@relation_name)
@relation_hash.delete('trace') # old export files have trace
@relation_hash.delete('token')
@relation_hash.delete('commands')
imported_object
elsif @relation_name == :merge_requests
......
......@@ -115,5 +115,15 @@ module Gitlab
string_or_array.split(',').map(&:strip)
end
def deep_indifferent_access(data)
if data.is_a?(Array)
data.map(&method(:deep_indifferent_access))
elsif data.is_a?(Hash)
data.with_indifferent_access
else
data
end
end
end
end
# frozen_string_literal: true
module Serializers
# This serializer exports data as JSON,
# it is designed to be used with interwork compatibility between MySQL and PostgreSQL
# implementations, as used version of MySQL does not support native json type
#
# Secondly, the loader makes the resulting hash to have deep indifferent access
class JSON
class << self
def dump(obj)
# MySQL stores data as text
# look at ./config/initializers/ar_mysql_jsonb_support.rb
if Gitlab::Database.mysql?
obj = ActiveSupport::JSON.encode(obj)
end
obj
end
def load(data)
return if data.nil?
# On MySQL we store data as text
# look at ./config/initializers/ar_mysql_jsonb_support.rb
if Gitlab::Database.mysql?
data = ActiveSupport::JSON.decode(data)
end
Gitlab::Utils.deep_indifferent_access(data)
end
end
end
end
......@@ -7,7 +7,6 @@ FactoryBot.define do
stage_idx 0
ref 'master'
tag false
commands 'ls -a'
protected false
created_at 'Di 29. Okt 09:50:00 CET 2013'
pending
......@@ -15,7 +14,8 @@ FactoryBot.define do
options do
{
image: 'ruby:2.1',
services: ['postgres']
services: ['postgres'],
script: ['ls -a']
}
end
......@@ -28,7 +28,6 @@ FactoryBot.define do
pipeline factory: :ci_pipeline
trait :degenerated do
commands nil
options nil
yaml_variables nil
end
......@@ -95,33 +94,53 @@ FactoryBot.define do
trait :teardown_environment do
environment 'staging'
options environment: { name: 'staging',
action: 'stop',
url: 'http://staging.example.com/$CI_JOB_NAME' }
options do
{
script: %w(ls),
environment: { name: 'staging',
action: 'stop',
url: 'http://staging.example.com/$CI_JOB_NAME' }
}
end
end
trait :deploy_to_production do
environment 'production'
options environment: { name: 'production',
url: 'http://prd.example.com/$CI_JOB_NAME' }
options do
{
script: %w(ls),
environment: { name: 'production',
url: 'http://prd.example.com/$CI_JOB_NAME' }
}
end
end
trait :start_review_app do
environment 'review/$CI_COMMIT_REF_NAME'
options environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
on_stop: 'stop_review_app' }
options do
{
script: %w(ls),
environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
on_stop: 'stop_review_app' }
}
end
end
trait :stop_review_app do
name 'stop_review_app'
environment 'review/$CI_COMMIT_REF_NAME'
options environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
action: 'stop' }
options do
{
script: %w(ls),
environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
action: 'stop' }
}
end
end
trait :allowed_to_fail do
......@@ -142,7 +161,13 @@ FactoryBot.define do
trait :schedulable do
self.when 'delayed'
options start_in: '1 minute'
options do
{
script: ['ls -a'],
start_in: '1 minute'
}
end
end
trait :actionable do
......@@ -265,6 +290,7 @@ FactoryBot.define do
{
image: { name: 'ruby:2.1', entrypoint: '/bin/sh' },
services: ['postgres', { name: 'docker:stable-dind', entrypoint: '/bin/sh', command: 'sleep 30', alias: 'docker' }],
script: %w(echo),
after_script: %w(ls date),
artifacts: {
name: 'artifacts_file',
......
......@@ -5,7 +5,7 @@ describe 'Merge request < User sees mini pipeline graph', :js do
let(:user) { project.creator }
let(:merge_request) { create(:merge_request, source_project: project, head_pipeline: pipeline) }
let(:pipeline) { create(:ci_empty_pipeline, project: project, ref: 'master', status: 'running', sha: project.commit.id) }
let(:build) { create(:ci_build, pipeline: pipeline, stage: 'test', commands: 'test') }
let(:build) { create(:ci_build, pipeline: pipeline, stage: 'test') }
before do
build.run
......
......@@ -272,8 +272,7 @@ describe 'Environments page', :js do
create(:ci_build, :scheduled,
pipeline: pipeline,
name: 'delayed job',
stage: 'test',
commands: 'test')
stage: 'test')
end
let!(:deployment) do
......@@ -304,8 +303,7 @@ describe 'Environments page', :js do
create(:ci_build, :expired_scheduled,
pipeline: pipeline,
name: 'delayed job',
stage: 'test',
commands: 'test')
stage: 'test')
end
it "shows 00:00:00 as the remaining time" do
......
......@@ -18,7 +18,7 @@ describe 'Pipeline', :js do
let!(:build_failed) do
create(:ci_build, :failed,
pipeline: pipeline, stage: 'test', name: 'test', commands: 'test')
pipeline: pipeline, stage: 'test', name: 'test')
end
let!(:build_running) do
......
......@@ -109,8 +109,7 @@ describe 'Pipelines', :js do
context 'when pipeline is cancelable' do
let!(:build) do
create(:ci_build, pipeline: pipeline,
stage: 'test',
commands: 'test')
stage: 'test')
end
before do
......@@ -140,8 +139,7 @@ describe 'Pipelines', :js do
context 'when pipeline is retryable' do
let!(:build) do
create(:ci_build, pipeline: pipeline,
stage: 'test',
commands: 'test')
stage: 'test')
end
before do
......@@ -202,8 +200,7 @@ describe 'Pipelines', :js do
create(:ci_build, :manual,
pipeline: pipeline,
name: 'manual build',
stage: 'test',
commands: 'test')
stage: 'test')
end
before do
......@@ -237,8 +234,7 @@ describe 'Pipelines', :js do
create(:ci_build, :scheduled,
pipeline: pipeline,
name: 'delayed job',
stage: 'test',
commands: 'test')
stage: 'test')
end
before do
......@@ -262,8 +258,7 @@ describe 'Pipelines', :js do
create(:ci_build, :expired_scheduled,
pipeline: pipeline,
name: 'delayed job',
stage: 'test',
commands: 'test')
stage: 'test')
end
it "shows 00:00:00 as the remaining time" do
......
......@@ -14,8 +14,7 @@ describe Projects::JobsController, '(JavaScript fixtures)', type: :controller do
create(:ci_build, :scheduled,
pipeline: pipeline,
name: 'delayed job',
stage: 'test',
commands: 'test')
stage: 'test')
end
render_views
......
......@@ -5,6 +5,11 @@ require 'spec_helper'
describe Gitlab::BackgroundMigration::PopulateExternalPipelineSource, :migration, schema: 20180916011959 do
let(:migration) { described_class.new }
before do
# This migration was created before we introduced metadata configs
stub_feature_flags(ci_build_metadata_config: false)
end
let!(:internal_pipeline) { create(:ci_pipeline, source: :web) }
let(:pipelines) { [internal_pipeline, unknown_pipeline].map(&:id) }
......
......@@ -18,13 +18,6 @@ describe Gitlab::Ci::Build::Step do
end
end
context 'when commands are specified' do
it_behaves_like 'has correct script' do
let(:job) { create(:ci_build, :no_options, commands: "ls -la\ndate") }
let(:script) { ['ls -la', 'date'] }
end
end
context 'when script option is specified' do
it_behaves_like 'has correct script' do
let(:job) { create(:ci_build, :no_options, options: { script: ["ls -la\necho aaa", "date"] }) }
......@@ -62,7 +55,7 @@ describe Gitlab::Ci::Build::Step do
end
context 'when after_script is not empty' do
let(:job) { create(:ci_build, options: { after_script: ['ls -la', 'date'] }) }
let(:job) { create(:ci_build, options: { script: ['bash'], after_script: ['ls -la', 'date'] }) }
it 'fabricates an object' do
expect(subject.name).to eq(:after_script)
......
......@@ -153,7 +153,6 @@ describe Gitlab::Ci::Config::Entry::Global do
rspec: { name: :rspec,
script: %w[rspec ls],
before_script: %w(ls pwd),
commands: "ls\npwd\nrspec\nls",
image: { name: 'ruby:2.2' },
services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }],
stage: 'test',
......@@ -166,7 +165,6 @@ describe Gitlab::Ci::Config::Entry::Global do
spinach: { name: :spinach,
before_script: [],
script: %w[spinach],
commands: 'spinach',
image: { name: 'ruby:2.2' },
services: [{ name: 'postgres:9.1' }, { name: 'mysql:5.5' }],
stage: 'test',
......
......@@ -255,7 +255,6 @@ describe Gitlab::Ci::Config::Entry::Job do
.to eq(name: :rspec,
before_script: %w[ls pwd],
script: %w[rspec],
commands: "ls\npwd\nrspec",
stage: 'test',
ignore: false,
after_script: %w[cleanup],
......@@ -264,16 +263,6 @@ describe Gitlab::Ci::Config::Entry::Job do
end
end
end
describe '#commands' do
let(:config) do
{ before_script: %w[ls pwd], script: 'rspec' }
end
it 'returns a string of commands concatenated with new line character' do
expect(entry.commands).to eq "ls\npwd\nrspec"
end
end
end
describe '#manual_action?' do
......
......@@ -65,14 +65,12 @@ describe Gitlab::Ci::Config::Entry::Jobs do
expect(entry.value).to eq(
rspec: { name: :rspec,
script: %w[rspec],
commands: 'rspec',
ignore: false,
stage: 'test',
only: { refs: %w[branches tags] },
except: {} },
spinach: { name: :spinach,
script: %w[spinach],
commands: 'spinach',
ignore: false,
stage: 'test',
only: { refs: %w[branches tags] },
......
......@@ -6,8 +6,7 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
let(:attributes) do
{ name: 'rspec',
ref: 'master',
commands: 'rspec' }
ref: 'master' }
end
subject do
......@@ -18,7 +17,7 @@ describe Gitlab::Ci::Pipeline::Seed::Build do
it 'returns hash attributes of a build' do
expect(subject.attributes).to be_a Hash
expect(subject.attributes)
.to include(:name, :project, :ref, :commands)
.to include(:name, :project, :ref)
end
end
......
......@@ -21,7 +21,6 @@ module Gitlab
stage: "test",
stage_idx: 1,
name: "rspec",
commands: "pwd\nrspec",
coverage_regex: nil,
tag_list: [],
options: {
......@@ -155,7 +154,6 @@ module Gitlab
builds:
[{ stage_idx: 1,
stage: "test",
commands: "rspec",