Commit 0103d5be authored by Kamil Trzciński's avatar Kamil Trzciński
Browse files

Add config_options|variables to BuildMetadata

These are data columns that store runtime configuration
of build needed to execute it on runner and within pipeline.

The definition of this data is that once used, and when no longer
needed (due to retry capability) they can be freely removed.

They use `jsonb` on PostgreSQL, and `text` on MySQL (due to lacking
support for json datatype on old enough version).
parent b647ad96
...@@ -8,10 +8,15 @@ class Build < CommitStatus ...@@ -8,10 +8,15 @@ class Build < CommitStatus
include ObjectStorage::BackgroundMove include ObjectStorage::BackgroundMove
include Presentable include Presentable
include Importable include Importable
include IgnorableColumn
include Gitlab::Utils::StrongMemoize include Gitlab::Utils::StrongMemoize
include Deployable include Deployable
include HasRef include HasRef
BuildArchivedError = Class.new(StandardError)
ignore_column :commands
belongs_to :project, inverse_of: :builds belongs_to :project, inverse_of: :builds
belongs_to :runner belongs_to :runner
belongs_to :trigger_request belongs_to :trigger_request
...@@ -31,7 +36,7 @@ class Build < CommitStatus ...@@ -31,7 +36,7 @@ class Build < CommitStatus
has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
end end
has_one :metadata, class_name: 'Ci::BuildMetadata' has_one :metadata, class_name: 'Ci::BuildMetadata', autosave: true
has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build
accepts_nested_attributes_for :runner_session accepts_nested_attributes_for :runner_session
...@@ -273,11 +278,14 @@ def pages_generator? ...@@ -273,11 +278,14 @@ def pages_generator?
# degenerated build is one that cannot be run by Runner # degenerated build is one that cannot be run by Runner
def degenerated? def degenerated?
self.options.nil? self.options.blank?
end end
def degenerate! def degenerate!
self.update!(options: nil, yaml_variables: nil, commands: nil) Build.transaction do
self.update!(options: nil, yaml_variables: nil)
self.metadata&.destroy
end
end end
def archived? def archived?
...@@ -624,11 +632,23 @@ def coverage_regex ...@@ -624,11 +632,23 @@ def coverage_regex
end end
def when def when
read_attribute(:when) || build_attributes_from_config[:when] || 'on_success' read_attribute(:when) || 'on_success'
end
def options
read_metadata_attribute(:options, :config_options, {})
end end
def yaml_variables def yaml_variables
read_attribute(:yaml_variables) || build_attributes_from_config[:yaml_variables] || [] read_metadata_attribute(:yaml_variables, :config_variables, [])
end
def options=(value)
write_metadata_attribute(:options, :config_options, value)
end
def yaml_variables=(value)
write_metadata_attribute(:yaml_variables, :config_variables, value)
end end
def user_variables def user_variables
...@@ -904,8 +924,11 @@ def environment_url ...@@ -904,8 +924,11 @@ def environment_url
# have the old integer only format. This method returns the retry option # have the old integer only format. This method returns the retry option
# normalized as a hash in 11.5+ format. # normalized as a hash in 11.5+ format.
def normalized_retry def normalized_retry
value = options&.dig(:retry) strong_memoize(:normalized_retry) do
value.is_a?(Integer) ? { max: value } : value.to_h value = options&.dig(:retry)
value = value.is_a?(Integer) ? { max: value } : value.to_h
value.with_indifferent_access
end
end end
def build_attributes_from_config def build_attributes_from_config
...@@ -929,5 +952,20 @@ def update_project_statistics(difference) ...@@ -929,5 +952,20 @@ def update_project_statistics(difference)
def project_destroyed? def project_destroyed?
project.pending_delete? project.pending_delete?
end end
def read_metadata_attribute(legacy_key, metadata_key, default_value = nil)
read_attribute(legacy_key) || metadata&.read_attribute(metadata_key) || default_value
end
def write_metadata_attribute(legacy_key, metadata_key, value)
# save to metadata or this model depending on the state of feature flag
if Feature.enabled?(:ci_build_metadata_config)
ensure_metadata.write_attribute(metadata_key, value)
write_attribute(legacy_key, nil)
else
write_attribute(legacy_key, value)
metadata&.write_attribute(metadata_key, nil)
end
end
end end
end end
...@@ -13,8 +13,12 @@ class BuildMetadata < ActiveRecord::Base ...@@ -13,8 +13,12 @@ class BuildMetadata < ActiveRecord::Base
belongs_to :build, class_name: 'Ci::Build' belongs_to :build, class_name: 'Ci::Build'
belongs_to :project belongs_to :project
before_create :set_build_project
validates :build, presence: true validates :build, presence: true
validates :project, presence: true
serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
serialize :config_variables, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
chronic_duration_attr_reader :timeout_human_readable, :timeout chronic_duration_attr_reader :timeout_human_readable, :timeout
...@@ -33,5 +37,11 @@ def update_timeout_state ...@@ -33,5 +37,11 @@ def update_timeout_state
update(timeout: timeout, timeout_source: timeout_source) update(timeout: timeout, timeout_source: timeout_source)
end end
private
def set_build_project
self.project_id ||= self.build.project_id
end
end end
end end
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
module Ci module Ci
class RetryBuildService < ::BaseService class RetryBuildService < ::BaseService
CLONE_ACCESSORS = %i[pipeline project ref tag options commands name CLONE_ACCESSORS = %i[pipeline project ref tag options name
allow_failure stage stage_id stage_idx trigger_request allow_failure stage stage_id stage_idx trigger_request
yaml_variables when environment coverage_regex yaml_variables when environment coverage_regex
description tag_list protected].freeze description tag_list protected].freeze
......
...@@ -13,20 +13,23 @@ ...@@ -13,20 +13,23 @@
%tbody %tbody
- @stages.each do |stage| - @stages.each do |stage|
- @builds.select { |build| build[:stage] == stage }.each do |build| - @builds.select { |build| build[:stage] == stage }.each do |build|
- job = @jobs[build[:name].to_sym]
%tr %tr
%td #{stage.capitalize} Job - #{build[:name]} %td #{stage.capitalize} Job - #{build[:name]}
%td %td
%pre= build[:commands] %pre= job[:before_script].to_a.join('\n')
%pre= job[:script].to_a.join('\n')
%pre= job[:after_script].to_a.join('\n')
%br %br
%b Tag list: %b Tag list:
= build[:tag_list].to_a.join(", ") = build[:tag_list].to_a.join(", ")
%br %br
%b Only policy: %b Only policy:
= @jobs[build[:name].to_sym][:only].to_a.join(", ") = job[:only].to_a.join(", ")
%br %br
%b Except policy: %b Except policy:
= @jobs[build[:name].to_sym][:except].to_a.join(", ") = job[:except].to_a.join(", ")
%br %br
%b Environment: %b Environment:
= build[:environment] = build[:environment]
......
# frozen_string_literal: true
require 'active_record/connection_adapters/abstract_mysql_adapter'
require 'active_record/connection_adapters/mysql/schema_definitions'
# MySQL (5.6) and MariaDB (10.1) are currently supported versions within GitLab,
# Since they do not support native `json` datatype we force to emulate it as `text`
if Gitlab::Database.mysql?
module ActiveRecord
module ConnectionAdapters
class AbstractMysqlAdapter
JSON_DATASIZE = 1.megabyte
NATIVE_DATABASE_TYPES.merge!(
json: { name: "text", limit: JSON_DATASIZE },
jsonb: { name: "text", limit: JSON_DATASIZE }
)
end
module MySQL
module ColumnMethods
# We add `jsonb` helper, as `json` is already defined for `MySQL` since Rails 5
def jsonb(*args, **options)
args.each { |name| column(name, :json, options) }
end
end
end
end
end
end
...@@ -102,14 +102,15 @@ def create_merge_request_pipelines ...@@ -102,14 +102,15 @@ def create_merge_request_pipelines
[] []
end end
def create_pipeline!(project, ref, commit) def create_pipeline!(project, ref, commit)
project.ci_pipelines.create!(sha: commit.id, ref: ref, source: :push) project.ci_pipelines.create!(sha: commit.id, ref: ref, source: :push)
end end
def build_create!(pipeline, opts = {}) def build_create!(pipeline, opts = {})
attributes = job_attributes(pipeline, opts) attributes = job_attributes(pipeline, opts)
.merge(commands: '$ build command')
attributes[:options] ||= {}
attributes[:options][:script] = 'build command'
Ci::Build.create!(attributes).tap do |build| Ci::Build.create!(attributes).tap do |build|
# We need to set build trace and artifacts after saving a build # We need to set build trace and artifacts after saving a build
......
# frozen_string_literal: true
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddOptionsToBuildMetadata < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :ci_builds_metadata, :config_options, :jsonb
add_column :ci_builds_metadata, :config_variables, :jsonb
end
end
...@@ -374,6 +374,8 @@ ...@@ -374,6 +374,8 @@
t.integer "project_id", null: false t.integer "project_id", null: false
t.integer "timeout" t.integer "timeout"
t.integer "timeout_source", default: 1, null: false t.integer "timeout_source", default: 1, null: false
t.jsonb "config_options"
t.jsonb "config_variables"
t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id", unique: true, using: :btree t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id", unique: true, using: :btree
t.index ["project_id"], name: "index_ci_builds_metadata_on_project_id", using: :btree t.index ["project_id"], name: "index_ci_builds_metadata_on_project_id", using: :btree
end end
......
...@@ -325,6 +325,31 @@ This ensures all timestamps have a time zone specified. This in turn means exist ...@@ -325,6 +325,31 @@ This ensures all timestamps have a time zone specified. This in turn means exist
suddenly use a different timezone when the system's timezone changes. It also makes it very clear which suddenly use a different timezone when the system's timezone changes. It also makes it very clear which
timezone was used in the first place. timezone was used in the first place.
## Storing JSON in database
The Rails 5 natively supports `JSONB` (binary JSON) column type.
Example migration adding this column:
```ruby
class AddOptionsToBuildMetadata < ActiveRecord::Migration[5.0]
DOWNTIME = false
def change
add_column :ci_builds_metadata, :config_options, :jsonb
end
end
```
On MySQL the `JSON` and `JSONB` is translated to `TEXT 1MB`, as `JSONB` is PostgreSQL only feature.
For above reason you have to use a serializer to provide a translation layer
in order to support PostgreSQL and MySQL seamlessly:
```ruby
class BuildMetadata
serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
end
```
## Testing ## Testing
......
...@@ -15,7 +15,6 @@ class << self ...@@ -15,7 +15,6 @@ class << self
def from_commands(job) def from_commands(job)
self.new(:script).tap do |step| self.new(:script).tap do |step|
step.script = job.options[:before_script].to_a + job.options[:script].to_a step.script = job.options[:before_script].to_a + job.options[:script].to_a
step.script = job.commands.split("\n") if step.script.empty?
step.timeout = job.metadata_timeout step.timeout = job.metadata_timeout
step.when = WHEN_ON_SUCCESS step.when = WHEN_ON_SUCCESS
end end
......
...@@ -95,7 +95,7 @@ class Job < ::Gitlab::Config::Entry::Node ...@@ -95,7 +95,7 @@ class Job < ::Gitlab::Config::Entry::Node
helpers :before_script, :script, :stage, :type, :after_script, helpers :before_script, :script, :stage, :type, :after_script,
:cache, :image, :services, :only, :except, :variables, :cache, :image, :services, :only, :except, :variables,
:artifacts, :commands, :environment, :coverage, :retry, :artifacts, :environment, :coverage, :retry,
:parallel :parallel
attributes :script, :tags, :allow_failure, :when, :dependencies, attributes :script, :tags, :allow_failure, :when, :dependencies,
...@@ -121,10 +121,6 @@ def value ...@@ -121,10 +121,6 @@ def value
@config.merge(to_hash.compact) @config.merge(to_hash.compact)
end end
def commands
(before_script_value.to_a + script_value.to_a).join("\n")
end
def manual_action? def manual_action?
self.when == 'manual' self.when == 'manual'
end end
...@@ -156,7 +152,6 @@ def to_hash ...@@ -156,7 +152,6 @@ def to_hash
{ name: name, { name: name,
before_script: before_script_value, before_script: before_script_value,
script: script_value, script: script_value,
commands: commands,
image: image_value, image: image_value,
services: services_value, services: services_value,
stage: stage_value, stage: stage_value,
......
...@@ -33,7 +33,6 @@ def build_attributes(name) ...@@ -33,7 +33,6 @@ def build_attributes(name)
{ stage_idx: @stages.index(job[:stage]), { stage_idx: @stages.index(job[:stage]),
stage: job[:stage], stage: job[:stage],
commands: job[:commands],
tag_list: job[:tags] || [], tag_list: job[:tags] || [],
name: job[:name].to_s, name: job[:name].to_s,
allow_failure: job[:ignore], allow_failure: job[:ignore],
......
...@@ -148,6 +148,7 @@ excluded_attributes: ...@@ -148,6 +148,7 @@ excluded_attributes:
- :when - :when
- :artifacts_file - :artifacts_file
- :artifacts_metadata - :artifacts_metadata
- :commands
push_event_payload: push_event_payload:
- :event_id - :event_id
project_badges: project_badges:
......
...@@ -150,6 +150,7 @@ def generate_imported_object ...@@ -150,6 +150,7 @@ def generate_imported_object
if BUILD_MODELS.include?(@relation_name) if BUILD_MODELS.include?(@relation_name)
@relation_hash.delete('trace') # old export files have trace @relation_hash.delete('trace') # old export files have trace
@relation_hash.delete('token') @relation_hash.delete('token')
@relation_hash.delete('commands')
imported_object imported_object
elsif @relation_name == :merge_requests elsif @relation_name == :merge_requests
......
...@@ -115,5 +115,15 @@ def ensure_array_from_string(string_or_array) ...@@ -115,5 +115,15 @@ def ensure_array_from_string(string_or_array)
string_or_array.split(',').map(&:strip) string_or_array.split(',').map(&:strip)
end end
def deep_indifferent_access(data)
if data.is_a?(Array)
data.map(&method(:deep_indifferent_access))
elsif data.is_a?(Hash)
data.with_indifferent_access
else
data
end
end
end end
end end
# frozen_string_literal: true
module Serializers
# This serializer exports data as JSON,
# it is designed to be used with interwork compatibility between MySQL and PostgreSQL
# implementations, as used version of MySQL does not support native json type
#
# Secondly, the loader makes the resulting hash to have deep indifferent access
class JSON
class << self
def dump(obj)
# MySQL stores data as text
# look at ./config/initializers/ar_mysql_jsonb_support.rb
if Gitlab::Database.mysql?
obj = ActiveSupport::JSON.encode(obj)
end
obj
end
def load(data)
return if data.nil?
# On MySQL we store data as text
# look at ./config/initializers/ar_mysql_jsonb_support.rb
if Gitlab::Database.mysql?
data = ActiveSupport::JSON.decode(data)
end
Gitlab::Utils.deep_indifferent_access(data)
end
end
end
end
...@@ -7,7 +7,6 @@ ...@@ -7,7 +7,6 @@
stage_idx 0 stage_idx 0
ref 'master' ref 'master'
tag false tag false
commands 'ls -a'
protected false protected false
created_at 'Di 29. Okt 09:50:00 CET 2013' created_at 'Di 29. Okt 09:50:00 CET 2013'
pending pending
...@@ -15,7 +14,8 @@ ...@@ -15,7 +14,8 @@
options do options do
{ {
image: 'ruby:2.1', image: 'ruby:2.1',
services: ['postgres'] services: ['postgres'],
script: ['ls -a']
} }
end end
...@@ -28,7 +28,6 @@ ...@@ -28,7 +28,6 @@
pipeline factory: :ci_pipeline pipeline factory: :ci_pipeline
trait :degenerated do trait :degenerated do
commands nil
options nil options nil
yaml_variables nil yaml_variables nil
end end
...@@ -95,33 +94,53 @@ ...@@ -95,33 +94,53 @@
trait :teardown_environment do trait :teardown_environment do
environment 'staging' environment 'staging'
options environment: { name: 'staging', options do
action: 'stop', {
url: 'http://staging.example.com/$CI_JOB_NAME' } script: %w(ls),
environment: { name: 'staging',
action: 'stop',
url: 'http://staging.example.com/$CI_JOB_NAME' }
}
end
end end
trait :deploy_to_production do trait :deploy_to_production do
environment 'production' environment 'production'
options environment: { name: 'production', options do
url: 'http://prd.example.com/$CI_JOB_NAME' } {
script: %w(ls),
environment: { name: 'production',
url: 'http://prd.example.com/$CI_JOB_NAME' }
}
end
end end
trait :start_review_app do trait :start_review_app do
environment 'review/$CI_COMMIT_REF_NAME' environment 'review/$CI_COMMIT_REF_NAME'
options environment: { name: 'review/$CI_COMMIT_REF_NAME', options do
url: 'http://staging.example.com/$CI_JOB_NAME', {
on_stop: 'stop_review_app' } script: %w(ls),
environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
on_stop: 'stop_review_app' }
}
end
end end
trait :stop_review_app do trait :stop_review_app do
name 'stop_review_app'