GitLab wurde erfolgreich aktualisiert. Durch regelmäßige Updates bleibt das THM GitLab sicher. Danke für Ihre Geduld.

Commit 0103d5be authored by Kamil Trzciński's avatar Kamil Trzciński

Add config_options|variables to BuildMetadata

These are data columns that store runtime configuration
of build needed to execute it on runner and within pipeline.

The definition of this data is that once used, and when no longer
needed (due to retry capability) they can be freely removed.

They use `jsonb` on PostgreSQL, and `text` on MySQL (due to lacking
support for json datatype on old enough version).
parent b647ad96
......@@ -8,10 +8,15 @@ class Build < CommitStatus
include ObjectStorage::BackgroundMove
include Presentable
include Importable
include IgnorableColumn
include Gitlab::Utils::StrongMemoize
include Deployable
include HasRef
BuildArchivedError = Class.new(StandardError)
ignore_column :commands
belongs_to :project, inverse_of: :builds
belongs_to :runner
belongs_to :trigger_request
......@@ -31,7 +36,7 @@ class Build < CommitStatus
has_one :"job_artifacts_#{key}", -> { where(file_type: value) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
end
has_one :metadata, class_name: 'Ci::BuildMetadata'
has_one :metadata, class_name: 'Ci::BuildMetadata', autosave: true
has_one :runner_session, class_name: 'Ci::BuildRunnerSession', validate: true, inverse_of: :build
accepts_nested_attributes_for :runner_session
......@@ -273,11 +278,14 @@ def pages_generator?
# degenerated build is one that cannot be run by Runner
def degenerated?
self.options.nil?
self.options.blank?
end
def degenerate!
self.update!(options: nil, yaml_variables: nil, commands: nil)
Build.transaction do
self.update!(options: nil, yaml_variables: nil)
self.metadata&.destroy
end
end
def archived?
......@@ -624,11 +632,23 @@ def coverage_regex
end
def when
read_attribute(:when) || build_attributes_from_config[:when] || 'on_success'
read_attribute(:when) || 'on_success'
end
def options
read_metadata_attribute(:options, :config_options, {})
end
def yaml_variables
read_attribute(:yaml_variables) || build_attributes_from_config[:yaml_variables] || []
read_metadata_attribute(:yaml_variables, :config_variables, [])
end
def options=(value)
write_metadata_attribute(:options, :config_options, value)
end
def yaml_variables=(value)
write_metadata_attribute(:yaml_variables, :config_variables, value)
end
def user_variables
......@@ -904,8 +924,11 @@ def environment_url
# have the old integer only format. This method returns the retry option
# normalized as a hash in 11.5+ format.
def normalized_retry
value = options&.dig(:retry)
value.is_a?(Integer) ? { max: value } : value.to_h
strong_memoize(:normalized_retry) do
value = options&.dig(:retry)
value = value.is_a?(Integer) ? { max: value } : value.to_h
value.with_indifferent_access
end
end
def build_attributes_from_config
......@@ -929,5 +952,20 @@ def update_project_statistics(difference)
def project_destroyed?
project.pending_delete?
end
def read_metadata_attribute(legacy_key, metadata_key, default_value = nil)
read_attribute(legacy_key) || metadata&.read_attribute(metadata_key) || default_value
end
def write_metadata_attribute(legacy_key, metadata_key, value)
# save to metadata or this model depending on the state of feature flag
if Feature.enabled?(:ci_build_metadata_config)
ensure_metadata.write_attribute(metadata_key, value)
write_attribute(legacy_key, nil)
else
write_attribute(legacy_key, value)
metadata&.write_attribute(metadata_key, nil)
end
end
end
end
......@@ -13,8 +13,12 @@ class BuildMetadata < ActiveRecord::Base
belongs_to :build, class_name: 'Ci::Build'
belongs_to :project
before_create :set_build_project
validates :build, presence: true
validates :project, presence: true
serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
serialize :config_variables, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
chronic_duration_attr_reader :timeout_human_readable, :timeout
......@@ -33,5 +37,11 @@ def update_timeout_state
update(timeout: timeout, timeout_source: timeout_source)
end
private
def set_build_project
self.project_id ||= self.build.project_id
end
end
end
......@@ -2,7 +2,7 @@
module Ci
class RetryBuildService < ::BaseService
CLONE_ACCESSORS = %i[pipeline project ref tag options commands name
CLONE_ACCESSORS = %i[pipeline project ref tag options name
allow_failure stage stage_id stage_idx trigger_request
yaml_variables when environment coverage_regex
description tag_list protected].freeze
......
......@@ -13,20 +13,23 @@
%tbody
- @stages.each do |stage|
- @builds.select { |build| build[:stage] == stage }.each do |build|
- job = @jobs[build[:name].to_sym]
%tr
%td #{stage.capitalize} Job - #{build[:name]}
%td
%pre= build[:commands]
%pre= job[:before_script].to_a.join('\n')
%pre= job[:script].to_a.join('\n')
%pre= job[:after_script].to_a.join('\n')
%br
%b Tag list:
= build[:tag_list].to_a.join(", ")
%br
%b Only policy:
= @jobs[build[:name].to_sym][:only].to_a.join(", ")
= job[:only].to_a.join(", ")
%br
%b Except policy:
= @jobs[build[:name].to_sym][:except].to_a.join(", ")
= job[:except].to_a.join(", ")
%br
%b Environment:
= build[:environment]
......
# frozen_string_literal: true
require 'active_record/connection_adapters/abstract_mysql_adapter'
require 'active_record/connection_adapters/mysql/schema_definitions'
# MySQL (5.6) and MariaDB (10.1) are currently supported versions within GitLab,
# Since they do not support native `json` datatype we force to emulate it as `text`
if Gitlab::Database.mysql?
module ActiveRecord
module ConnectionAdapters
class AbstractMysqlAdapter
JSON_DATASIZE = 1.megabyte
NATIVE_DATABASE_TYPES.merge!(
json: { name: "text", limit: JSON_DATASIZE },
jsonb: { name: "text", limit: JSON_DATASIZE }
)
end
module MySQL
module ColumnMethods
# We add `jsonb` helper, as `json` is already defined for `MySQL` since Rails 5
def jsonb(*args, **options)
args.each { |name| column(name, :json, options) }
end
end
end
end
end
end
......@@ -102,14 +102,15 @@ def create_merge_request_pipelines
[]
end
def create_pipeline!(project, ref, commit)
project.ci_pipelines.create!(sha: commit.id, ref: ref, source: :push)
end
def build_create!(pipeline, opts = {})
attributes = job_attributes(pipeline, opts)
.merge(commands: '$ build command')
attributes[:options] ||= {}
attributes[:options][:script] = 'build command'
Ci::Build.create!(attributes).tap do |build|
# We need to set build trace and artifacts after saving a build
......
# frozen_string_literal: true
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddOptionsToBuildMetadata < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :ci_builds_metadata, :config_options, :jsonb
add_column :ci_builds_metadata, :config_variables, :jsonb
end
end
......@@ -374,6 +374,8 @@
t.integer "project_id", null: false
t.integer "timeout"
t.integer "timeout_source", default: 1, null: false
t.jsonb "config_options"
t.jsonb "config_variables"
t.index ["build_id"], name: "index_ci_builds_metadata_on_build_id", unique: true, using: :btree
t.index ["project_id"], name: "index_ci_builds_metadata_on_project_id", using: :btree
end
......
......@@ -325,6 +325,31 @@ This ensures all timestamps have a time zone specified. This in turn means exist
suddenly use a different timezone when the system's timezone changes. It also makes it very clear which
timezone was used in the first place.
## Storing JSON in database
The Rails 5 natively supports `JSONB` (binary JSON) column type.
Example migration adding this column:
```ruby
class AddOptionsToBuildMetadata < ActiveRecord::Migration[5.0]
DOWNTIME = false
def change
add_column :ci_builds_metadata, :config_options, :jsonb
end
end
```
On MySQL the `JSON` and `JSONB` is translated to `TEXT 1MB`, as `JSONB` is PostgreSQL only feature.
For above reason you have to use a serializer to provide a translation layer
in order to support PostgreSQL and MySQL seamlessly:
```ruby
class BuildMetadata
serialize :config_options, Serializers::JSON # rubocop:disable Cop/ActiveRecordSerialize
end
```
## Testing
......
......@@ -15,7 +15,6 @@ class << self
def from_commands(job)
self.new(:script).tap do |step|
step.script = job.options[:before_script].to_a + job.options[:script].to_a
step.script = job.commands.split("\n") if step.script.empty?
step.timeout = job.metadata_timeout
step.when = WHEN_ON_SUCCESS
end
......
......@@ -95,7 +95,7 @@ class Job < ::Gitlab::Config::Entry::Node
helpers :before_script, :script, :stage, :type, :after_script,
:cache, :image, :services, :only, :except, :variables,
:artifacts, :commands, :environment, :coverage, :retry,
:artifacts, :environment, :coverage, :retry,
:parallel
attributes :script, :tags, :allow_failure, :when, :dependencies,
......@@ -121,10 +121,6 @@ def value
@config.merge(to_hash.compact)
end
def commands
(before_script_value.to_a + script_value.to_a).join("\n")
end
def manual_action?
self.when == 'manual'
end
......@@ -156,7 +152,6 @@ def to_hash
{ name: name,
before_script: before_script_value,
script: script_value,
commands: commands,
image: image_value,
services: services_value,
stage: stage_value,
......
......@@ -33,7 +33,6 @@ def build_attributes(name)
{ stage_idx: @stages.index(job[:stage]),
stage: job[:stage],
commands: job[:commands],
tag_list: job[:tags] || [],
name: job[:name].to_s,
allow_failure: job[:ignore],
......
......@@ -148,6 +148,7 @@ excluded_attributes:
- :when
- :artifacts_file
- :artifacts_metadata
- :commands
push_event_payload:
- :event_id
project_badges:
......
......@@ -150,6 +150,7 @@ def generate_imported_object
if BUILD_MODELS.include?(@relation_name)
@relation_hash.delete('trace') # old export files have trace
@relation_hash.delete('token')
@relation_hash.delete('commands')
imported_object
elsif @relation_name == :merge_requests
......
......@@ -115,5 +115,15 @@ def ensure_array_from_string(string_or_array)
string_or_array.split(',').map(&:strip)
end
def deep_indifferent_access(data)
if data.is_a?(Array)
data.map(&method(:deep_indifferent_access))
elsif data.is_a?(Hash)
data.with_indifferent_access
else
data
end
end
end
end
# frozen_string_literal: true
module Serializers
# This serializer exports data as JSON,
# it is designed to be used with interwork compatibility between MySQL and PostgreSQL
# implementations, as used version of MySQL does not support native json type
#
# Secondly, the loader makes the resulting hash to have deep indifferent access
class JSON
class << self
def dump(obj)
# MySQL stores data as text
# look at ./config/initializers/ar_mysql_jsonb_support.rb
if Gitlab::Database.mysql?
obj = ActiveSupport::JSON.encode(obj)
end
obj
end
def load(data)
return if data.nil?
# On MySQL we store data as text
# look at ./config/initializers/ar_mysql_jsonb_support.rb
if Gitlab::Database.mysql?
data = ActiveSupport::JSON.decode(data)
end
Gitlab::Utils.deep_indifferent_access(data)
end
end
end
end
......@@ -7,7 +7,6 @@
stage_idx 0
ref 'master'
tag false
commands 'ls -a'
protected false
created_at 'Di 29. Okt 09:50:00 CET 2013'
pending
......@@ -15,7 +14,8 @@
options do
{
image: 'ruby:2.1',
services: ['postgres']
services: ['postgres'],
script: ['ls -a']
}
end
......@@ -28,7 +28,6 @@
pipeline factory: :ci_pipeline
trait :degenerated do
commands nil
options nil
yaml_variables nil
end
......@@ -95,33 +94,53 @@
trait :teardown_environment do
environment 'staging'
options environment: { name: 'staging',
action: 'stop',
url: 'http://staging.example.com/$CI_JOB_NAME' }
options do
{
script: %w(ls),
environment: { name: 'staging',
action: 'stop',
url: 'http://staging.example.com/$CI_JOB_NAME' }
}
end
end
trait :deploy_to_production do
environment 'production'
options environment: { name: 'production',
url: 'http://prd.example.com/$CI_JOB_NAME' }
options do
{
script: %w(ls),
environment: { name: 'production',
url: 'http://prd.example.com/$CI_JOB_NAME' }
}
end
end
trait :start_review_app do
environment 'review/$CI_COMMIT_REF_NAME'
options environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
on_stop: 'stop_review_app' }
options do
{
script: %w(ls),
environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
on_stop: 'stop_review_app' }
}
end
end
trait :stop_review_app do
name 'stop_review_app'
environment 'review/$CI_COMMIT_REF_NAME'
options environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
action: 'stop' }
options do
{
script: %w(ls),
environment: { name: 'review/$CI_COMMIT_REF_NAME',
url: 'http://staging.example.com/$CI_JOB_NAME',
action: 'stop' }
}
end
end
trait :allowed_to_fail do
......@@ -142,7 +161,13 @@
trait :schedulable do
self.when 'delayed'
options start_in: '1 minute'
options do
{
script: ['ls -a'],
start_in: '1 minute'
}
end
end
trait :actionable do
......@@ -265,6 +290,7 @@
{
image: { name: 'ruby:2.1', entrypoint: '/bin/sh' },
services: ['postgres', { name: 'docker:stable-dind', entrypoint: '/bin/sh', command: 'sleep 30', alias: 'docker' }],
script: %w(echo),
after_script: %w(ls date),
artifacts: {
name: 'artifacts_file',
......
......@@ -5,7 +5,7 @@
let(:user) { project.creator }
let(:merge_request) { create(:merge_request, source_project: project, head_pipeline: pipeline) }
let(:pipeline) { create(:ci_empty_pipeline, project: project, ref: 'master', status: 'running', sha: project.commit.id) }
let(:build) { create(:ci_build, pipeline: pipeline, stage: 'test', commands: 'test') }
let(:build) { create(:ci_build, pipeline: pipeline, stage: 'test') }
before do
build.run
......
......@@ -272,8 +272,7 @@ def stop_button_selector
create(:ci_build, :scheduled,
pipeline: pipeline,
name: 'delayed job',
stage: 'test',
commands: 'test')
stage: 'test')
end
let!(:deployment) do
......@@ -304,8 +303,7 @@ def stop_button_selector
create(:ci_build, :expired_scheduled,
pipeline: pipeline,
name: 'delayed job',
stage: 'test',
commands: 'test')
stage: 'test')
end
it "shows 00:00:00 as the remaining time" do
......
......@@ -18,7 +18,7 @@
let!(:build_failed) do
create(:ci_build, :failed,
pipeline: pipeline, stage: 'test', name: 'test', commands: 'test')
pipeline: pipeline, stage: 'test', name: 'test')
end
let!(:build_running) do
......
......@@ -109,8 +109,7 @@
context 'when pipeline is cancelable' do
let!(:build) do
create(:ci_build, pipeline: pipeline,
stage: 'test',
commands: 'test')
stage: 'test')
end
before do
......@@ -140,8 +139,7 @@
context 'when pipeline is retryable' do
let!(:build) do
create(:ci_build, pipeline: pipeline,
stage: 'test',
commands: 'test')
stage: 'test')
end
before do
......@@ -202,8 +200,7 @@
create(:ci_build, :manual,
pipeline: pipeline,
name: 'manual build',
stage: 'test',
commands: 'test')
stage: 'test')
end
before do
......@@ -237,8 +234,7 @@
create(:ci_build, :scheduled,
pipeline: pipeline,
name: 'delayed job',
stage: 'test',
commands: 'test')
stage: 'test')
end
before do
......@@ -262,8 +258,7 @@
create(:ci_build, :expired_scheduled,
pipeline: pipeline,
name: 'delayed job',
stage: 'test',
commands: 'test')
stage: 'test')
end
it "shows 00:00:00 as the remaining time" do
......
......@@ -14,8 +14,7 @@
create(:ci_build, :scheduled,
pipeline: pipeline,
name: 'delayed job',
stage: 'test',
commands: 'test')
stage: 'test')
end
render_views
......
......@@ -5,6 +5,11 @@
describe Gitlab::BackgroundMigration::PopulateExternalPipelineSource, :migration, schema: 20180916011959 do
let(:migration) { described_class.new }
before do
# This migration was created before we introduced metadata configs
stub_feature_flags(ci_build_metadata_config: false)