Commit 4b0036b8 authored by Grzegorz Bizon's avatar Grzegorz Bizon

Merge branch 'allow-to-recursively-include' into 'master'

Allow to recursively expand includes

Closes #55937 and #56836

See merge request gitlab-org/gitlab-ce!24356
parents 8c6c1a3f c78861bc
---
title: Allow to recursively expand includes
merge_request: 24356
author:
type: added
......@@ -1594,6 +1594,9 @@ You can only use files that are currently tracked by Git on the same branch
your configuration file is on. In other words, when using a `include:local`, make
sure that both `.gitlab-ci.yml` and the local file are on the same branch.
All [nested includes](#nested-includes) will be executed in the scope of the same project,
so it is possible to use local, project, remote or template includes.
NOTE: **Note:**
Including local files through Git submodules paths is not supported.
......@@ -1606,7 +1609,7 @@ include:
### `include:file`
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/53903) in GitLab 11.7.
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/53903) in GitLab 11.9.
To include files from another private project under the same GitLab instance,
use `include:file`. This file is referenced using full paths relative to the
......@@ -1635,6 +1638,10 @@ include:
file: '/templates/.gitlab-ci-template.yml'
```
All nested includes will be executed in the scope of the target project,
so it is possible to used local (relative to target project), project, remote
or template includes.
### `include:template`
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/53445) in GitLab 11.7.
......@@ -1650,6 +1657,9 @@ include:
- template: Auto-DevOps.gitlab-ci.yml
```
All nested includes will be executed only with the permission of the user,
so it is possible to use project, remote or template includes.
### `include:remote`
`include:remote` can be used to include a file from a different location,
......@@ -1662,10 +1672,16 @@ include:
- remote: 'https://gitlab.com/awesome-project/raw/master/.gitlab-ci-template.yml'
```
NOTE: **Note for GitLab admins:**
In order to include files from another repository inside your local network,
you may need to enable the **Allow requests to the local network from hooks and services** checkbox
located in the **Admin area > Settings > Network > Outbound requests** section.
All nested includes will be executed without context as public user, so only another remote,
or public project, or template is allowed.
### Nested includes
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/53903) in GitLab 11.7.
Nested includes allow you to compose a set of includes.
A total of 50 includes is allowed.
Duplicate includes are considered a configuration error.
### `include` examples
......@@ -1834,6 +1850,51 @@ In this case, if `install_dependencies` and `deploy` were not repeated in
`.gitlab-ci.yml`, they would not be part of the script for the `production`
job in the combined CI configuration.
#### Using nested includes
The examples below show how includes can be nested from different sources
using a combination of different methods.
In this example, `.gitlab-ci.yml` includes local the file `/.gitlab-ci/another-config.yml`:
```yaml
includes:
- local: /.gitlab-ci/another-config.yml
```
The `/.gitlab-ci/another-config.yml` includes a template and the `/templates/docker-workflow.yml` file
from another project:
```yaml
includes:
- template: Bash.gitlab-ci.yml
- project: /group/my-project
file: /templates/docker-workflow.yml
```
The `/templates/docker-workflow.yml` present in `/group/my-project` includes two local files
of the `/group/my-project`:
```yaml
includes:
- local: : /templates/docker-build.yml
- local: : /templates/docker-testing.yml
```
Our `/templates/docker-build.yml` present in `/group/my-project` adds a `docker-build` job:
```yaml
docker-build:
script: docker build -t my-image .
```
Our second `/templates/docker-test.yml` present in `/group/my-project` adds a `docker-test` job:
```yaml
docker-test:
script: docker run my-image /run/tests.sh
```
## `extends`
> Introduced in GitLab 11.3.
......
......@@ -84,7 +84,8 @@ module Gitlab
Config::External::Processor.new(config,
project: project,
sha: sha || project.repository.root_ref_sha,
user: user).perform
user: user,
expandset: Set.new).perform
end
end
end
......
......@@ -12,7 +12,7 @@ module Gitlab
YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
Context = Struct.new(:project, :sha, :user)
Context = Struct.new(:project, :sha, :user, :expandset)
def initialize(params, context)
@params = params
......@@ -43,13 +43,27 @@ module Gitlab
end
def to_hash
@hash ||= Gitlab::Config::Loader::Yaml.new(content).load!
rescue Gitlab::Config::Loader::FormatError
nil
expanded_content_hash
end
protected
def expanded_content_hash
return unless content_hash
strong_memoize(:expanded_content_yaml) do
expand_includes(content_hash)
end
end
def content_hash
strong_memoize(:content_yaml) do
Gitlab::Config::Loader::Yaml.new(content).load!
end
rescue Gitlab::Config::Loader::FormatError
nil
end
def validate!
validate_location!
validate_content! if errors.none?
......@@ -73,6 +87,14 @@ module Gitlab
errors.push("Included file `#{location}` does not have valid YAML syntax!")
end
end
def expand_includes(hash)
External::Processor.new(hash, **expand_context).perform
end
def expand_context
{ project: nil, sha: nil, user: nil, expandset: context.expandset }
end
end
end
end
......
......@@ -31,6 +31,13 @@ module Gitlab
def fetch_local_content
context.project.repository.blob_data_at(context.sha, location)
end
def expand_context
super.merge(
project: context.project,
sha: context.sha,
user: context.user)
end
end
end
end
......
......@@ -64,6 +64,13 @@ module Gitlab
project.commit(ref_name).try(:sha)
end
end
def expand_context
super.merge(
project: project,
sha: sha,
user: context.user)
end
end
end
end
......
......@@ -7,6 +7,8 @@ module Gitlab
class Mapper
include Gitlab::Utils::StrongMemoize
MAX_INCLUDES = 50
FILE_CLASSES = [
External::File::Remote,
External::File::Template,
......@@ -14,25 +16,34 @@ module Gitlab
External::File::Project
].freeze
AmbigiousSpecificationError = Class.new(StandardError)
Error = Class.new(StandardError)
AmbigiousSpecificationError = Class.new(Error)
DuplicateIncludesError = Class.new(Error)
TooManyIncludesError = Class.new(Error)
def initialize(values, project:, sha:, user:, expandset:)
raise Error, 'Expanded needs to be `Set`' unless expandset.is_a?(Set)
def initialize(values, project:, sha:, user:)
@locations = Array.wrap(values.fetch(:include, []))
@project = project
@sha = sha
@user = user
@expandset = expandset
end
def process
return [] if locations.empty?
locations
.compact
.map(&method(:normalize_location))
.each(&method(:verify_duplicates!))
.map(&method(:select_first_matching))
end
private
attr_reader :locations, :project, :sha, :user
attr_reader :locations, :project, :sha, :user, :expandset
# convert location if String to canonical form
def normalize_location(location)
......@@ -51,6 +62,23 @@ module Gitlab
end
end
def verify_duplicates!(location)
if expandset.count >= MAX_INCLUDES
raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!"
end
# We scope location to context, as this allows us to properly support
# relative incldues, and similarly looking relative in another project
# does not trigger duplicate error
scoped_location = location.merge(
context_project: project,
context_sha: sha)
unless expandset.add?(scoped_location)
raise DuplicateIncludesError, "Include `#{location.to_json}` was already included!"
end
end
def select_first_matching(location)
matching = FILE_CLASSES.map do |file_class|
file_class.new(location, context)
......@@ -63,7 +91,7 @@ module Gitlab
def context
strong_memoize(:context) do
External::File::Base::Context.new(project, sha, user)
External::File::Base::Context.new(project, sha, user, expandset)
end
end
end
......
......@@ -7,11 +7,11 @@ module Gitlab
class Processor
IncludeError = Class.new(StandardError)
def initialize(values, project:, sha:, user:)
def initialize(values, project:, sha:, user:, expandset:)
@values = values
@external_files = External::Mapper.new(values, project: project, sha: sha, user: user).process
@external_files = External::Mapper.new(values, project: project, sha: sha, user: user, expandset: expandset).process
@content = {}
rescue External::Mapper::AmbigiousSpecificationError => e
rescue External::Mapper::Error => e
raise IncludeError, e.message
end
......
......@@ -3,7 +3,7 @@
require 'fast_spec_helper'
describe Gitlab::Ci::Config::External::File::Base do
let(:context) { described_class::Context.new(nil, 'HEAD', nil) }
let(:context) { described_class::Context.new(nil, 'HEAD', nil, Set.new) }
let(:test_class) do
Class.new(described_class) do
......@@ -79,4 +79,20 @@ describe Gitlab::Ci::Config::External::File::Base do
end
end
end
describe '#to_hash' do
context 'with includes' do
let(:location) { 'some/file/config.yml' }
let(:content) { 'include: { template: Bash.gitlab-ci.yml }'}
before do
allow_any_instance_of(test_class)
.to receive(:content).and_return(content)
end
it 'does expand hash to include the template' do
expect(subject.to_hash).to include(:before_script)
end
end
end
end
......@@ -4,8 +4,10 @@ require 'spec_helper'
describe Gitlab::Ci::Config::External::File::Local do
set(:project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:context) { described_class::Context.new(project, '12345', nil) }
let(:sha) { '12345' }
let(:context) { described_class::Context.new(project, sha, user, Set.new) }
let(:params) { { local: location } }
let(:local_file) { described_class.new(params, context) }
......@@ -103,4 +105,36 @@ describe Gitlab::Ci::Config::External::File::Local do
expect(local_file.error_message).to eq("Local file `#{location}` does not exist!")
end
end
describe '#expand_context' do
let(:location) { 'location.yml' }
subject { local_file.send(:expand_context) }
it 'inherits project, user and sha' do
is_expected.to include(user: user, project: project, sha: sha)
end
end
describe '#to_hash' do
context 'properly includes another local file in the same repository' do
let(:location) { 'some/file/config.yml' }
let(:content) { 'include: { local: another-config.yml }'}
let(:another_location) { 'another-config.yml' }
let(:another_content) { 'rspec: JOB' }
before do
allow(project.repository).to receive(:blob_data_at).with(sha, location)
.and_return(content)
allow(project.repository).to receive(:blob_data_at).with(sha, another_location)
.and_return(another_content)
end
it 'does expand hash to include the template' do
expect(local_file.to_hash).to include(:rspec)
end
end
end
end
......@@ -3,12 +3,13 @@
require 'spec_helper'
describe Gitlab::Ci::Config::External::File::Project do
set(:context_project) { create(:project) }
set(:project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:context_user) { user }
let(:context) { described_class::Context.new(nil, '12345', context_user) }
let(:subject) { described_class.new(params, context) }
let(:context) { described_class::Context.new(context_project, '12345', context_user, Set.new) }
let(:project_file) { described_class.new(params, context) }
before do
project.add_developer(user)
......@@ -19,7 +20,7 @@ describe Gitlab::Ci::Config::External::File::Project do
let(:params) { { file: 'file.yml', project: 'project' } }
it 'should return true' do
expect(subject).to be_matching
expect(project_file).to be_matching
end
end
......@@ -27,7 +28,7 @@ describe Gitlab::Ci::Config::External::File::Project do
let(:params) { { file: 'file.yml' } }
it 'should return false' do
expect(subject).not_to be_matching
expect(project_file).not_to be_matching
end
end
......@@ -35,7 +36,7 @@ describe Gitlab::Ci::Config::External::File::Project do
let(:params) { { project: 'project' } }
it 'should return false' do
expect(subject).not_to be_matching
expect(project_file).not_to be_matching
end
end
......@@ -43,7 +44,7 @@ describe Gitlab::Ci::Config::External::File::Project do
let(:params) { {} }
it 'should return false' do
expect(subject).not_to be_matching
expect(project_file).not_to be_matching
end
end
end
......@@ -61,15 +62,15 @@ describe Gitlab::Ci::Config::External::File::Project do
end
it 'should return true' do
expect(subject).to be_valid
expect(project_file).to be_valid
end
context 'when user does not have permission to access file' do
let(:context_user) { create(:user) }
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` not found or access denied!")
expect(project_file).not_to be_valid
expect(project_file.error_message).to include("Project `#{project.full_path}` not found or access denied!")
end
end
end
......@@ -86,7 +87,7 @@ describe Gitlab::Ci::Config::External::File::Project do
end
it 'should return true' do
expect(subject).to be_valid
expect(project_file).to be_valid
end
end
......@@ -102,8 +103,8 @@ describe Gitlab::Ci::Config::External::File::Project do
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` file `/file.yml` is empty!")
expect(project_file).not_to be_valid
expect(project_file.error_message).to include("Project `#{project.full_path}` file `/file.yml` is empty!")
end
end
......@@ -113,8 +114,8 @@ describe Gitlab::Ci::Config::External::File::Project do
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` reference `I-Do-Not-Exist` does not exist!")
expect(project_file).not_to be_valid
expect(project_file.error_message).to include("Project `#{project.full_path}` reference `I-Do-Not-Exist` does not exist!")
end
end
......@@ -124,8 +125,8 @@ describe Gitlab::Ci::Config::External::File::Project do
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` file `/invalid-file.yml` does not exist!")
expect(project_file).not_to be_valid
expect(project_file.error_message).to include("Project `#{project.full_path}` file `/invalid-file.yml` does not exist!")
end
end
......@@ -135,12 +136,22 @@ describe Gitlab::Ci::Config::External::File::Project do
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include('Included file `/invalid-file` does not have YAML extension!')
expect(project_file).not_to be_valid
expect(project_file.error_message).to include('Included file `/invalid-file` does not have YAML extension!')
end
end
end
describe '#expand_context' do
let(:params) { { file: 'file.yml', project: project.full_path, ref: 'master' } }
subject { project_file.send(:expand_context) }
it 'inherits user, and target project and sha' do
is_expected.to include(user: user, project: project, sha: project.commit('master').id)
end
end
private
def stub_project_blob(ref, path)
......
......@@ -3,7 +3,7 @@
require 'spec_helper'
describe Gitlab::Ci::Config::External::File::Remote do
let(:context) { described_class::Context.new(nil, '12345', nil) }
let(:context) { described_class::Context.new(nil, '12345', nil, Set.new) }
let(:params) { { remote: location } }
let(:remote_file) { described_class.new(params, context) }
let(:location) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
......@@ -181,4 +181,14 @@ describe Gitlab::Ci::Config::External::File::Remote do
end
end
end
describe '#expand_context' do
let(:params) { { remote: 'http://remote' } }
subject { remote_file.send(:expand_context) }
it 'drops all parameters' do
is_expected.to include(user: nil, project: nil, sha: nil)
end
end
end
......@@ -3,18 +3,21 @@
require 'spec_helper'
describe Gitlab::Ci::Config::External::File::Template do
let(:context) { described_class::Context.new(nil, '12345') }
set(:project) { create(:project) }
set(:user) { create(:user) }
let(:context) { described_class::Context.new(project, '12345', user, Set.new) }
let(:template) { 'Auto-DevOps.gitlab-ci.yml' }
let(:params) { { template: template } }
subject { described_class.new(params, context) }
let(:template_file) { described_class.new(params, context) }
describe '#matching?' do
context 'when a template is specified' do
let(:params) { { template: 'some-template' } }
it 'should return true' do
expect(subject).to be_matching
expect(template_file).to be_matching
end
end
......@@ -22,7 +25,7 @@ describe Gitlab::Ci::Config::External::File::Template do
let(:params) { { template: nil } }
it 'should return false' do
expect(subject).not_to be_matching
expect(template_file).not_to be_matching
end
end
......@@ -30,7 +33,7 @@ describe Gitlab::Ci::Config::External::File::Template do
let(:params) { {} }
it 'should return false' do
expect(subject).not_to be_matching
expect(template_file).not_to be_matching
end
end
end
......@@ -40,7 +43,7 @@ describe Gitlab::Ci::Config::External::File::Template do
let(:template) { 'Auto-DevOps.gitlab-ci.yml' }
it 'should return true' do
expect(subject).to be_valid
expect(template_file).to be_valid
end
end
......@@ -48,8 +51,8 @@ describe Gitlab::Ci::Config::External::File::Template do
let(:template) { 'Template.yml' }
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include('Template file `Template.yml` is not a valid location!')
expect(template_file).not_to be_valid
expect(template_file.error_message).to include('Template file `Template.yml` is not a valid location!')
end
end
......@@ -57,14 +60,14 @@ describe Gitlab::Ci::Config::External::File::Template do
let(:template) { 'I-Do-Not-Have-This-Template.gitlab-ci.yml' }
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include('Included file `I-Do-Not-Have-This-Template.gitlab-ci.yml` is empty or does not exist!')
expect(template_file).not_to be_valid
expect(template_file.error_message).to include('Included file `I-Do-Not-Have-This-Template.gitlab-ci.yml` is empty or does not exist!')
end
end
end
describe '#template_name' do
let(:template_name) { subject.send(:template_name) }
let(:template_name) { template_file.send(:template_name) }
context 'when template does end with .gitlab-ci.yml' do
let(:template) { 'my-template.gitlab-ci.yml' }
......@@ -90,4 +93,14 @@ describe Gitlab::Ci::Config::External::File::Template do
end
end
end
describe '#expand_context' do
let(:location) { 'location.yml' }
subject { template_file.send(:expand_context) }
it 'drops all parameters' do
is_expected.to include(user: nil, project: nil, sha: nil)
end
end
end
......@@ -9,6 +9,7 @@ describe Gitlab::Ci::Config::External::Mapper do
let(:local_file) { '/lib/gitlab/ci/templates/non-existent-file.yml' }
let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
let(:template_file) { 'Auto-DevOps.gitlab-ci.yml' }
let(:expandset) { Set.new }
let(:file_content) do
<<~HEREDOC
......@@ -21,7 +22,7 @@ describe Gitlab::Ci::Config::External::Mapper do
end
describe '#process' do
subject { described_class.new(values, project: project, sha: '123456', user: user).process }
subject { described_class.new(values, project: project, sha: '123456', user: user, expandset: expandset).process }
context "when single 'include' keyword is defined" do
context 'when the string is a local file' do
......@@ -141,5 +142,37 @@ describe Gitlab::Ci::Config::External::Mapper do
expect(subject).to be_empty
end
end
context "when duplicate 'include' is defined" do
let(:values) do
{ include: [
{ 'local' => local_file },
{ 'local' => local_file }
],
image: 'ruby:2.2' }
end
it 'raises an exception' do
expect { subject }.to raise_error(described_class::DuplicateIncludesError)
end
end
context "when too many 'includes' are defined" do
let(:values) do
{ include: [
{ 'local' => local_file },
{ 'remote' => remote_url }
],
image: 'ruby:2.2' }
end
before do
stub_const("#{described_class}::MAX_INCLUDES", 1)
end
it 'raises an exception' do
expect { subject }.to raise_error(described_class::TooManyIncludesError)
end
end
end
end
......@@ -4,15 +4,20 @@ require 'spec_helper'
describe Gitlab::Ci::Config::External::Processor do
set(:project) { create(:project, :repository) }
set(:another_project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:processor) { described_class.new(values, project: project, sha: '12345', user: user) }
let(:expandset) { Set.new }
let(:sha) { '12345' }
let(:processor) { described_class.new(values, project: project, sha: '12345', user: user, expandset: expandset) }
before do
project.add_developer(user)
end
describe "#perform" do
subject { processor.perform }
context 'when no external files defined' do
let(:values) { { image: 'ruby:2.2' } }
......@@ -190,5 +195,80 @@ describe Gitlab::Ci::Config::External::Processor do
expect(processor.perform[:image]).to eq('ruby:2.2')
end
end
context "when a nested includes are defined" do
let(:values) do
{
include: [
{ local: '/local/file.yml' }