Commit c4d615c9 authored by Kamil Trzciński's avatar Kamil Trzciński

Allow to include files from another projects

This adds `project:, file:, ref:` specification support.
parent b97b85c3
......@@ -8,7 +8,7 @@ class Projects::Ci::LintsController < Projects::ApplicationController
def create
@content = params[:content]
@error = Gitlab::Ci::YamlProcessor.validation_message(@content, yaml_processor_options)
@error = Gitlab::Ci::YamlProcessor.validation_message(@content, yaml_processor_options)
@status = @error.blank?
if @error.blank?
......@@ -24,6 +24,10 @@ class Projects::Ci::LintsController < Projects::ApplicationController
private
def yaml_processor_options
{ project: @project, sha: project.repository.commit.sha }
{
project: @project,
user: current_user,
sha: project.repository.commit.sha
}
end
end
......@@ -10,16 +10,16 @@ module BlobViewer
self.file_types = %i(gitlab_ci)
self.binary = false
def validation_message(project, sha)
def validation_message(opts)
return @validation_message if defined?(@validation_message)
prepare!
@validation_message = Gitlab::Ci::YamlProcessor.validation_message(blob.data, { project: project, sha: sha })
@validation_message = Gitlab::Ci::YamlProcessor.validation_message(blob.data, opts)
end
def valid?(project, sha)
validation_message(project, sha).blank?
def valid?(opts)
validation_message(opts).blank?
end
end
end
......@@ -496,7 +496,7 @@ module Ci
return @config_processor if defined?(@config_processor)
@config_processor ||= begin
::Gitlab::Ci::YamlProcessor.new(ci_yaml_file, { project: project, sha: sha })
::Gitlab::Ci::YamlProcessor.new(ci_yaml_file, { project: project, sha: sha, user: user })
rescue Gitlab::Ci::YamlProcessor::ValidationError => e
self.yaml_errors = e.message
nil
......
- if viewer.valid?(@project, @commit.sha)
- if viewer.valid?(project: @project, sha: @commit.sha, user: @current_user)
= icon('check fw')
This GitLab CI configuration is valid.
- else
= icon('warning fw')
This GitLab CI configuration is invalid:
= viewer.validation_message(@project, @commit.sha)
= viewer.validation_message(project: @project, sha: @commit.sha, user: @current_user)
= link_to 'Learn more', help_page_path('ci/yaml/README')
---
title: Allow to include files from another projects in gitlab-ci.yml
merge_request: 24101
author:
type: added
......@@ -1649,7 +1649,7 @@ test:
> Behaviour expanded in GitLab 10.8 to allow more flexible overriding.
> [Moved](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21603)
to GitLab Core in 11.4
> In GitLab 11.7, support for including [GitLab-supplied templates](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/lib/gitlab/ci/templates) directly [was added](https://gitlab.com/gitlab-org/gitlab-ce/issues/53445).
> In GitLab 11.7, support for [including GitLab-supplied templates directly](https://gitlab.com/gitlab-org/gitlab-ce/issues/53445) and support for [including templates from another repository](https://gitlab.com/gitlab-org/gitlab-ce/issues/53903) was added.
Using the `include` keyword, you can allow the inclusion of external YAML files.
......@@ -1724,7 +1724,7 @@ include:
---
`include` supports three types of files:
`include` supports four types of files:
- **local** to the same repository, referenced by using full paths in the same
repository, with `/` being the root directory. For example:
......@@ -1750,6 +1750,32 @@ include:
NOTE: **Note:**
We don't support the inclusion of local files through Git submodules paths.
- **file** from another repository, referenced by using full paths in the same
repository, with `/` being the root directory. For example:
```yaml
include:
project: 'my-group/my-project'
file: '/templates/.gitlab-ci-template.yml'
```
You can also specify `ref:`. The default `ref:` is the `HEAD` of the project:
```yaml
include:
- project: 'my-group/my-project'
ref: master
file: '/templates/.gitlab-ci-template.yml'
- project: 'my-group/my-project'
ref: v1.0.0
file: '/templates/.gitlab-ci-template.yml'
- project: 'my-group/my-project'
ref: 787123b47f14b552955ca2786bc9542ae66fee5b # git sha
file: '/templates/.gitlab-ci-template.yml'
```
- **remote** in a different location, accessed using HTTP/HTTPS, referenced
using the full URL. For example:
......
......@@ -8,7 +8,8 @@ module API
requires :content, type: String, desc: 'Content of .gitlab-ci.yml'
end
post '/lint' do
error = Gitlab::Ci::YamlProcessor.validation_message(params[:content])
error = Gitlab::Ci::YamlProcessor.validation_message(params[:content],
user: current_user)
status 200
......
......@@ -8,9 +8,9 @@ module Gitlab
class Config
ConfigError = Class.new(StandardError)
def initialize(config, opts = {})
def initialize(config, project: nil, sha: nil, user: nil)
@config = Config::Extendable
.new(build_config(config, opts))
.new(build_config(config, project: project, sha: sha, user: user))
.to_hash
@global = Entry::Global.new(@config)
......@@ -70,20 +70,21 @@ module Gitlab
private
def build_config(config, opts = {})
def build_config(config, project:, sha:, user:)
initial_config = Gitlab::Config::Loader::Yaml.new(config).load!
project = opts.fetch(:project, nil)
if project
process_external_files(initial_config, project, opts)
process_external_files(initial_config, project: project, sha: sha, user: user)
else
initial_config
end
end
def process_external_files(config, project, opts)
sha = opts.fetch(:sha) { project.repository.root_ref_sha }
Config::External::Processor.new(config, project: project, sha: sha).perform
def process_external_files(config, project:, sha:, user:)
Config::External::Processor.new(config,
project: project,
sha: sha || project.repository.root_ref_sha,
user: user).perform
end
end
end
......
......@@ -12,7 +12,7 @@ module Gitlab
YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze
Context = Struct.new(:project, :sha)
Context = Struct.new(:project, :sha, :user)
def initialize(params, context)
@params = params
......
# frozen_string_literal: true
module Gitlab
module Ci
class Config
module External
module File
class Project < Base
include Gitlab::Utils::StrongMemoize
attr_reader :project_name, :ref_name
def initialize(params, context = {})
@location = params[:file]
@project_name = params[:project]
@ref_name = params[:ref] || 'HEAD'
super
end
def matching?
super && project_name.present?
end
def content
strong_memoize(:content) { fetch_local_content }
end
private
def validate_content!
if !can_access_local_content?
errors.push("Project `#{project_name}` not found or access denied!")
elsif sha.nil?
errors.push("Project `#{project_name}` reference `#{ref_name}` does not exist!")
elsif content.nil?
errors.push("Project `#{project_name}` file `#{location}` does not exist!")
elsif content.blank?
errors.push("Project `#{project_name}` file `#{location}` is empty!")
end
end
def project
strong_memoize(:project) do
::Project.find_by_full_path(project_name)
end
end
def can_access_local_content?
Ability.allowed?(context.user, :download_code, project)
end
def fetch_local_content
return unless can_access_local_content?
return unless sha
project.repository.blob_data_at(sha, location)
rescue GRPC::NotFound, GRPC::Internal
nil
end
def sha
strong_memoize(:sha) do
project.commit(ref_name).try(:sha)
end
end
end
end
end
end
end
end
......@@ -10,15 +10,17 @@ module Gitlab
FILE_CLASSES = [
External::File::Remote,
External::File::Template,
External::File::Local
External::File::Local,
External::File::Project
].freeze
AmbigiousSpecificationError = Class.new(StandardError)
def initialize(values, project:, sha:)
def initialize(values, project:, sha:, user:)
@locations = Array.wrap(values.fetch(:include, []))
@project = project
@sha = sha
@user = user
end
def process
......@@ -61,7 +63,7 @@ module Gitlab
def context
strong_memoize(:context) do
External::File::Base::Context.new(project, sha)
External::File::Base::Context.new(project, sha, user)
end
end
end
......
......@@ -7,9 +7,9 @@ module Gitlab
class Processor
IncludeError = Class.new(StandardError)
def initialize(values, project:, sha:)
def initialize(values, project:, sha:, user:)
@values = values
@external_files = External::Mapper.new(values, project: project, sha: sha).process
@external_files = External::Mapper.new(values, project: project, sha: sha, user: user).process
@content = {}
rescue External::Mapper::AmbigiousSpecificationError => e
raise IncludeError, e.message
......
......@@ -10,7 +10,7 @@ module Gitlab
attr_reader :cache, :stages, :jobs
def initialize(config, opts = {})
@ci_config = Gitlab::Ci::Config.new(config, opts)
@ci_config = Gitlab::Ci::Config.new(config, **opts)
@config = @ci_config.to_hash
unless @ci_config.valid?
......
......@@ -3,7 +3,7 @@
require 'fast_spec_helper'
describe Gitlab::Ci::Config::External::File::Base do
let(:context) { described_class::Context.new(nil, 'HEAD') }
let(:context) { described_class::Context.new(nil, 'HEAD', nil) }
let(:test_class) do
Class.new(described_class) do
......
......@@ -5,7 +5,7 @@ require 'spec_helper'
describe Gitlab::Ci::Config::External::File::Local do
set(:project) { create(:project, :repository) }
let(:context) { described_class::Context.new(project, '12345') }
let(:context) { described_class::Context.new(project, '12345', nil) }
let(:params) { { local: location } }
let(:local_file) { described_class.new(params, context) }
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Config::External::File::Project do
set(:project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:context_user) { user }
let(:context) { described_class::Context.new(nil, '12345', context_user) }
let(:subject) { described_class.new(params, context) }
before do
project.add_developer(user)
end
describe '#matching?' do
context 'when a file and project is specified' do
let(:params) { { file: 'file.yml', project: 'project' } }
it 'should return true' do
expect(subject).to be_matching
end
end
context 'with only file is specified' do
let(:params) { { file: 'file.yml' } }
it 'should return false' do
expect(subject).not_to be_matching
end
end
context 'with only project is specified' do
let(:params) { { project: 'project' } }
it 'should return false' do
expect(subject).not_to be_matching
end
end
context 'with a missing local key' do
let(:params) { {} }
it 'should return false' do
expect(subject).not_to be_matching
end
end
end
describe '#valid?' do
context 'when a valid path is used' do
let(:params) do
{ project: project.full_path, file: '/file.yml' }
end
let(:root_ref_sha) { project.repository.root_ref_sha }
before do
stub_project_blob(root_ref_sha, '/file.yml') { 'image: ruby:2.1' }
end
it 'should return true' do
expect(subject).to be_valid
end
context 'when user does not have permission to access file' do
let(:context_user) { create(:user) }
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` not found or access denied!")
end
end
end
context 'when a valid path with custom ref is used' do
let(:params) do
{ project: project.full_path, ref: 'master', file: '/file.yml' }
end
let(:ref_sha) { project.commit('master').sha }
before do
stub_project_blob(ref_sha, '/file.yml') { 'image: ruby:2.1' }
end
it 'should return true' do
expect(subject).to be_valid
end
end
context 'when an empty file is used' do
let(:params) do
{ project: project.full_path, file: '/file.yml' }
end
let(:root_ref_sha) { project.repository.root_ref_sha }
before do
stub_project_blob(root_ref_sha, '/file.yml') { '' }
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` file `/file.yml` is empty!")
end
end
context 'when non-existing ref is used' do
let(:params) do
{ project: project.full_path, ref: 'I-Do-Not-Exist', file: '/file.yml' }
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` reference `I-Do-Not-Exist` does not exist!")
end
end
context 'when non-existing file is requested' do
let(:params) do
{ project: project.full_path, file: '/invalid-file.yml' }
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include("Project `#{project.full_path}` file `/invalid-file.yml` does not exist!")
end
end
context 'when file is not a yaml file' do
let(:params) do
{ project: project.full_path, file: '/invalid-file' }
end
it 'should return false' do
expect(subject).not_to be_valid
expect(subject.error_message).to include('Included file `/invalid-file` does not have YAML extension!')
end
end
end
private
def stub_project_blob(ref, path)
allow_any_instance_of(Repository)
.to receive(:blob_data_at)
.with(ref, path) { yield }
end
end
......@@ -3,7 +3,7 @@
require 'spec_helper'
describe Gitlab::Ci::Config::External::File::Remote do
let(:context) { described_class::Context.new(nil, '12345') }
let(:context) { described_class::Context.new(nil, '12345', nil) }
let(:params) { { remote: location } }
let(:remote_file) { described_class.new(params, context) }
let(:location) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
......
......@@ -4,6 +4,7 @@ require 'spec_helper'
describe Gitlab::Ci::Config::External::Mapper do
set(:project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:local_file) { '/lib/gitlab/ci/templates/non-existent-file.yml' }
let(:remote_url) { 'https://gitlab.com/gitlab-org/gitlab-ce/blob/1234/.gitlab-ci-1.yml' }
......@@ -20,7 +21,7 @@ describe Gitlab::Ci::Config::External::Mapper do
end
describe '#process' do
subject { described_class.new(values, project: project, sha: '123456').process }
subject { described_class.new(values, project: project, sha: '123456', user: user).process }
context "when single 'include' keyword is defined" do
context 'when the string is a local file' do
......
......@@ -4,8 +4,13 @@ require 'spec_helper'
describe Gitlab::Ci::Config::External::Processor do
set(:project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:processor) { described_class.new(values, project: project, sha: '12345') }
let(:processor) { described_class.new(values, project: project, sha: '12345', user: user) }
before do
project.add_developer(user)
end
describe "#perform" do
context 'when no external files defined' do
......
require 'spec_helper'
describe Gitlab::Ci::Config do
set(:user) { create(:user) }
let(:config) do
described_class.new(yml)
described_class.new(yml, project: nil, sha: nil, user: nil)
end
context 'when config is valid' do
......@@ -154,7 +156,7 @@ describe Gitlab::Ci::Config do
end
let(:config) do
described_class.new(gitlab_ci_yml, project: project, sha: '12345')
described_class.new(gitlab_ci_yml, project: project, sha: '12345', user: user)
end
before do
......@@ -228,7 +230,7 @@ describe Gitlab::Ci::Config do
expect(project.repository).to receive(:blob_data_at)
.with('eeff1122', local_location)
described_class.new(gitlab_ci_yml, project: project, sha: 'eeff1122')
described_class.new(gitlab_ci_yml, project: project, sha: 'eeff1122', user: user)
end
end
......@@ -236,7 +238,7 @@ describe Gitlab::Ci::Config do
it 'is using latest SHA on the default branch' do
expect(project.repository).to receive(:root_ref_sha)
described_class.new(gitlab_ci_yml, project: project)
described_class.new(gitlab_ci_yml, project: project, sha: nil, user: user)
end
end
end
......
......@@ -3,10 +3,10 @@ require 'spec_helper'
module Gitlab
module Ci
describe YamlProcessor do
subject { described_class.new(config) }
subject { described_class.new(config, user: nil) }
describe '#build_attributes' do
subject { described_class.new(config).build_attributes(:rspec) }
subject { described_class.new(config, user: nil).build_attributes(:rspec) }
describe 'attributes list' do
let(:config) do
......
......@@ -4,7 +4,9 @@ describe BlobViewer::GitlabCiYml do
include FakeBlobHelpers
include RepoHelpers
let(:project) { create(:project, :repository) }
set(:project) { create(:project, :repository) }
set(:user) { create(:user) }
let(:data) { File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml')) }
let(:blob) { fake_blob(path: '.gitlab-ci.yml', data: data) }
let(:sha) { sample_commit.id }
......@@ -14,12 +16,12 @@ describe BlobViewer::GitlabCiYml do
it 'calls prepare! on the viewer' do
expect(subject).to receive(:prepare!)
subject.validation_message(project, sha)
subject.validation_message(project: project, sha: sha, user: user)
end
context 'when the configuration is valid' do
it 'returns nil' do
expect(subject.validation_message(project, sha)).to be_nil
expect(subject.validation_message(project: project, sha: sha, user: user)).to be_nil
end
end
......@@ -27,7 +29,7 @@ describe BlobViewer::GitlabCiYml do
let(:data) { 'oof' }
it 'returns the error message' do
expect(subject.validation_message(project, sha)).to eq('Invalid configuration format')
expect(subject.validation_message(project: project, sha: sha, user: user)).to eq('Invalid configuration format')
end
end
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment