Commit 045d07ba authored by Kamil Trzciński's avatar Kamil Trzciński

Add Container Registry API

This includes a set of APIs to manipulate container registry.
This includes also an ability to delete tags based on requested
criteria, like keep-last-n, matching-name, older-than.
parent 267ce96e
# frozen_string_literal: true
class ContainerRepository < ActiveRecord::Base
include Gitlab::Utils::StrongMemoize
belongs_to :project
validates :name, length: { minimum: 0, allow_nil: false }
......@@ -8,6 +10,8 @@ class ContainerRepository < ActiveRecord::Base
delegate :client, to: :registry
scope :ordered, -> { order(:name) }
# rubocop: disable CodeReuse/ServiceClass
def registry
@registry ||= begin
......@@ -39,11 +43,12 @@ class ContainerRepository < ActiveRecord::Base
end
def tags
return @tags if defined?(@tags)
return [] unless manifest && manifest['tags']
@tags = manifest['tags'].map do |tag|
ContainerRegistry::Tag.new(self, tag)
strong_memoize(:tags) do
manifest['tags'].sort.map do |tag|
ContainerRegistry::Tag.new(self, tag)
end
end
end
......
# frozen_string_literal: true
class ContainerRepositoryPolicy < BasePolicy
delegate { @subject.project }
end
......@@ -3,7 +3,7 @@
class ContainerRepositoryEntity < Grape::Entity
include RequestAwareEntity
expose :id, :path, :location
expose :id, :name, :path, :location, :created_at
expose :tags_path do |repository|
project_registry_repository_tags_path(project, repository, format: :json)
......
......@@ -3,7 +3,7 @@
class ContainerTagEntity < Grape::Entity
include RequestAwareEntity
expose :name, :location, :revision, :short_revision, :total_size, :created_at
expose :name, :path, :location, :digest, :revision, :short_revision, :total_size, :created_at
expose :destroy_path, if: -> (*) { can_destroy? } do |tag|
project_registry_repository_tag_path(project, tag.repository, tag.name)
......
......@@ -6,9 +6,14 @@
#
# `#try_obtain_lease` takes a block which will be run if it was able to
# obtain the lease. Implement `#lease_timeout` to configure the timeout
# for the exclusive lease. Optionally override `#lease_key` to set the
# for the exclusive lease.
#
# Optionally override `#lease_key` to set the
# lease key, it defaults to the class name with underscores.
#
# Optionally override `#lease_release?` to prevent the job to
# be re-executed more often than LEASE_TIMEOUT.
#
module ExclusiveLeaseGuard
extend ActiveSupport::Concern
......@@ -23,7 +28,7 @@ module ExclusiveLeaseGuard
begin
yield lease
ensure
release_lease(lease)
release_lease(lease) if lease_release?
end
end
......@@ -40,6 +45,10 @@ module ExclusiveLeaseGuard
"#{self.class.name} does not implement #{__method__}"
end
def lease_release?
true
end
def release_lease(uuid)
Gitlab::ExclusiveLease.cancel(lease_key, uuid)
end
......
# frozen_string_literal: true
module Projects
module ContainerRepository
class CleanupTagsService < BaseService
def execute(container_repository)
return error('feature disabled') unless can_use?
return error('access denied') unless can_admin?
tags = container_repository.tags
tags_by_digest = group_by_digest(tags)
tags = without_latest(tags)
tags = filter_by_name(tags)
tags = with_manifest(tags)
tags = order_by_date(tags)
tags = filter_keep_n(tags)
tags = filter_by_older_than(tags)
deleted_tags = delete_tags(tags, tags_by_digest)
success(deleted: deleted_tags.map(&:name))
end
private
def delete_tags(tags_to_delete, tags_by_digest)
deleted_digests = group_by_digest(tags_to_delete).select do |digest, tags|
delete_tag_digest(digest, tags, tags_by_digest[digest])
end
deleted_digests.values.flatten
end
def delete_tag_digest(digest, tags, other_tags)
# Issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
# we have to remove all tags due
# to Docker Distribution bug unable
# to delete single tag
return unless tags.count == other_tags.count
# delete all tags
tags.map(&:delete)
end
def group_by_digest(tags)
tags.group_by(&:digest)
end
def without_latest(tags)
tags.reject(&:latest?)
end
def with_manifest(tags)
tags.select(&:valid?)
end
def order_by_date(tags)
now = DateTime.now
tags.sort_by { |tag| tag.created_at || now }.reverse
end
def filter_by_name(tags)
regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z")
tags.select do |tag|
regex.scan(tag.name).any?
end
end
def filter_keep_n(tags)
tags.drop(params['keep_n'].to_i)
end
def filter_by_older_than(tags)
return tags unless params['older_than']
older_than = ChronicDuration.parse(params['older_than']).seconds.ago
tags.select do |tag|
tag.created_at && tag.created_at < older_than
end
end
def can_admin?
can?(current_user, :admin_container_image, project)
end
def can_use?
Feature.enabled?(:container_registry_cleanup, project, default_enabled: true)
end
end
end
end
......@@ -90,13 +90,15 @@
- object_pool:object_pool_join
- object_pool:object_pool_destroy
- container_repository:delete_container_repository
- container_repository:cleanup_container_repository
- default
- mailers # ActionMailer::DeliveryJob.queue_name
- authorized_projects
- background_migration
- create_gpg_signature
- delete_container_repository
- delete_merged_branches
- delete_user
- email_receiver
......
# frozen_string_literal: true
class CleanupContainerRepositoryWorker
include ApplicationWorker
include ExclusiveLeaseGuard
queue_namespace :container_repository
LEASE_TIMEOUT = 1.hour
attr_reader :container_repository, :current_user
def perform(current_user_id, container_repository_id, params)
@current_user = User.find_by_id(current_user_id)
@container_repository = ContainerRepository.find_by_id(container_repository_id)
return unless valid?
try_obtain_lease do
Projects::ContainerRepository::CleanupTagsService
.new(project, current_user, params)
.execute(container_repository)
end
end
private
def valid?
current_user && container_repository && project
end
def project
container_repository&.project
end
# For ExclusiveLeaseGuard concern
def lease_key
@lease_key ||= "container_repository:cleanup_tags:#{container_repository.id}"
end
# For ExclusiveLeaseGuard concern
def lease_timeout
LEASE_TIMEOUT
end
# For ExclusiveLeaseGuard concern
def lease_release?
# we don't allow to execute this worker
# more often than LEASE_TIMEOUT
# for given container repository
false
end
end
......@@ -4,6 +4,8 @@ class DeleteContainerRepositoryWorker
include ApplicationWorker
include ExclusiveLeaseGuard
queue_namespace :container_repository
LEASE_TIMEOUT = 1.hour
attr_reader :container_repository
......
---
title: Add Container Registry API with cleanup function
merge_request: 24303
author:
type: added
......@@ -47,7 +47,6 @@
- [project_service, 1]
- [delete_user, 1]
- [todos_destroyer, 1]
- [delete_container_repository, 1]
- [delete_merged_branches, 1]
- [authorized_projects, 1]
- [expire_build_instance_artifacts, 1]
......@@ -81,6 +80,7 @@
- [delete_diff_files, 1]
- [detect_repository_languages, 1]
- [auto_devops, 2]
- [container_repository, 1]
- [object_pool, 1]
- [repository_cleanup, 1]
- [delete_stored_files, 1]
......
# frozen_string_literal: true
class MigrateDeleteContainerRepositoryWorker < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
sidekiq_queue_migrate('delete_container_repository', to: 'container_repository:delete_container_repository')
end
def down
sidekiq_queue_migrate('container_repository:delete_container_repository', to: 'delete_container_repository')
end
end
......@@ -16,6 +16,7 @@ The following API resources are available:
- [Broadcast messages](broadcast_messages.md)
- [Code snippets](snippets.md)
- [Commits](commits.md)
- [Container Registry](container_registry.md)
- [Custom attributes](custom_attributes.md)
- [Deploy keys](deploy_keys.md), and [deploy keys for multiple projects](deploy_key_multiple_projects.md)
- [Deployments](deployments.md)
......
# Container Registry API
## List registry repositories
Get a list of registry repositories in a project.
```
GET /projects/:id/registry/repositories
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories"
```
Example response:
```json
[
{
"id": 1,
"name": "",
"path": "group/project",
"location": "gitlab.example.com:5000/group/project",
"created_at": "2019-01-10T13:38:57.391Z"
},
{
"id": 2,
"name": "releases",
"path": "group/project/releases",
"location": "gitlab.example.com:5000/group/project/releases",
"created_at": "2019-01-10T13:39:08.229Z"
}
]
```
## Delete registry repository
Get a list of repository commits in a project.
This operation is executed asynchronously and it might take
time to get executed.
```
DELETE /projects/:id/registry/repositories/:repository_id
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
| `repository_id` | integer | yes | The ID of registry repository
```bash
curl -X DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2"
```
## List repository tags
Get a list of tags for given registry repository.
```
GET /projects/:id/registry/repositories/:repository_id/tags
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
| `repository_id` | integer | yes | The ID of registry repository
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
Example response:
```json
[
{
"name": "A",
"path": "group/project:A",
"location": "gitlab.example.com:5000/group/project:A"
},
{
"name": "latest",
"path": "group/project:latest",
"location": "gitlab.example.com:5000/group/project:latest"
}
]
```
## Delete repository tags (in bulk)
Delete repository tags in bulk based on given criteria.
This API performs a following set of the operations:
1. It schedules asynchronous job executed in background,
1. It never removes tag named `latest`,
1. It removes the tags matching given `name_regex` only,
1. It orders all tags by creation date. The creation date is time of the manifest creation. It is not a time of tag push,
1. It keeps N latest matching tags (if specified),
1. It only removes tags that are older than (if specified).
These operations are executed asynchronously and it might
take time to get executed. This API can be run at most
once an hour for given container repository.
Due to [Docker Distribution deficiency](ce-21405) it does
not remove tags whose manifest is shared by multiple tags
```
DELETE /projects/:id/registry/repositories/:repository_id/tags
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
| `repository_id` | integer | yes | The ID of registry repository
| `name_regex` | string | yes | The regex of the name to delete. To delete all tags specify `.*`
| `keep_n` | integer | no | The amount of latest tags of given name to keep
| `older_than` | string | no | Tags to delete that are older than given timespec, written in human readable form `1h`, `1d`, `1month` |
Examples:
1. Remove tag names that are matching GIT SHA, keep always at least 5, and remove ones that are older than 2 days:
```bash
curl -X DELETE -F 'name_regex=[0-9a-z]{40}' -F 'keep_n=5' -F 'older_than=2d' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
2. Remove all tags, but keep always latest 5:
```bash
curl -X DELETE -F 'name_regex=.*' -F 'keep_n=5' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
3. Remove all tags that are older than 1 month:
```bash
curl -X DELETE -F 'name_regex=.*' -F 'older_than=1month' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags"
```
## Get a details repository tag
Get a details of registry repository tag
```
GET /projects/:id/registry/repositories/:repository_id/tags/:tag_name
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
| `repository_id` | integer | yes | The ID of registry repository
| `tag_name` | string | yes | The name of tag
```bash
curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0"
```
Example response:
```json
{
"name": "v10.0.0",
"path": "group/project:latest",
"location": "gitlab.example.com:5000/group/project:latest",
"revision": "e9ed9d87c881d8c2fd3a31b41904d01ba0b836e7fd15240d774d811a1c248181",
"short_revision": "e9ed9d87c",
"digest": "sha256:c3490dcf10ffb6530c1303522a1405dfaf7daecd8f38d3e6a1ba19ea1f8a1751",
"created_at": "2019-01-06T16:49:51.272+00:00",
"total_size": 350224384
}
```
## Delete a repository tag
Delete a registry repository tag
```
DELETE /projects/:id/registry/repositories/:repository_id/tags/:tag_name
```
| Attribute | Type | Required | Description |
| --------- | ---- | -------- | ----------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user
| `repository_id` | integer | yes | The ID of registry repository
| `tag_name` | string | yes | The name of tag
```bash
curl -X DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags/v10.0.0"
```
[ce-21405]: https://gitlab.com/gitlab-org/gitlab-ce/issues/21405
......@@ -100,6 +100,7 @@ module API
mount ::API::CircuitBreakers
mount ::API::Commits
mount ::API::CommitStatuses
mount ::API::ContainerRegistry
mount ::API::DeployKeys
mount ::API::Deployments
mount ::API::Environments
......
# frozen_string_literal: true
module API
class ContainerRegistry < Grape::API
include PaginationParams
REGISTRY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(
tag_name: API::NO_SLASH_URL_PART_REGEX)
before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) }
before { authorize_read_container_images! }
params do
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get a project container repositories' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::Repository
end
params do
use :pagination
end
get ':id/registry/repositories' do
repositories = user_project.container_repositories.ordered
present paginate(repositories), with: Entities::ContainerRegistry::Repository
end
desc 'Delete repository' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
end
delete ':id/registry/repositories/:repository_id', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_admin_container_image!
DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id)
status :accepted
end
desc 'Get a list of repositories tags' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::Tag
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
use :pagination
end
get ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_read_container_image!
tags = Kaminari.paginate_array(repository.tags)
present paginate(tags), with: Entities::ContainerRegistry::Tag
end
desc 'Delete repository tags (in bulk)' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all'
optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name'
optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month'
end
delete ':id/registry/repositories/:repository_id/tags', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_admin_container_image!
CleanupContainerRepositoryWorker.perform_async(current_user.id, repository.id,
declared_params.except(:repository_id)) # rubocop: disable CodeReuse/ActiveRecord
status :accepted
end
desc 'Get a details about repository tag' do
detail 'This feature was introduced in GitLab 11.8.'
success Entities::ContainerRegistry::TagDetails
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :tag_name, type: String, desc: 'The name of the tag'
end
get ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_read_container_image!
validate_tag!
present tag, with: Entities::ContainerRegistry::TagDetails
end
desc 'Delete repository tag' do
detail 'This feature was introduced in GitLab 11.8.'
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
requires :tag_name, type: String, desc: 'The name of the tag'
end
delete ':id/registry/repositories/:repository_id/tags/:tag_name', requirements: REGISTRY_ENDPOINT_REQUIREMENTS do
authorize_destroy_container_image!
validate_tag!
tag.delete
status :ok
end
end
helpers do
def authorize_read_container_images!
authorize! :read_container_image, user_project
end
def authorize_read_container_image!
authorize! :read_container_image, repository
end
def authorize_update_container_image!
authorize! :update_container_image, repository
end
def authorize_destroy_container_image!
authorize! :admin_container_image, repository
end
def authorize_admin_container_image!
authorize! :admin_container_image, repository
end
def repository
@repository ||= user_project.container_repositories.find(params[:repository_id])
end
def tag
@tag ||= repository.tag(params[:tag_name])
end
def validate_tag!
not_found!('Tag') unless tag.valid?
end
end
end
end
# frozen_string_literal: true
module API
module Entities
module ContainerRegistry
class Repository < Grape::Entity
expose :id
expose :name
expose :path
expose :location
expose :created_at
end
class Tag < Grape::Entity
expose :name
expose :path
expose :location
end
class TagDetails < Tag
expose :revision
expose :short_revision
expose :digest
expose :created_at
expose :total_size
end
end
end
end
......@@ -2,6 +2,8 @@
module ContainerRegistry
class Tag
include Gitlab::Utils::StrongMemoize
attr_reader :repository, :name
delegate :registry, :client, to: :repository
......@@ -15,6 +17,10 @@ module ContainerRegistry
manifest.present?
end
def latest?
name == "latest"
end
def v1?
manifest && manifest['schemaVersion'] == 1
end
......@@ -24,7 +30,9 @@ module ContainerRegistry
end
def manifest
@manifest ||= client.repository_manifest(repository.path, name)
strong_memoize(:manifest) do
client.repository_manifest(repository.path, name)
end
end
def path
......@@ -42,36 +50,44 @@ module ContainerRegistry
end