Commit 2cfd0b59 authored by Douwe Maan's avatar Douwe Maan

Archive repositories in background worker.

parent 33a8f53f
web: bundle exec unicorn_rails -p ${PORT:="3000"} -E ${RAILS_ENV:="development"} -c ${UNICORN_CONFIG:="config/unicorn.rb"}
worker: bundle exec sidekiq -q post_receive -q mailer -q system_hook -q project_web_hook -q gitlab_shell -q common -q default
worker: bundle exec sidekiq -q post_receive -q mailer -q archive_repo -q system_hook -q project_web_hook -q gitlab_shell -q common -q default
......@@ -15,14 +15,18 @@ class Projects::RepositoriesController < Projects::ApplicationController
render_404 and return
end
file_path = ArchiveRepositoryService.new.execute(@project, params[:ref], params[:format])
begin
file_path = ArchiveRepositoryService.new(@project, params[:ref], params[:format]).execute
rescue
return render_404
end
if file_path
# Send file to user
response.headers["Content-Length"] = File.open(file_path).size.to_s
send_file file_path
else
render_404
redirect_to request.fullpath
end
end
end
......@@ -267,6 +267,9 @@ class Repository
# Remove archives older than 2 hours
def clean_old_archives
repository_downloads_path = Gitlab.config.gitlab.repository_downloads_path
return unless File.directory?(repository_downloads_path)
Gitlab::Popen.popen(%W(find #{repository_downloads_path} -not -path #{repository_downloads_path} -mmin +120 -delete))
end
......
class ArchiveRepositoryService
def execute(project, ref, format)
storage_path = Gitlab.config.gitlab.repository_downloads_path
attr_reader :project, :ref, :format
unless File.directory?(storage_path)
FileUtils.mkdir_p(storage_path)
def initialize(project, ref, format)
format ||= 'tar.gz'
@project, @ref, @format = project, ref, format
end
def execute
project.repository.clean_old_archives
raise "No archive file path" unless file_path
return file_path if archived?
unless archiving?
RepositoryArchiveWorker.perform_async(project.id, ref, format)
end
format ||= 'tar.gz'
repository = project.repository
repository.clean_old_archives
repository.archive_repo(ref, storage_path, format.downcase)
archived = wait_until_archived
file_path if archived
end
private
def storage_path
Gitlab.config.gitlab.repository_downloads_path
end
def archive_args
@archive_args ||= [ref, storage_path, format.downcase]
end
def file_path
@file_path ||= project.repository.archive_file_path(*archive_args)
end
def pid_file_path
@pid_file_path ||= project.repository.archive_pid_file_path(*archive_args)
end
def archived?
File.exist?(file_path)
end
def archiving?
File.exist?(pid_file_path)
end
def wait_until_archived
timeout = 5.0
t1 = Time.now
begin
sleep 0.1
success = archived?
t2 = Time.now
end until success || t2 - t1 >= timeout
success
end
end
class RepositoryArchiveWorker
include Sidekiq::Worker
sidekiq_options queue: :archive_repo
attr_accessor :project, :ref, :format
def perform(project_id, ref, format)
@project = Project.find(project_id)
@ref, @format = ref, format
repository = project.repository
repository.clean_old_archives
return if archived? || archiving?
repository.archive_repo(*archive_args)
end
private
def storage_path
Gitlab.config.gitlab.repository_downloads_path
end
def archive_args
@archive_args ||= [ref, storage_path, format.downcase]
end
def file_path
@file_path ||= project.repository.archive_file_path(*archive_args)
end
def pid_file_path
@pid_file_path ||= project.repository.archive_pid_file_path(*archive_args)
end
def archived?
File.exist?(file_path)
end
def archiving?
File.exist?(pid_file_path)
end
end
......@@ -133,10 +133,11 @@ module API
authorize! :download_code, user_project
begin
file_path = ArchiveRepositoryService.new.execute(
user_project,
params[:sha],
params[:format])
file_path = ArchiveRepositoryService.new(
user_project,
params[:sha],
params[:format]
).execute
rescue
not_found!('File')
end
......@@ -149,7 +150,7 @@ module API
env['api.format'] = :binary
present data
else
not_found!('File')
redirect request.fullpath
end
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment