Commit a00578ce authored by Robert Speicher's avatar Robert Speicher

Absorb gitlab_git

parent aec04a47
......@@ -16,6 +16,8 @@ gem 'default_value_for', '~> 3.0.0'
gem 'mysql2', '~> 0.3.16', group: :mysql
gem 'pg', '~> 0.18.2', group: :postgres
gem 'rugged', '~> 0.24.0'
# Authentication libraries
gem 'devise', '~> 4.2'
gem 'doorkeeper', '~> 4.2.0'
......@@ -49,10 +51,6 @@ gem 'u2f', '~> 0.2.1'
# Browser detection
gem 'browser', '~> 2.2'
# Extracting information from a git repository
# Provide access to Gitlab::Git library
gem 'gitlab_git', '~> 10.7.0'
# LDAP Auth
# GitLab fork with several improvements to original library. For full list of changes
# see https://github.com/intridea/omniauth-ldap/compare/master...gitlabhq:master
......
......@@ -255,11 +255,6 @@ GEM
mime-types (>= 1.16, < 3)
posix-spawn (~> 0.3)
gitlab-markup (1.5.0)
gitlab_git (10.7.0)
activesupport (~> 4.0)
charlock_holmes (~> 0.7.3)
github-linguist (~> 4.7.0)
rugged (~> 0.24.0)
gitlab_omniauth-ldap (1.2.1)
net-ldap (~> 0.9)
omniauth (~> 1.0)
......@@ -857,7 +852,6 @@ DEPENDENCIES
github-linguist (~> 4.7.0)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-markup (~> 1.5.0)
gitlab_git (~> 10.7.0)
gitlab_omniauth-ldap (~> 1.2.1)
gollum-lib (~> 4.2)
gollum-rugged_adapter (~> 0.4.2)
......@@ -942,6 +936,7 @@ DEPENDENCIES
rubocop-rspec (~> 1.5.0)
ruby-fogbugz (~> 0.2.1)
ruby-prof (~> 0.16.2)
rugged (~> 0.24.0)
sanitize (~> 2.0)
sass-rails (~> 5.0.6)
scss_lint (~> 0.47.0)
......@@ -988,4 +983,4 @@ DEPENDENCIES
wikicloth (= 0.8.1)
BUNDLED WITH
1.13.6
1.13.7
class MergeRequestDiff < ActiveRecord::Base
include Sortable
include Importable
include EncodingHelper
include Gitlab::Git::EncodingHelper
# Prevent store of diff if commits amount more then 500
COMMITS_SAFE_SIZE = 100
......
module Gitlab
module Git
# Class for parsing Git attribute files and extracting the attributes for
# file patterns.
#
# Unlike Rugged this parser only needs a single IO call (a call to `open`),
# vastly reducing the time spent in extracting attributes.
#
# This class _only_ supports parsing the attributes file located at
# `$GIT_DIR/info/attributes` as GitLab doesn't use any other files
# (`.gitattributes` is copied to this particular path).
#
# Basic usage:
#
# attributes = Gitlab::Git::Attributes.new(some_repo.path)
#
# attributes.attributes('README.md') # => { "eol" => "lf }
class Attributes
# path - The path to the Git repository.
def initialize(path)
@path = File.expand_path(path)
@patterns = nil
end
# Returns all the Git attributes for the given path.
#
# path - A path to a file for which to get the attributes.
#
# Returns a Hash.
def attributes(path)
full_path = File.join(@path, path)
patterns.each do |pattern, attrs|
return attrs if File.fnmatch?(pattern, full_path)
end
{}
end
# Returns a Hash containing the file patterns and their attributes.
def patterns
@patterns ||= parse_file
end
# Parses an attribute string.
#
# These strings can be in the following formats:
#
# text # => { "text" => true }
# -text # => { "text" => false }
# key=value # => { "key" => "value" }
#
# string - The string to parse.
#
# Returns a Hash containing the attributes and their values.
def parse_attributes(string)
values = {}
dash = '-'
equal = '='
binary = 'binary'
string.split(/\s+/).each do |chunk|
# Data such as "foo = bar" should be treated as "foo" and "bar" being
# separate boolean attributes.
next if chunk == equal
key = chunk
# Input: "-foo"
if chunk.start_with?(dash)
key = chunk.byteslice(1, chunk.length - 1)
value = false
# Input: "foo=bar"
elsif chunk.include?(equal)
key, value = chunk.split(equal, 2)
# Input: "foo"
else
value = true
end
values[key] = value
# When the "binary" option is set the "diff" option should be set to
# the inverse. If "diff" is later set it should overwrite the
# automatically set value.
values['diff'] = false if key == binary && value
end
values
end
# Iterates over every line in the attributes file.
def each_line
full_path = File.join(@path, 'info/attributes')
return unless File.exist?(full_path)
File.open(full_path, 'r') do |handle|
handle.each_line do |line|
break unless line.valid_encoding?
yield line.strip
end
end
end
private
# Parses the Git attributes file.
def parse_file
pairs = []
comment = '#'
each_line do |line|
next if line.start_with?(comment) || line.empty?
pattern, attrs = line.split(/\s+/, 2)
parsed = attrs ? parse_attributes(attrs) : {}
pairs << [File.join(@path, pattern), parsed]
end
# Newer entries take precedence over older entries.
pairs.reverse.to_h
end
end
end
end
require_relative 'encoding_helper'
module Gitlab
module Git
class Blame
include Gitlab::Git::EncodingHelper
attr_reader :lines, :blames
def initialize(repository, sha, path)
@repo = repository
@sha = sha
@path = path
@lines = []
@blames = load_blame
end
def each
@blames.each do |blame|
yield(
Gitlab::Git::Commit.new(blame.commit),
blame.line
)
end
end
private
def load_blame
cmd = %W(git --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path})
# Read in binary mode to ensure ASCII-8BIT
raw_output = IO.popen(cmd, 'rb') {|io| io.read }
output = encode_utf8(raw_output)
process_raw_blame output
end
def process_raw_blame(output)
lines, final = [], []
info, commits = {}, {}
# process the output
output.split("\n").each do |line|
if line[0, 1] == "\t"
lines << line[1, line.size]
elsif m = /^(\w{40}) (\d+) (\d+)/.match(line)
commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i
commits[commit_id] = nil unless commits.key?(commit_id)
info[lineno] = [commit_id, old_lineno]
end
end
# load all commits in single call
commits.keys.each do |key|
commits[key] = @repo.lookup(key)
end
# get it together
info.sort.each do |lineno, (commit_id, old_lineno)|
commit = commits[commit_id]
final << BlameLine.new(lineno, old_lineno, commit, lines[lineno - 1])
end
@lines = final
end
end
class BlameLine
attr_accessor :lineno, :oldlineno, :commit, :line
def initialize(lineno, oldlineno, commit, line)
@lineno = lineno
@oldlineno = oldlineno
@commit = commit
@line = line
end
end
end
end
require_relative 'encoding_helper'
require_relative 'path_helper'
module Gitlab
module Git
class Blob
include Linguist::BlobHelper
include Gitlab::Git::EncodingHelper
# This number is the maximum amount of data that we want to display to
# the user. We load as much as we can for encoding detection
# (Linguist) and LFS pointer parsing. All other cases where we need full
# blob data should use load_all_data!.
MAX_DATA_DISPLAY_SIZE = 10485760
attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary
class << self
def find(repository, sha, path)
commit = repository.lookup(sha)
root_tree = commit.tree
blob_entry = find_entry_by_path(repository, root_tree.oid, path)
return nil unless blob_entry
if blob_entry[:type] == :commit
submodule_blob(blob_entry, path, sha)
else
blob = repository.lookup(blob_entry[:oid])
if blob
Blob.new(
id: blob.oid,
name: blob_entry[:name],
size: blob.size,
data: blob.content(MAX_DATA_DISPLAY_SIZE),
mode: blob_entry[:filemode].to_s(8),
path: path,
commit_id: sha,
binary: blob.binary?
)
end
end
end
def raw(repository, sha)
blob = repository.lookup(sha)
Blob.new(
id: blob.oid,
size: blob.size,
data: blob.content(MAX_DATA_DISPLAY_SIZE),
binary: blob.binary?
)
end
# Recursive search of blob id by path
#
# Ex.
# blog/ # oid: 1a
# app/ # oid: 2a
# models/ # oid: 3a
# file.rb # oid: 4a
#
#
# Blob.find_entry_by_path(repo, '1a', 'app/file.rb') # => '4a'
#
def find_entry_by_path(repository, root_id, path)
root_tree = repository.lookup(root_id)
# Strip leading slashes
path[/^\/*/] = ''
path_arr = path.split('/')
entry = root_tree.find do |entry|
entry[:name] == path_arr[0]
end
return nil unless entry
if path_arr.size > 1
return nil unless entry[:type] == :tree
path_arr.shift
find_entry_by_path(repository, entry[:oid], path_arr.join('/'))
else
[:blob, :commit].include?(entry[:type]) ? entry : nil
end
end
def submodule_blob(blob_entry, path, sha)
Blob.new(
id: blob_entry[:oid],
name: blob_entry[:name],
data: '',
path: path,
commit_id: sha,
)
end
# Commit file in repository and return commit sha
#
# options should contain next structure:
# file: {
# content: 'Lorem ipsum...',
# path: 'documents/story.txt',
# update: true
# },
# author: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# committer: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# commit: {
# message: 'Wow such commit',
# branch: 'master',
# update_ref: false
# }
#
# rubocop:disable Metrics/AbcSize
# rubocop:disable Metrics/CyclomaticComplexity
# rubocop:disable Metrics/PerceivedComplexity
def commit(repository, options, action = :add)
file = options[:file]
update = file[:update].nil? ? true : file[:update]
author = options[:author]
committer = options[:committer]
commit = options[:commit]
repo = repository.rugged
ref = commit[:branch]
update_ref = commit[:update_ref].nil? ? true : commit[:update_ref]
parents = []
mode = 0o100644
unless ref.start_with?('refs/')
ref = 'refs/heads/' + ref
end
path_name = PathHelper.normalize_path(file[:path])
# Abort if any invalid characters remain (e.g. ../foo)
raise Repository::InvalidBlobName.new("Invalid path") if path_name.each_filename.to_a.include?('..')
filename = path_name.to_s
index = repo.index
unless repo.empty?
rugged_ref = repo.references[ref]
raise Repository::InvalidRef.new("Invalid branch name") unless rugged_ref
last_commit = rugged_ref.target
index.read_tree(last_commit.tree)
parents = [last_commit]
end
if action == :remove
index.remove(filename)
else
file_entry = index.get(filename)
if action == :rename
old_path_name = PathHelper.normalize_path(file[:previous_path])
old_filename = old_path_name.to_s
file_entry = index.get(old_filename)
index.remove(old_filename) unless file_entry.blank?
end
if file_entry
raise Repository::InvalidBlobName.new("Filename already exists; update not allowed") unless update
# Preserve the current file mode if one is available
mode = file_entry[:mode] if file_entry[:mode]
end
content = file[:content]
detect = CharlockHolmes::EncodingDetector.new.detect(content) if content
unless detect && detect[:type] == :binary
# When writing to the repo directly as we are doing here,
# the `core.autocrlf` config isn't taken into account.
content.gsub!("\r\n", "\n") if repository.autocrlf
end
oid = repo.write(content, :blob)
index.add(path: filename, oid: oid, mode: mode)
end
opts = {}
opts[:tree] = index.write_tree(repo)
opts[:author] = author
opts[:committer] = committer
opts[:message] = commit[:message]
opts[:parents] = parents
opts[:update_ref] = ref if update_ref
Rugged::Commit.create(repo, opts)
end
# rubocop:enable Metrics/AbcSize
# rubocop:enable Metrics/CyclomaticComplexity
# rubocop:enable Metrics/PerceivedComplexity
# Remove file from repository and return commit sha
#
# options should contain next structure:
# file: {
# path: 'documents/story.txt'
# },
# author: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# committer: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# commit: {
# message: 'Remove FILENAME',
# branch: 'master'
# }
#
def remove(repository, options)
commit(repository, options, :remove)
end
# Rename file from repository and return commit sha
#
# options should contain next structure:
# file: {
# previous_path: 'documents/old_story.txt'
# path: 'documents/story.txt'
# content: 'Lorem ipsum...',
# update: true
# },
# author: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# committer: {
# email: 'user@example.com',
# name: 'Test User',
# time: Time.now
# },
# commit: {
# message: 'Rename FILENAME',
# branch: 'master'
# }
#
def rename(repository, options)
commit(repository, options, :rename)
end
end
def initialize(options)
%w(id name path size data mode commit_id binary).each do |key|
self.send("#{key}=", options[key.to_sym])
end
@loaded_all_data = false
# Retain the actual size before it is encoded
@loaded_size = @data.bytesize if @data
end
def binary?
@binary.nil? ? super : @binary == true
end
def empty?
!data || data == ''
end
def data
encode! @data
end
# Load all blob data (not just the first MAX_DATA_DISPLAY_SIZE bytes) into
# memory as a Ruby string.
def load_all_data!(repository)
return if @data == '' # don't mess with submodule blobs
return @data if @loaded_all_data
@loaded_all_data = true
@data = repository.lookup(id).content
@loaded_size = @data.bytesize
end
def name
encode! @name
end
# Valid LFS object pointer is a text file consisting of
# version
# oid
# size
# see https://github.com/github/git-lfs/blob/v1.1.0/docs/spec.md#the-pointer
def lfs_pointer?
has_lfs_version_key? && lfs_oid.present? && lfs_size.present?
end
def lfs_oid
if has_lfs_version_key?
oid = data.match(/(?<=sha256:)([0-9a-f]{64})/)
return oid[1] if oid
end
nil
end
def lfs_size
if has_lfs_version_key?
size = data.match(/(?<=size )([0-9]+)/)
return size[1] if size
end
nil
end
def truncated?
size && (size > loaded_size)
end
private
def has_lfs_version_key?
!empty? && text? && data.start_with?("version https://git-lfs.github.com/spec")
end
end
end
end
module Gitlab
module Git
class BlobSnippet
include Linguist::BlobHelper
attr_accessor :ref
attr_accessor :lines
attr_accessor :filename
attr_accessor :startline
def initialize(ref, lines, startline, filename)
@ref, @lines, @startline, @filename = ref, lines, startline, filename
end
def data
lines.join("\n") if lines
end
def name
filename
end
def size
data.length
end
def mode
nil
end
end
end
end
module Gitlab
module Git
class Branch < Ref
end
end
end
# Gitlab::Git::Commit is a wrapper around native Rugged::Commit object
module Gitlab
module Git
class Commit
include Gitlab::Git::EncodingHelper
attr_accessor :raw_commit, :head, :refs
SERIALIZE_KEYS = [
:id, :message, :parent_ids,
:authored_date, :author_name, :author_email,
:committed_date, :committer_name, :committer_email
].freeze
attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator
def ==(other)
return false unless other.is_a?(Gitlab::Git::Commit)
methods = [:message, :parent_ids, :authored_date, :author_name,
:author_email, :committed_date, :committer_name,
:committer_email]
methods.all? do |method|
send(method) == other.send(method)
end
end
class << self
# Get commits collection
#
# Ex.
# Commit.where(
# repo: repo,
# ref: 'master',
# path: 'app/models',
# limit: 10,
# offset: 5,
# )
#
def where(options)
repo = options.delete(:repo)
raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log)
repo.log(options).map { |c| decorate(c) }
end
# Get single commit
#
# Ex.
# Commit.find(repo, '29eda46b')
#
# Commit.find(repo, 'master')
#
def find(repo, commit_id = "HEAD")
return decorate(commit_id) if commit_id.is_a?(Rugged::Commit)
obj = if commit_id.is_a?(String)
repo.rev_parse_target(commit_id)
else
Ref.dereference_object(commit_id)
end
return nil unless obj.is_a?(Rugged::Commit)
decorate(obj)
rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository
nil
end
# Get last commit for HEAD
#
# Ex.
# Commit.last(repo)
#
def last(repo)
find(repo)
end
# Get last commit for specified path and ref
#
# Ex.
# Commit.last_for_path(repo, '29eda46b', 'app/models')
#
# Commit.last_for_path(repo, 'master', 'Gemfile')
#
def last_for_path(repo, ref, path = nil)
where(
repo: repo,
ref: ref,
path: path,
limit: 1
).first