Commit f9475e29 authored by Francisco Javier López's avatar Francisco Javier López Committed by Douwe Maan

Uploads to wiki stored inside the wiki git repository

parent 0689900c
module WikiHelper
include API::Helpers::RelatedResourcesHelpers
# Produces a pure text breadcrumb for a given page.
#
# page_slug - The slug of a WikiPage object.
......@@ -39,4 +41,8 @@ module WikiHelper
end
end
end
def wiki_attachment_upload_url
expose_url(api_v4_projects_wikis_attachments_path(id: @project.id))
end
end
......@@ -7,8 +7,8 @@ module Files
def initialize(*args)
super
@author_email = params[:author_email]
@author_name = params[:author_name]
@author_email = params[:author_email] || current_user&.email
@author_name = params[:author_name] || current_user&.name
@commit_message = params[:commit_message]
@last_commit_sha = params[:last_commit_sha]
......
# frozen_string_literal: true
module Wikis
class CreateAttachmentService < Files::CreateService
ATTACHMENT_PATH = 'uploads'.freeze
MAX_FILENAME_LENGTH = 255
delegate :wiki, to: :project
delegate :repository, to: :wiki
def initialize(*args)
super
@file_name = truncate_file_name(params[:file_name])
@file_path = File.join(ATTACHMENT_PATH, SecureRandom.hex, @file_name) if @file_name
@commit_message ||= "Upload attachment #{@file_name}"
@branch_name ||= wiki.default_branch
end
def create_commit!
commit_result(create_transformed_commit(@file_content))
end
private
def truncate_file_name(file_name)
return unless file_name.present?
return file_name if file_name.length <= MAX_FILENAME_LENGTH
extension = File.extname(file_name)
truncate_at = MAX_FILENAME_LENGTH - extension.length - 1
base_name = File.basename(file_name, extension)[0..truncate_at]
base_name + extension
end
def validate!
validate_file_name!
validate_permissions!
end
def validate_file_name!
raise_error('The file name cannot be empty') unless @file_name
end
def validate_permissions!
unless can?(current_user, :create_wiki, project)
raise_error('You are not allowed to push to the wiki')
end
end
def create_transformed_commit(content)
repository.create_file(
current_user,
@file_path,
content,
message: @commit_message,
branch_name: @branch_name,
author_email: @author_email,
author_name: @author_name)
end
def commit_result(commit_id)
{
file_name: @file_name,
file_path: @file_path,
branch: @branch_name,
commit: commit_id
}
end
end
end
......@@ -122,12 +122,6 @@ class FileUploader < GitlabUploader
}
end
def markdown_link
markdown = +"[#{markdown_name}](#{secure_url})"
markdown.prepend("!") if image_or_video? || dangerous?
markdown
end
def to_h
{
alt: markdown_name,
......@@ -192,10 +186,6 @@ class FileUploader < GitlabUploader
storage.delete_dir!(store_dir) # only remove when empty
end
def markdown_name
(image_or_video? ? File.basename(filename, File.extname(filename)) : filename).gsub("]", "\\]")
end
def identifier
@identifier ||= filename
end
......
......@@ -2,32 +2,7 @@
# Extra methods for uploader
module UploaderHelper
IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze
# We recommend using the .mp4 format over .mov. Videos in .mov format can
# still be used but you really need to make sure they are served with the
# proper MIME type video/mp4 and not video/quicktime or your videos won't play
# on IE >= 9.
# http://archive.sublimevideo.info/20150912/docs.sublimevideo.net/troubleshooting.html
VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze
# These extension types can contain dangerous code and should only be embedded inline with
# proper filtering. They should always be tagged as "Content-Disposition: attachment", not "inline".
DANGEROUS_EXT = %w[svg].freeze
def image?
extension_match?(IMAGE_EXT)
end
def video?
extension_match?(VIDEO_EXT)
end
def image_or_video?
image? || video?
end
def dangerous?
extension_match?(DANGEROUS_EXT)
end
include Gitlab::FileMarkdownLinkBuilder
private
......
......@@ -41,3 +41,8 @@
= render 'sidebar'
#delete-wiki-modal.modal.fade
- content_for :scripts_body do
-# haml-lint:disable InlineJavaScript
:javascript
window.uploads_path = "#{wiki_attachment_upload_url}";
---
title: Store wiki uploads inside git repository
merge_request: 21362
author:
type: added
......@@ -97,12 +97,12 @@ curl --data "format=rdoc&title=Hello&content=Hello world" --header "PRIVATE-TOKE
Example response:
```json
{
{
"content" : "Hello world",
"format" : "markdown",
"slug" : "Hello",
"title" : "Hello"
}
}
```
## Edit an existing wiki page
......@@ -154,6 +154,44 @@ DELETE /projects/:id/wikis/:slug
curl --request DELETE --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects/1/wikis/foo"
```
On success the HTTP status code is `204` and no JSON response is expected.
On success the HTTP status code is `204` and no JSON response is expected.
[ce-13372]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/13372
## Upload an attachment to the wiki repository
Uploads a file to the attachment folder inside the wiki's repository. The
attachment folder is the `uploads` folder.
```
POST /projects/:id/wikis/attachments
```
| Attribute | Type | Required | Description |
| ------------- | ------- | -------- | ---------------------------- |
| `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
| `file` | string | yes | The attachment to be uploaded |
| `branch` | string | no | The name of the branch. Defaults to the wiki repository default branch |
To upload a file from your filesystem, use the `--form` argument. This causes
cURL to post data using the header `Content-Type: multipart/form-data`.
The `file=` parameter must point to a file on your filesystem and be preceded
by `@`. For example:
```bash
curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "file=@dk.png" https://gitlab.example.com/api/v4/projects/1/wikis/attachments
```
Example response:
```json
{
"file_name" : "dk.png",
"file_path" : "uploads/6a061c4cf9f1c28cb22c384b4b8d4e3c/dk.png",
"branch" : "master",
"link" : {
"url" : "uploads/6a061c4cf9f1c28cb22c384b4b8d4e3c/dk.png",
"markdown" : "![dk](uploads/6a061c4cf9f1c28cb22c384b4b8d4e3c/dk.png)"
}
}
```
......@@ -10,6 +10,28 @@ module API
expose :content
end
class WikiAttachment < Grape::Entity
include Gitlab::FileMarkdownLinkBuilder
expose :file_name
expose :file_path
expose :branch
expose :link do
expose :file_path, as: :url
expose :markdown do |_entity|
self.markdown_link
end
end
def filename
object.file_name
end
def secure_url
object.file_path
end
end
class UserSafe < Grape::Entity
expose :id, :name, :username
end
......
module API
class Wikis < Grape::API
helpers do
def commit_params(attrs)
{
file_name: attrs[:file][:filename],
file_content: File.read(attrs[:file][:tempfile]),
branch_name: attrs[:branch]
}
end
params :wiki_page_params do
requires :content, type: String, desc: 'Content of a wiki page'
requires :title, type: String, desc: 'Title of a wiki page'
......@@ -84,6 +92,29 @@ module API
status 204
WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page)
end
desc 'Upload an attachment to the wiki repository' do
detail 'This feature was introduced in GitLab 11.3.'
success Entities::WikiAttachment
end
params do
requires :file, type: File, desc: 'The attachment file to be uploaded'
optional :branch, type: String, desc: 'The name of the branch'
end
post ":id/wikis/attachments", requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
authorize! :create_wiki, user_project
result = ::Wikis::CreateAttachmentService.new(user_project,
current_user,
commit_params(declared_params(include_missing: false))).execute
if result[:status] == :success
status(201)
present OpenStruct.new(result[:result]), with: Entities::WikiAttachment
else
render_api_error!(result[:message], 400)
end
end
end
end
end
# frozen_string_literal: true
require 'uri'
module Banzai
module Filter
# HTML filter that "fixes" links to pages/files in a wiki.
......@@ -13,8 +11,12 @@ module Banzai
def call
return doc unless project_wiki?
doc.search('a:not(.gfm)').each do |el|
process_link_attr el.attribute('href')
doc.search('a:not(.gfm)').each { |el| process_link_attr(el.attribute('href')) }
doc.search('video').each { |el| process_link_attr(el.attribute('src')) }
doc.search('img').each do |el|
attr = el.attribute('data-src') || el.attribute('src')
process_link_attr(attr)
end
doc
......
......@@ -10,11 +10,16 @@ module Banzai
def apply_rules
# Special case: relative URLs beginning with `/uploads/` refer to
# user-uploaded files and will be handled elsewhere.
return @uri.to_s if @uri.relative? && @uri.path.starts_with?('/uploads/')
# user-uploaded files will be handled elsewhere.
return @uri.to_s if public_upload?
# Special case: relative URLs beginning with Wikis::CreateAttachmentService::ATTACHMENT_PATH
# refer to user-uploaded files to the wiki repository.
unless repository_upload?
apply_file_link_rules!
apply_hierarchical_link_rules!
end
apply_file_link_rules!
apply_hierarchical_link_rules!
apply_relative_link_rules!
@uri.to_s
end
......@@ -39,6 +44,14 @@ module Banzai
@uri = Addressable::URI.parse(link)
end
end
def public_upload?
@uri.relative? && @uri.path.starts_with?('/uploads/')
end
def repository_upload?
@uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH)
end
end
end
end
......
# Builds the markdown link of a file
# It needs the methods filename and secure_url (final destination url) to be defined.
module Gitlab
module FileMarkdownLinkBuilder
include FileTypeDetection
def markdown_link
return unless name = markdown_name
markdown = "[#{name.gsub(']', '\\]')}](#{secure_url})"
markdown.prepend("!") if image_or_video? || dangerous?
markdown
end
def markdown_name
return unless filename.present?
image_or_video? ? File.basename(filename, File.extname(filename)) : filename
end
end
end
# frozen_string_literal: true
# File helpers methods.
# It needs the method filename to be defined.
module Gitlab
module FileTypeDetection
IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze
# We recommend using the .mp4 format over .mov. Videos in .mov format can
# still be used but you really need to make sure they are served with the
# proper MIME type video/mp4 and not video/quicktime or your videos won't play
# on IE >= 9.
# http://archive.sublimevideo.info/20150912/docs.sublimevideo.net/troubleshooting.html
VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze
# These extension types can contain dangerous code and should only be embedded inline with
# proper filtering. They should always be tagged as "Content-Disposition: attachment", not "inline".
DANGEROUS_EXT = %w[svg].freeze
def image?
extension_match?(IMAGE_EXT)
end
def video?
extension_match?(VIDEO_EXT)
end
def image_or_video?
image? || video?
end
def dangerous?
extension_match?(DANGEROUS_EXT)
end
private
def extension_match?(extensions)
return false unless filename
extension = File.extname(filename).delete('.')
extensions.include?(extension.downcase)
end
end
end
......@@ -146,6 +146,8 @@ describe "User creates wiki page" do
expect(page).to have_selector(".katex", count: 3).and have_content("2+2 is 4")
end
end
it_behaves_like 'wiki file attachments'
end
context "in a group namespace", :js do
......
......@@ -3,6 +3,7 @@ require 'spec_helper'
describe 'User updates wiki page' do
shared_examples 'wiki page user update' do
let(:user) { create(:user) }
before do
project.add_maintainer(user)
sign_in(user)
......@@ -55,6 +56,8 @@ describe 'User updates wiki page' do
expect(page).to have_content('Updated Wiki Content')
end
it_behaves_like 'wiki file attachments'
end
end
......@@ -64,14 +67,14 @@ describe 'User updates wiki page' do
before do
visit(project_wikis_path(project))
click_link('Edit')
end
context 'in a user namespace' do
let(:project) { create(:project, :wiki_repo, namespace: user.namespace) }
it 'updates a page' do
click_link('Edit')
# Commit message field should have correct value.
expect(page).to have_field('wiki[message]', with: 'Update home')
......@@ -84,8 +87,6 @@ describe 'User updates wiki page' do
end
it 'shows a validation error message' do
click_link('Edit')
fill_in(:wiki_content, with: '')
click_button('Save changes')
......@@ -97,8 +98,6 @@ describe 'User updates wiki page' do
end
it 'shows the emoji autocompletion dropdown', :js do
click_link('Edit')
find('#wiki_content').native.send_keys('')
fill_in(:wiki_content, with: ':')
......@@ -106,8 +105,6 @@ describe 'User updates wiki page' do
end
it 'shows the error message' do
click_link('Edit')
wiki_page.update(content: 'Update')
click_button('Save changes')
......@@ -116,30 +113,27 @@ describe 'User updates wiki page' do
end
it 'updates a page' do
click_on('Edit')
fill_in('Content', with: 'Updated Wiki Content')
click_on('Save changes')
expect(page).to have_content('Updated Wiki Content')
end
it 'cancels edititng of a page' do
click_on('Edit')
it 'cancels editing of a page' do
page.within(:css, '.wiki-form .form-actions') do
click_on('Cancel')
end
expect(current_path).to eq(project_wiki_path(project, wiki_page))
end
it_behaves_like 'wiki file attachments'
end
context 'in a group namespace' do
let(:project) { create(:project, :wiki_repo, namespace: create(:group, :public)) }
it 'updates a page' do
click_link('Edit')
# Commit message field should have correct value.
expect(page).to have_field('wiki[message]', with: 'Update home')
......@@ -151,6 +145,8 @@ describe 'User updates wiki page' do
expect(page).to have_content("Last edited by #{user.name}")
expect(page).to have_content('My awesome wiki!')
end
it_behaves_like 'wiki file attachments'
end
end
......@@ -222,6 +218,8 @@ describe 'User updates wiki page' do
expect(current_path).to eq(project_wiki_path(project, "foo1/bar1/#{page_name}"))
end
it_behaves_like 'wiki file attachments'
end
end
......
......@@ -93,7 +93,7 @@ describe 'User views a wiki page' do
allow(wiki_file).to receive(:mime_type).and_return('image/jpeg')
allow_any_instance_of(ProjectWiki).to receive(:find_file).with('image.jpg', nil).and_return(wiki_file)
expect(page).to have_xpath('//img[@data-src="image.jpg"]')
expect(page).to have_xpath("//img[@data-src='#{project.wiki.wiki_base_path}/image.jpg']")
expect(page).to have_link('image', href: "#{project.wiki.wiki_base_path}/image.jpg")
click_on('image')
......
......@@ -7,6 +7,7 @@ describe Banzai::Filter::WikiLinkFilter do
let(:project) { build_stubbed(:project, :public, name: "wiki_link_project", namespace: namespace) }
let(:user) { double }
let(:wiki) { ProjectWiki.new(project, user) }
let(:repository_upload_folder) { Wikis::CreateAttachmentService::ATTACHMENT_PATH }
it "doesn't rewrite absolute links" do
filtered_link = filter("<a href='http://example.com:8000/'>Link</a>", project_wiki: wiki).children[0]
......@@ -20,6 +21,45 @@ describe Banzai::Filter::WikiLinkFilter do
expect(filtered_link.attribute('href').value).to eq('/uploads/a.test')
end
describe "when links point to the #{Wikis::CreateAttachmentService::ATTACHMENT_PATH} folder" do
context 'with an "a" html tag' do
it 'rewrites links' do
filtered_link = filter("<a href='#{repository_upload_folder}/a.test'>Link</a>", project_wiki: wiki).children[0]
expect(filtered_link.attribute('href').value).to eq("#{wiki.wiki_base_path}/#{repository_upload_folder}/a.test")
end
end
context 'with "img" html tag' do
let(:path) { "#{wiki.wiki_base_path}/#{repository_upload_folder}/a.jpg" }
context 'inside an "a" html tag' do
it 'rewrites links' do
filtered_elements = filter("<a href='#{repository_upload_folder}/a.jpg'><img src='#{repository_upload_folder}/a.jpg'>example</img></a>", project_wiki: wiki)
expect(filtered_elements.search('img').first.attribute('src').value).to eq(path)
expect(filtered_elements.search('a').first.attribute('href').value).to eq(path)
end
end
context 'outside an "a" html tag' do
it 'rewrites links' do
filtered_link = filter("<img src='#{repository_upload_folder}/a.jpg'>example</img>", project_wiki: wiki).children[0]
expect(filtered_link.attribute('src').value).to eq(path)
end
end
end
context 'with "video" html tag' do
it 'rewrites links' do
filtered_link = filter("<video src='#{repository_upload_folder}/a.mp4'></video>", project_wiki: wiki).children[0]
expect(filtered_link.attribute('src').value).to eq("#{wiki.wiki_base_path}/#{repository_upload_folder}/a.mp4")
end
end
end
describe "invalid links" do
invalid_links = ["http://:8080", "http://", "http://:8080/path"]
......
# frozen_string_literal: true
require 'rails_helper'
describe Gitlab::FileMarkdownLinkBuilder do
let(:custom_class) do
Class.new do
include Gitlab::FileMarkdownLinkBuilder
end.new
end
before do
allow(custom_class).to receive(:filename).and_return(filename)
end
describe 'markdown_link' do
let(:url) { "/uploads/#{filename}"}
before do
allow(custom_class).to receive(:secure_url).and_return(url)
end
context 'when file name has the character ]' do
let(:filename) { 'd]k.png' }
it 'escapes the character' do
expect(custom_class.markdown_link).to eq '![d\\]k](/uploads/d]k.png)'
end
end
context 'when file is an image or video' do
let(:filename) { 'dk.png' }
it 'returns preview markdown link' do
expect(custom_class.markdown_link).to eq '![dk](/uploads/dk.png)'
end
end
context 'when file is not an image or video' do
let(:filename) { 'dk.zip' }
it 'returns markdown link' do
expect(custom_class.markdown_link).to eq '[dk.zip](/uploads/dk.zip)'
end
end
context 'when file name is blank' do
let(:filename) { nil }
it 'returns nil' do
expect(custom_class.markdown_link).to eq nil
end
end
end
describe 'mardown_name' do
context 'when file is an image or video' do
let(:filename) { 'dk.png' }
it 'retrieves the name without the extension' do
expect(custom_class.markdown_name).to eq 'dk'
end
end
context 'when file is not an image or video' do
let(:filename) { 'dk.zip' }
it 'retrieves the name with the extesion' do
expect(custom_class.markdown_name).to eq 'dk.zip'
end
end
context 'when file name is blank' do
let(:filename) { nil }
it 'returns nil' do
expect(custom_class.markdown_name).to eq nil
end
end
end
end
# frozen_string_literal: true
require 'rails_helper'
describe Gitlab::FileTypeDetection do
def upload_fixture(filename)
fixture_file_upload(File.join('spec', 'fixtures', filename))
end
describe '#image_or_video?' do
context 'when class is an uploader' do
let(:uploader) do
example_uploader = Class.new(CarrierWave::Uploader::Base) do
include Gitlab::FileTypeDetection
storage :file
end
example_uploader.new
end
it 'returns true for an image file' do
uploader.store!(upload_fixture('dk.png'))
expect(uploader).to be_image_or_video
end
it 'returns true for a video file' do
uploader.store!(upload_fixture('video_sample.mp4'))