Uploads to wiki stored inside the wiki git repository

app/helpers/wiki_helper.rb

 module WikiHelper
+  include API::Helpers::RelatedResourcesHelpers
+
   # Produces a pure text breadcrumb for a given page.
   #
   # page_slug - The slug of a WikiPage object.
@@ -39,4 +41,8 @@ def wiki_page_errors(error)
       end
     end
   end
+
+  def wiki_attachment_upload_url
+    expose_url(api_v4_projects_wikis_attachments_path(id: @project.id))
+  end
 end

app/services/files/base_service.rb

@@ -7,8 +7,8 @@ class BaseService < Commits::CreateService
     def initialize(*args)
       super
 
-      @author_email = params[:author_email]
-      @author_name = params[:author_name]
+      @author_email = params[:author_email] || current_user&.email
+      @author_name = params[:author_name] || current_user&.name
       @commit_message = params[:commit_message]
       @last_commit_sha = params[:last_commit_sha]
 

app/services/wikis/create_attachment_service.rb

+# frozen_string_literal: true
+
+module Wikis
+  class CreateAttachmentService < Files::CreateService
+    ATTACHMENT_PATH = 'uploads'.freeze
+    MAX_FILENAME_LENGTH = 255
+
+    delegate :wiki, to: :project
+    delegate :repository, to: :wiki
+
+    def initialize(*args)
+      super
+
+      @file_name = truncate_file_name(params[:file_name])
+      @file_path = File.join(ATTACHMENT_PATH, SecureRandom.hex, @file_name) if @file_name
+      @commit_message ||= "Upload attachment #{@file_name}"
+      @branch_name ||= wiki.default_branch
+    end
+
+    def create_commit!
+      commit_result(create_transformed_commit(@file_content))
+    end
+
+    private
+
+    def truncate_file_name(file_name)
+      return unless file_name.present?
+      return file_name if file_name.length <= MAX_FILENAME_LENGTH
+
+      extension = File.extname(file_name)
+      truncate_at = MAX_FILENAME_LENGTH - extension.length - 1
+      base_name = File.basename(file_name, extension)[0..truncate_at]
+      base_name + extension
+    end
+
+    def validate!
+      validate_file_name!
+      validate_permissions!
+    end
+
+    def validate_file_name!
+      raise_error('The file name cannot be empty') unless @file_name
+    end
+
+    def validate_permissions!
+      unless can?(current_user, :create_wiki, project)
+        raise_error('You are not allowed to push to the wiki')
+      end
+    end
+
+    def create_transformed_commit(content)
+      repository.create_file(
+        current_user,
+        @file_path,
+        content,
+        message: @commit_message,
+        branch_name: @branch_name,
+        author_email: @author_email,
+        author_name: @author_name)
+    end
+
+    def commit_result(commit_id)
+      {
+        file_name: @file_name,
+        file_path: @file_path,
+        branch: @branch_name,
+        commit: commit_id
+      }
+    end
+  end
+end

app/uploaders/file_uploader.rb

@@ -122,12 +122,6 @@ def store_dirs
     }
   end
 
-  def markdown_link
-    markdown = +"[#{markdown_name}](#{secure_url})"
-    markdown.prepend("!") if image_or_video? || dangerous?
-    markdown
-  end
-
   def to_h
     {
       alt:      markdown_name,
@@ -192,10 +186,6 @@ def prune_store_dir
     storage.delete_dir!(store_dir) # only remove when empty
   end
 
-  def markdown_name
-    (image_or_video? ? File.basename(filename, File.extname(filename)) : filename).gsub("]", "\\]")
-  end
-
   def identifier
     @identifier ||= filename
   end

app/uploaders/uploader_helper.rb

@@ -2,32 +2,7 @@
 
 # Extra methods for uploader
 module UploaderHelper
-  IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze
-  # We recommend using the .mp4 format over .mov. Videos in .mov format can
-  # still be used but you really need to make sure they are served with the
-  # proper MIME type video/mp4 and not video/quicktime or your videos won't play
-  # on IE >= 9.
-  # http://archive.sublimevideo.info/20150912/docs.sublimevideo.net/troubleshooting.html
-  VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze
-  # These extension types can contain dangerous code and should only be embedded inline with
-  # proper filtering. They should always be tagged as "Content-Disposition: attachment", not "inline".
-  DANGEROUS_EXT = %w[svg].freeze
-
-  def image?
-    extension_match?(IMAGE_EXT)
-  end
-
-  def video?
-    extension_match?(VIDEO_EXT)
-  end
-
-  def image_or_video?
-    image? || video?
-  end
-
-  def dangerous?
-    extension_match?(DANGEROUS_EXT)
-  end
+  include Gitlab::FileMarkdownLinkBuilder
 
   private
 

app/views/projects/wikis/edit.html.haml

@@ -41,3 +41,8 @@
 = render 'sidebar'
 
 #delete-wiki-modal.modal.fade
+
+- content_for :scripts_body do
+  -# haml-lint:disable InlineJavaScript
+  :javascript
+    window.uploads_path = "#{wiki_attachment_upload_url}";

changelogs/unreleased/fj-33475-files-inside-wiki-repo.yml

+---
+title: Store wiki uploads inside git repository
+merge_request: 21362
+author:
+type: added

doc/api/wikis.md

@@ -97,12 +97,12 @@ curl --data "format=rdoc&title=Hello&content=Hello world" --header "PRIVATE-TOKE
 Example response:
 
 ```json
-{ 
+{
   "content" : "Hello world",
   "format" : "markdown",
   "slug" : "Hello",
   "title" : "Hello"
-} 
+}
 ```
 
 ## Edit an existing wiki page
@@ -154,6 +154,44 @@ DELETE /projects/:id/wikis/:slug
 curl --request DELETE --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects/1/wikis/foo"
 ```
 
-On success the HTTP status code is `204` and no JSON response is expected. 
+On success the HTTP status code is `204` and no JSON response is expected.
 
 [ce-13372]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/13372
+
+## Upload an attachment to the wiki repository
+
+Uploads a file to the attachment folder inside the wiki's repository. The
+ attachment folder is the `uploads` folder.
+
+```
+POST /projects/:id/wikis/attachments
+```
+
+| Attribute     | Type    | Required | Description                  |
+| ------------- | ------- | -------- | ---------------------------- |
+| `id`      | integer/string    | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
+| `file` | string | yes | The attachment to be uploaded |
+| `branch` | string | no | The name of the branch. Defaults to the wiki repository default branch |
+
+To upload a file from your filesystem, use the `--form` argument. This causes
+cURL to post data using the header `Content-Type: multipart/form-data`.
+The `file=` parameter must point to a file on your filesystem and be preceded
+by `@`. For example:
+
+```bash
+curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "file=@dk.png" https://gitlab.example.com/api/v4/projects/1/wikis/attachments
+```
+
+Example response:
+
+```json
+{
+  "file_name" : "dk.png",
+  "file_path" : "uploads/6a061c4cf9f1c28cb22c384b4b8d4e3c/dk.png",
+  "branch" : "master",
+  "link" : {
+    "url" : "uploads/6a061c4cf9f1c28cb22c384b4b8d4e3c/dk.png",
+    "markdown" : "![dk](uploads/6a061c4cf9f1c28cb22c384b4b8d4e3c/dk.png)"
+  }
+}
+```

lib/api/entities.rb

@@ -10,6 +10,28 @@ class WikiPage < WikiPageBasic
       expose :content
     end
 
+    class WikiAttachment < Grape::Entity
+      include Gitlab::FileMarkdownLinkBuilder
+
+      expose :file_name
+      expose :file_path
+      expose :branch
+      expose :link do
+        expose :file_path, as: :url
+        expose :markdown do |_entity|
+          self.markdown_link
+        end
+      end
+
+      def filename
+        object.file_name
+      end
+
+      def secure_url
+        object.file_path
+      end
+    end
+
     class UserSafe < Grape::Entity
       expose :id, :name, :username
     end

lib/api/wikis.rb

 module API
   class Wikis < Grape::API
     helpers do
+      def commit_params(attrs)
+        {
+          file_name: attrs[:file][:filename],
+          file_content: File.read(attrs[:file][:tempfile]),
+          branch_name: attrs[:branch]
+        }
+      end
+
       params :wiki_page_params do
         requires :content, type: String, desc: 'Content of a wiki page'
         requires :title, type: String, desc: 'Title of a wiki page'
@@ -84,6 +92,29 @@ class Wikis < Grape::API
         status 204
         WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page)
       end
+
+      desc 'Upload an attachment to the wiki repository' do
+        detail 'This feature was introduced in GitLab 11.3.'
+        success Entities::WikiAttachment
+      end
+      params do
+        requires :file, type: File, desc: 'The attachment file to be uploaded'
+        optional :branch, type: String, desc: 'The name of the branch'
+      end
+      post ":id/wikis/attachments", requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do
+        authorize! :create_wiki, user_project
+
+        result = ::Wikis::CreateAttachmentService.new(user_project,
+                                                      current_user,
+                                                      commit_params(declared_params(include_missing: false))).execute
+
+        if result[:status] == :success
+          status(201)
+          present OpenStruct.new(result[:result]), with: Entities::WikiAttachment
+        else
+          render_api_error!(result[:message], 400)
+        end
+      end
     end
   end
 end

lib/banzai/filter/wiki_link_filter.rb

 # frozen_string_literal: true
 
-require 'uri'
-
 module Banzai
   module Filter
     # HTML filter that "fixes" links to pages/files in a wiki.
@@ -13,8 +11,12 @@ class WikiLinkFilter < HTML::Pipeline::Filter
       def call
         return doc unless project_wiki?
 
-        doc.search('a:not(.gfm)').each do |el|
-          process_link_attr el.attribute('href')
+        doc.search('a:not(.gfm)').each { |el| process_link_attr(el.attribute('href')) }
+        doc.search('video').each { |el| process_link_attr(el.attribute('src')) }
+        doc.search('img').each do |el|
+          attr = el.attribute('data-src') || el.attribute('src')
+
+          process_link_attr(attr)
         end
 
         doc

lib/banzai/filter/wiki_link_filter/rewriter.rb

@@ -10,11 +10,16 @@ def initialize(link_string, wiki:, slug:)
 
         def apply_rules
           # Special case: relative URLs beginning with `/uploads/` refer to
-          # user-uploaded files and will be handled elsewhere.
-          return @uri.to_s if @uri.relative? && @uri.path.starts_with?('/uploads/')
+          # user-uploaded files will be handled elsewhere.
+          return @uri.to_s if public_upload?
+
+          # Special case: relative URLs beginning with Wikis::CreateAttachmentService::ATTACHMENT_PATH
+          # refer to user-uploaded files to the wiki repository.
+          unless repository_upload?
+            apply_file_link_rules!
+            apply_hierarchical_link_rules!
+          end
 
-          apply_file_link_rules!
-          apply_hierarchical_link_rules!
           apply_relative_link_rules!
           @uri.to_s
         end
@@ -39,6 +44,14 @@ def apply_relative_link_rules!
             @uri = Addressable::URI.parse(link)
           end
         end
+
+        def public_upload?
+          @uri.relative? && @uri.path.starts_with?('/uploads/')
+        end
+
+        def repository_upload?
+          @uri.relative? && @uri.path.starts_with?(Wikis::CreateAttachmentService::ATTACHMENT_PATH)
+        end
       end
     end
   end

lib/gitlab/file_markdown_link_builder.rb

+# Builds the markdown link of a file
+# It needs the methods filename and secure_url (final destination url) to be defined.
+module Gitlab
+  module FileMarkdownLinkBuilder
+    include FileTypeDetection
+
+    def markdown_link
+      return unless name = markdown_name
+
+      markdown = "[#{name.gsub(']', '\\]')}](#{secure_url})"
+      markdown.prepend("!") if image_or_video? || dangerous?
+      markdown
+    end
+
+    def markdown_name
+      return unless filename.present?
+
+      image_or_video? ? File.basename(filename, File.extname(filename)) : filename
+    end
+  end
+end

lib/gitlab/file_type_detection.rb

+# frozen_string_literal: true
+
+# File helpers methods.
+# It needs the method filename to be defined.
+module Gitlab
+  module FileTypeDetection
+    IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze
+    # We recommend using the .mp4 format over .mov. Videos in .mov format can
+    # still be used but you really need to make sure they are served with the
+    # proper MIME type video/mp4 and not video/quicktime or your videos won't play
+    # on IE >= 9.
+    # http://archive.sublimevideo.info/20150912/docs.sublimevideo.net/troubleshooting.html
+    VIDEO_EXT = %w[mp4 m4v mov webm ogv].freeze
+    # These extension types can contain dangerous code and should only be embedded inline with
+    # proper filtering. They should always be tagged as "Content-Disposition: attachment", not "inline".
+    DANGEROUS_EXT = %w[svg].freeze
+
+    def image?
+      extension_match?(IMAGE_EXT)
+    end
+
+    def video?
+      extension_match?(VIDEO_EXT)
+    end
+
+    def image_or_video?
+      image? || video?
+    end
+
+    def dangerous?
+      extension_match?(DANGEROUS_EXT)
+    end
+
+    private
+
+    def extension_match?(extensions)
+      return false unless filename
+
+      extension = File.extname(filename).delete('.')
+      extensions.include?(extension.downcase)
+    end
+  end
+end

spec/features/projects/wiki/user_creates_wiki_page_spec.rb

@@ -146,6 +146,8 @@
           expect(page).to have_selector(".katex", count: 3).and have_content("2+2 is 4")
         end
       end
+
+      it_behaves_like 'wiki file attachments'
     end
 
     context "in a group namespace", :js do

spec/features/projects/wiki/user_updates_wiki_page_spec.rb

@@ -3,6 +3,7 @@
 describe 'User updates wiki page' do
   shared_examples 'wiki page user update' do
     let(:user) { create(:user) }
+
     before do
       project.add_maintainer(user)
       sign_in(user)
@@ -55,6 +56,8 @@
 
           expect(page).to have_content('Updated Wiki Content')
         end
+
+        it_behaves_like 'wiki file attachments'
       end
     end
 
@@ -64,14 +67,14 @@
 
       before do
         visit(project_wikis_path(project))
+
+        click_link('Edit')
       end
 
       context 'in a user namespace' do
         let(:project) { create(:project, :wiki_repo, namespace: user.namespace) }
 
         it 'updates a page' do
-          click_link('Edit')
-
           # Commit message field should have correct value.
           expect(page).to have_field('wiki[message]', with: 'Update home')
 
@@ -84,8 +87,6 @@
         end
 
         it 'shows a validation error message' do
-          click_link('Edit')
-
           fill_in(:wiki_content, with: '')
           click_button('Save changes')
 
@@ -97,8 +98,6 @@
         end
 
         it 'shows the emoji autocompletion dropdown', :js do
-          click_link('Edit')
-
           find('#wiki_content').native.send_keys('')
           fill_in(:wiki_content, with: ':')
 
@@ -106,8 +105,6 @@
         end
 
         it 'shows the error message' do
-          click_link('Edit')
-
           wiki_page.update(content: 'Update')
 
           click_button('Save changes')
@@ -116,30 +113,27 @@
         end
 
         it 'updates a page' do
-          click_on('Edit')
           fill_in('Content', with: 'Updated Wiki Content')
           click_on('Save changes')
 
           expect(page).to have_content('Updated Wiki Content')
         end
 
-        it 'cancels edititng of a page' do
-          click_on('Edit')
-
+        it 'cancels editing of a page' do
           page.within(:css, '.wiki-form .form-actions') do
             click_on('Cancel')
           end
 
           expect(current_path).to eq(project_wiki_path(project, wiki_page))
         end
+
+        it_behaves_like 'wiki file attachments'
       end
 
       context 'in a group namespace' do
         let(:project) { create(:project, :wiki_repo, namespace: create(:group, :public)) }
 
         it 'updates a page' do
-          click_link('Edit')
-
           # Commit message field should have correct value.
           expect(page).to have_field('wiki[message]', with: 'Update home')
 
@@ -151,6 +145,8 @@
           expect(page).to have_content("Last edited by #{user.name}")
           expect(page).to have_content('My awesome wiki!')
         end
+
+        it_behaves_like 'wiki file attachments'
       end
     end
 
@@ -222,6 +218,8 @@
 
         expect(current_path).to eq(project_wiki_path(project, "foo1/bar1/#{page_name}"))
       end
+
+      it_behaves_like 'wiki file attachments'
     end
   end
 

spec/features/projects/wiki/user_views_wiki_page_spec.rb

@@ -93,7 +93,7 @@
         allow(wiki_file).to receive(:mime_type).and_return('image/jpeg')
         allow_any_instance_of(ProjectWiki).to receive(:find_file).with('image.jpg', nil).and_return(wiki_file)
 
-        expect(page).to have_xpath('//img[@data-src="image.jpg"]')
+        expect(page).to have_xpath("//img[@data-src='#{project.wiki.wiki_base_path}/image.jpg']")
         expect(page).to have_link('image', href: "#{project.wiki.wiki_base_path}/image.jpg")
 
         click_on('image')

spec/lib/banzai/filter/wiki_link_filter_spec.rb

@@ -7,6 +7,7 @@
   let(:project)   { build_stubbed(:project, :public, name: "wiki_link_project", namespace: namespace) }
   let(:user) { double }
   let(:wiki) { ProjectWiki.new(project, user) }
+  let(:repository_upload_folder) { Wikis::CreateAttachmentService::ATTACHMENT_PATH }
 
   it "doesn't rewrite absolute links" do
     filtered_link = filter("<a href='http://example.com:8000/'>Link</a>", project_wiki: wiki).children[0]
@@ -20,6 +21,45 @@
     expect(filtered_link.attribute('href').value).to eq('/uploads/a.test')
   end
 
+  describe "when links point to the #{Wikis::CreateAttachmentService::ATTACHMENT_PATH} folder" do
+    context 'with an "a" html tag' do
+      it 'rewrites links' do
+        filtered_link = filter("<a href='#{repository_upload_folder}/a.test'>Link</a>", project_wiki: wiki).children[0]
+
+        expect(filtered_link.attribute('href').value).to eq("#{wiki.wiki_base_path}/#{repository_upload_folder}/a.test")
+      end
+    end
+
+    context 'with "img" html tag' do
+      let(:path) { "#{wiki.wiki_base_path}/#{repository_upload_folder}/a.jpg" }
+
+      context 'inside an "a" html tag' do
+        it 'rewrites links' do
+          filtered_elements = filter("<a href='#{repository_upload_folder}/a.jpg'><img src='#{repository_upload_folder}/a.jpg'>example</img></a>", project_wiki: wiki)
+
+          expect(filtered_elements.search('img').first.attribute('src').value).to eq(path)
+          expect(filtered_elements.search('a').first.attribute('href').value).to eq(path)
+        end
+      end
+
+      context 'outside an "a" html tag' do
+        it 'rewrites links' do
+          filtered_link = filter("<img src='#{repository_upload_folder}/a.jpg'>example</img>", project_wiki: wiki).children[0]
+
+          expect(filtered_link.attribute('src').value).to eq(path)
+        end
+      end
+    end
+
+    context 'with "video" html tag' do
+      it 'rewrites links' do
+        filtered_link = filter("<video src='#{repository_upload_folder}/a.mp4'></video>", project_wiki: wiki).children[0]
+
+        expect(filtered_link.attribute('src').value).to eq("#{wiki.wiki_base_path}/#{repository_upload_folder}/a.mp4")
+      end
+    end
+  end
+
   describe "invalid links" do
     invalid_links = ["http://:8080", "http://", "http://:8080/path"]
 

spec/lib/gitlab/file_markdown_link_builder_spec.rb

+# frozen_string_literal: true
+require 'rails_helper'
+
+describe Gitlab::FileMarkdownLinkBuilder do
+  let(:custom_class) do
+    Class.new do
+      include Gitlab::FileMarkdownLinkBuilder
+    end.new
+  end
+
+  before do
+    allow(custom_class).to receive(:filename).and_return(filename)
+  end
+
+  describe 'markdown_link' do
+    let(:url) { "/uploads/#{filename}"}