Commit 4e5897f5 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'tmp-reference-pipeline-and-caching' into 'master'

[Second try] Implement different Markdown rendering pipelines and cache Markdown

!1602 already got merged in bcd89a58, but it would appear the merge commit disappeared because of #3816 (or some other reason).

cc @rspeicher 

See merge request !2051
parents 3bfedbd2 10387f6b
......@@ -20,7 +20,7 @@ def link_to_gfm(body, url, html_options = {})
end
user = current_user if defined?(current_user)
gfm_body = Gitlab::Markdown.gfm(escaped_body, project: @project, current_user: user)
gfm_body = Gitlab::Markdown.render(escaped_body, project: @project, current_user: user, pipeline: :single_line)
fragment = Nokogiri::HTML::DocumentFragment.parse(gfm_body)
if fragment.children.size == 1 && fragment.children[0].name == 'a'
......@@ -46,23 +46,35 @@ def link_to_gfm(body, url, html_options = {})
end
def markdown(text, context = {})
process_markdown(text, context)
end
return "" unless text.present?
context[:project] ||= @project
# TODO (rspeicher): Remove all usages of this helper and just call `markdown`
# with a custom pipeline depending on the content being rendered
def gfm(text, options = {})
process_markdown(text, options, :gfm)
html = Gitlab::Markdown.render(text, context)
context.merge!(
current_user: (current_user if defined?(current_user)),
# RelativeLinkFilter
requested_path: @path,
project_wiki: @project_wiki,
ref: @ref
)
Gitlab::Markdown.post_process(html, context)
end
def asciidoc(text)
Gitlab::Asciidoc.render(text, {
commit: @commit,
project: @project,
project_wiki: @project_wiki,
Gitlab::Asciidoc.render(text,
project: @project,
current_user: (current_user if defined?(current_user)),
# RelativeLinkFilter
project_wiki: @project_wiki,
requested_path: @path,
ref: @ref
})
ref: @ref,
commit: @commit
)
end
# Return the first line of +text+, up to +max_chars+, after parsing the line
......@@ -178,26 +190,4 @@ def cross_project_reference(project, entity)
''
end
end
def process_markdown(text, options, method = :markdown)
return "" unless text.present?
options.reverse_merge!(
path: @path,
pipeline: :default,
project: @project,
project_wiki: @project_wiki,
ref: @ref
)
user = current_user if defined?(current_user)
html = if method == :gfm
Gitlab::Markdown.gfm(text, options)
else
Gitlab::Markdown.render(text, options)
end
Gitlab::Markdown.post_process(html, pipeline: options[:pipeline], project: @project, user: user)
end
end
......@@ -7,7 +7,7 @@ class Commit
include Referable
include StaticModel
attr_mentionable :safe_message
attr_mentionable :safe_message, pipeline: :single_line
participant :author, :committer, :notes
attr_accessor :project
......
......@@ -50,7 +50,8 @@ module Issuable
allow_nil: true,
prefix: true
attr_mentionable :title, :description
attr_mentionable :title, pipeline: :single_line
attr_mentionable :description, cache: true
participant :author, :assignee, :notes_with_associations
strip_attributes :title
end
......
......@@ -10,8 +10,9 @@ module Mentionable
module ClassMethods
# Indicate which attributes of the Mentionable to search for GFM references.
def attr_mentionable(*attrs)
mentionable_attrs.concat(attrs.map(&:to_s))
def attr_mentionable(attr, options = {})
attr = attr.to_s
mentionable_attrs << [attr, options]
end
# Accessor for attributes marked mentionable.
......@@ -37,19 +38,24 @@ def gfm_reference(from_project = nil)
"#{friendly_name} #{to_reference(from_project)}"
end
# Construct a String that contains possible GFM references.
def mentionable_text
self.class.mentionable_attrs.map { |attr| send(attr) }.compact.join("\n\n")
end
# The GFM reference to this Mentionable, which shouldn't be included in its #references.
def local_reference
self
end
def all_references(current_user = self.author, text = self.mentionable_text, load_lazy_references: true)
def all_references(current_user = self.author, text = nil, load_lazy_references: true)
ext = Gitlab::ReferenceExtractor.new(self.project, current_user, load_lazy_references: load_lazy_references)
ext.analyze(text)
if text
ext.analyze(text)
else
self.class.mentionable_attrs.each do |attr, options|
text = send(attr)
options[:cache_key] = [self, attr] if options.delete(:cache)
ext.analyze(text, options)
end
end
ext
end
......@@ -58,9 +64,7 @@ def mentioned_users(current_user = nil, load_lazy_references: true)
end
# Extract GFM references to other Mentionables from this Mentionable. Always excludes its #local_reference.
def referenced_mentionables(current_user = self.author, text = self.mentionable_text, load_lazy_references: true)
return [] if text.blank?
def referenced_mentionables(current_user = self.author, text = nil, load_lazy_references: true)
refs = all_references(current_user, text, load_lazy_references: load_lazy_references)
refs = (refs.issues + refs.merge_requests + refs.commits)
......@@ -70,8 +74,8 @@ def referenced_mentionables(current_user = self.author, text = self.mentionable_
refs.reject { |ref| ref == local_reference }
end
# Create a cross-reference Note for each GFM reference to another Mentionable found in +mentionable_text+.
def create_cross_references!(author = self.author, without = [], text = self.mentionable_text)
# Create a cross-reference Note for each GFM reference to another Mentionable found in the +mentionable_attrs+.
def create_cross_references!(author = self.author, without = [], text = nil)
refs = referenced_mentionables(author, text)
# We're using this method instead of Array diffing because that requires
......@@ -111,7 +115,7 @@ def create_new_cross_references!(author = self.author)
def detect_mentionable_changes
source = (changes.present? ? changes : previous_changes).dup
mentionable = self.class.mentionable_attrs
mentionable = self.class.mentionable_attrs.map { |attr, options| attr }
# Only include changed fields that are mentionable
source.select { |key, val| mentionable.include?(key) }
......
......@@ -29,7 +29,7 @@ class Note < ActiveRecord::Base
default_value_for :system, false
attr_mentionable :note
attr_mentionable :note, cache: true, pipeline: :note
participant :author
belongs_to :project
......
......@@ -662,6 +662,7 @@ def rename_repo
gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
reset_events_cache
@repository = nil
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
......
......@@ -12,7 +12,7 @@
.gray-content-block.middle-block
%h2.issue-title
= gfm escape_once(@milestone.title)
= markdown escape_once(@milestone.title), pipeline: :single_line
- if @milestone.complete? && @milestone.active?
.alert.alert-success.prepend-top-default
......
......@@ -2,4 +2,4 @@
.commit-row-title
= link_to truncate_sha(commit[:id]), namespace_project_commit_path(project.namespace, project, commit[:id]), class: "commit_short_id", alt: ''
&middot;
= gfm event_commit_title(commit[:message]), project: project
= markdown event_commit_title(commit[:message]), project: project, pipeline: :single_line
......@@ -18,7 +18,7 @@
.gray-content-block.middle-block
%h2.issue-title
= gfm escape_once(@milestone.title)
= markdown escape_once(@milestone.title), pipeline: :single_line
- if @milestone.complete? && @milestone.active?
.alert.alert-success.prepend-top-default
......
......@@ -52,10 +52,10 @@
.commit-box.gray-content-block.middle-block
%h3.commit-title
= gfm escape_once(@commit.title)
= markdown escape_once(@commit.title), pipeline: :single_line
- if @commit.description.present?
%pre.commit-description
= preserve(gfm(escape_once(@commit.description)))
= preserve(markdown(escape_once(@commit.description), pipeline: :single_line))
:javascript
$(".commit-info-row.branches").load("#{branches_namespace_project_commit_path(@project.namespace, @project, @commit.id)}");
......@@ -32,7 +32,7 @@
- if commit.description?
.commit-row-description.js-toggle-content
%pre
= preserve(gfm(escape_once(commit.description)))
= preserve(markdown(escape_once(commit.description), pipeline: :single_line))
.commit-row-info
= commit_author_link(commit, avatar: true, size: 24)
......
......@@ -17,7 +17,7 @@ xml.feed "xmlns" => "http://www.w3.org/2005/Atom", "xmlns:media" => "http://sear
xml.name commit.author_name
xml.email commit.author_email
end
xml.summary gfm(commit.description)
xml.summary markdown(commit.description, pipeline: :single_line)
end
end
end
......@@ -24,10 +24,10 @@
.col-sm-10
= f.text_area :description, class: "form-control", rows: 3, maxlength: 250
- if @project.repository.exists? && @project.repository.branch_names.any?
- unless @project.empty_repo?
.form-group
= f.label :default_branch, "Default Branch", class: 'control-label'
.col-sm-10= f.select(:default_branch, @repository.branch_names, {}, {class: 'select2 select-wide'})
.col-sm-10= f.select(:default_branch, @project.repository.branch_names, {}, {class: 'select2 select-wide'})
= render 'shared/visibility_level', f: f, visibility_level: @project.visibility_level, can_change_visibility_level: can_change_visibility_level?(@project, current_user), form_model: @project
......
.issue-closed-by-widget
= icon('check')
This issue will be closed automatically when merge request #{gfm(merge_requests_sentence(@closed_by_merge_requests))} is accepted.
This issue will be closed automatically when merge request #{markdown(merge_requests_sentence(@closed_by_merge_requests), pipeline: :gfm)} is accepted.
......@@ -38,13 +38,13 @@
.gray-content-block.middle-block
%h2.issue-title
= gfm escape_once(@issue.title)
= markdown escape_once(@issue.title), pipeline: :single_line
%div
- if @issue.description.present?
.description{class: can?(current_user, :update_issue, @issue) ? 'js-task-list-container' : ''}
.wiki
= preserve do
= markdown(@issue.description)
= markdown(@issue.description, cache_key: [@issue, "description"])
%textarea.hidden.js-task-list-field
= @issue.description
- if @closed_by_merge_requests.present?
......
.gray-content-block.middle-block
%h2.issue-title
= gfm escape_once(@merge_request.title)
= markdown escape_once(@merge_request.title), pipeline: :single_line
%div
- if @merge_request.description.present?
.description{class: can?(current_user, :update_merge_request, @merge_request) ? 'js-task-list-container' : ''}
.wiki
= preserve do
= markdown(@merge_request.description)
= markdown(@merge_request.description, cache_key: [@merge_request, "description"])
%textarea.hidden.js-task-list-field
= @merge_request.description
......@@ -26,4 +26,4 @@
%i.fa.fa-check
Accepting this merge request will close #{"issue".pluralize(@closes_issues.size)}
= succeed '.' do
!= gfm(issues_sentence(@closes_issues))
!= markdown issues_sentence(@closes_issues), pipeline: :gfm
......@@ -32,7 +32,7 @@
.gray-content-block.middle-block
%h2.issue-title
= gfm escape_once(@milestone.title)
= markdown escape_once(@milestone.title), pipeline: :single_line
%div
- if @milestone.description.present?
.description
......
......@@ -38,7 +38,7 @@
.note-body{class: note_editable?(note) ? 'js-task-list-container' : ''}
.note-text
= preserve do
= markdown(note.note, {no_header_anchors: true})
= markdown(note.note, pipeline: :note, cache_key: [note, "note"])
- if note_editable?(note)
= render 'projects/notes/edit_form', note: note
......
......@@ -12,7 +12,7 @@
= link_to namespace_project_commits_path(@project.namespace, @project, commit.id) do
%code= commit.short_id
= image_tag avatar_icon(commit.author_email), class: "", width: 16, alt: ''
= gfm escape_once(truncate(commit.title, length: 40))
= markdown escape_once(truncate(commit.title, length: 40)), pipeline: :single_line
%td
%span.pull-right.cgray
= time_ago_with_tooltip(commit.committed_date)
......@@ -22,4 +22,4 @@
.gray-content-block.middle-block
%h2.issue-title
= gfm escape_once(@snippet.title)
= markdown escape_once(@snippet.title), pipeline: :single_line
......@@ -7,6 +7,8 @@
# and recreated between test runs. Don't rely on the data there!
config.cache_classes = false
config.cache_store = :null_store
# Configure static asset server for tests with Cache-Control for performance
config.serve_static_files = true
config.static_cache_control = "public, max-age=3600"
......
......@@ -10,7 +10,9 @@
Settings.gitlab['session_expire_delay'] ||= 10080
end
unless Rails.env.test?
if Rails.env.test?
Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
else
Gitlab::Application.config.session_store(
:redis_store, # Using the cookie_store would enable session replay attacks.
servers: Rails.application.config.cache_store[1].merge(namespace: 'session:gitlab'), # re-use the Redis config from the Rails cache store
......
require 'asciidoctor'
require 'html/pipeline'
module Gitlab
# Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters
# the resulting HTML through HTML pipeline filters.
module Asciidoc
# Provide autoload paths for filters to prevent a circular dependency error
autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter'
DEFAULT_ADOC_ATTRS = [
'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab',
'env-gitlab', 'source-highlighter=html-pipeline'
......@@ -24,13 +20,11 @@ module Asciidoc
# :requested_path
# :ref
# asciidoc_opts - a Hash of options to pass to the Asciidoctor converter
# html_opts - a Hash of options for HTML output:
# :xhtml - output XHTML instead of HTML
#
def self.render(input, context, asciidoc_opts = {}, html_opts = {})
asciidoc_opts = asciidoc_opts.reverse_merge(
def self.render(input, context, asciidoc_opts = {})
asciidoc_opts.reverse_merge!(
safe: :secure,
backend: html_opts[:xhtml] ? :xhtml5 : :html5,
backend: :html5,
attributes: []
)
asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS)
......@@ -38,23 +32,10 @@ def self.render(input, context, asciidoc_opts = {}, html_opts = {})
html = ::Asciidoctor.convert(input, asciidoc_opts)
if context[:project]
result = HTML::Pipeline.new(filters).call(html, context)
save_opts = html_opts[:xhtml] ?
Nokogiri::XML::Node::SaveOptions::AS_XHTML : 0
html = result[:output].to_html(save_with: save_opts)
html = Gitlab::Markdown.render(html, context.merge(pipeline: :asciidoc))
end
html.html_safe
end
private
def self.filters
[
Gitlab::Markdown::RelativeLinkFilter
]
end
end
end
......@@ -19,24 +19,21 @@ module Markdown
# context - Hash of context options passed to our HTML Pipeline
#
# Returns an HTML-safe String
def self.render(markdown, context = {})
html = renderer.render(markdown)
html = gfm(html, context)
html.html_safe
def self.render(text, context = {})
cache_key = context.delete(:cache_key)
cache_key = full_cache_key(cache_key, context[:pipeline])
if cache_key
Rails.cache.fetch(cache_key) do
cacheless_render(text, context)
end
else
cacheless_render(text, context)
end
end
# Convert a Markdown String into HTML without going through the HTML
# Pipeline.
#
# Note that because the pipeline is skipped, SanitizationFilter is as well.
# Do not output the result of this method to the user.
#
# markdown - Markdown String
#
# Returns a String
def self.render_without_gfm(markdown)
renderer.render(markdown)
def self.render_result(text, context = {})
Pipeline[context[:pipeline]].call(text, context)
end
# Perform post-processing on an HTML String
......@@ -46,156 +43,73 @@ def self.render_without_gfm(markdown)
# permission to make (`RedactorFilter`).
#
# html - String to process
# options - Hash of options to customize output
# context - Hash of options to customize output
# :pipeline - Symbol pipeline type
# :project - Project
# :user - User object
#
# Returns an HTML-safe String
def self.post_process(html, options)
context = {
project: options[:project],
current_user: options[:user]
}
doc = post_processor.to_document(html, context)
def self.post_process(html, context)
context = Pipeline[context[:pipeline]].transform_context(context)
if options[:pipeline] == :atom
doc.to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML)
pipeline = Pipeline[:post_process]
if context[:xhtml]
pipeline.to_document(html, context).to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML)
else
doc.to_html
pipeline.to_html(html, context)
end.html_safe
end
# Provide autoload paths for filters to prevent a circular dependency error
autoload :AutolinkFilter, 'gitlab/markdown/autolink_filter'
autoload :CommitRangeReferenceFilter, 'gitlab/markdown/commit_range_reference_filter'
autoload :CommitReferenceFilter, 'gitlab/markdown/commit_reference_filter'
autoload :EmojiFilter, 'gitlab/markdown/emoji_filter'
autoload :ExternalIssueReferenceFilter, 'gitlab/markdown/external_issue_reference_filter'
autoload :ExternalLinkFilter, 'gitlab/markdown/external_link_filter'
autoload :IssueReferenceFilter, 'gitlab/markdown/issue_reference_filter'
autoload :LabelReferenceFilter, 'gitlab/markdown/label_reference_filter'
autoload :MergeRequestReferenceFilter, 'gitlab/markdown/merge_request_reference_filter'
autoload :RedactorFilter, 'gitlab/markdown/redactor_filter'
autoload :RelativeLinkFilter, 'gitlab/markdown/relative_link_filter'
autoload :SanitizationFilter, 'gitlab/markdown/sanitization_filter'
autoload :SnippetReferenceFilter, 'gitlab/markdown/snippet_reference_filter'
autoload :SyntaxHighlightFilter, 'gitlab/markdown/syntax_highlight_filter'
autoload :TableOfContentsFilter, 'gitlab/markdown/table_of_contents_filter'
autoload :TaskListFilter, 'gitlab/markdown/task_list_filter'
autoload :UserReferenceFilter, 'gitlab/markdown/user_reference_filter'
autoload :UploadLinkFilter, 'gitlab/markdown/upload_link_filter'
# Public: Parse the provided HTML with GitLab-Flavored Markdown
#
# html - HTML String
# options - A Hash of options used to customize output (default: {})
# :no_header_anchors - Disable header anchors in TableOfContentsFilter
# :path - Current path String
# :pipeline - Symbol pipeline type
# :project - Current Project object
# :project_wiki - Current ProjectWiki object
# :ref - Current ref String
#
# Returns an HTML-safe String
def self.gfm(html, options = {})
return '' unless html.present?
@pipeline ||= HTML::Pipeline.new(filters)
context = {
# SanitizationFilter
pipeline: options[:pipeline],
# EmojiFilter
asset_host: Gitlab::Application.config.asset_host,
asset_root: Gitlab.config.gitlab.base_url,
# ReferenceFilter
only_path: only_path_pipeline?(options[:pipeline]),
project: options[:project],
# RelativeLinkFilter
project_wiki: options[:project_wiki],
ref: options[:ref],
requested_path: options[:path],
# TableOfContentsFilter
no_header_anchors: options[:no_header_anchors]
}
@pipeline.to_html(html, context).html_safe
end
private
# Check if a pipeline enables the `only_path` context option
#
# Returns Boolean
def self.only_path_pipeline?(pipeline)
case pipeline
when :atom, :email
false
else
true
end
end
def self.redcarpet_options
# https://github.com/vmg/redcarpet#and-its-like-really-simple-to-use
@redcarpet_options ||= {
fenced_code_blocks: true,
footnotes: true,
lax_spacing: true,
no_intra_emphasis: true,