Commit a91bacf5 authored by Rémy Coutable's avatar Rémy Coutable

Merge branch 'am-fix-pagination-relative-links' into 'master'

Properly implement API pagination headers and add specs

Closes #57684

See merge request gitlab-org/gitlab-ce!25267
parents d133bf84 38bbc097
...@@ -13,6 +13,33 @@ def paginate(relation) ...@@ -13,6 +13,33 @@ def paginate(relation)
strategy.new(self).paginate(relation) strategy.new(self).paginate(relation)
end end
class Base
private
def per_page
@per_page ||= params[:per_page]
end
def base_request_uri
@base_request_uri ||= URI.parse(request.url).tap do |uri|
uri.host = Gitlab.config.gitlab.host
uri.port = nil
end
end
def build_page_url(query_params:)
base_request_uri.tap do |uri|
uri.query = query_params
end.to_s
end
def page_href(next_page_params = {})
query_params = params.merge(**next_page_params, per_page: per_page).to_query
build_page_url(query_params: query_params)
end
end
class KeysetPaginationInfo class KeysetPaginationInfo
attr_reader :relation, :request_context attr_reader :relation, :request_context
...@@ -85,7 +112,7 @@ def keys ...@@ -85,7 +112,7 @@ def keys
end end
end end
class KeysetPaginationStrategy class KeysetPaginationStrategy < Base
attr_reader :request_context attr_reader :request_context
delegate :params, :header, :request, to: :request_context delegate :params, :header, :request, to: :request_context
...@@ -141,10 +168,6 @@ def conditions(pagination) ...@@ -141,10 +168,6 @@ def conditions(pagination)
] ]
end end
def per_page
params[:per_page]
end
def add_default_pagination_headers def add_default_pagination_headers
header 'X-Per-Page', per_page.to_s header 'X-Per-Page', per_page.to_s
end end
...@@ -154,22 +177,12 @@ def add_navigation_links(next_page_params) ...@@ -154,22 +177,12 @@ def add_navigation_links(next_page_params)
header 'Link', link_for('next', next_page_params) header 'Link', link_for('next', next_page_params)
end end
def page_href(next_page_params)
request_url = request.url.split('?').first
request_params = params.dup
request_params[:per_page] = per_page
request_params.merge!(next_page_params) if next_page_params
"#{request_url}?#{request_params.to_query}"
end
def link_for(rel, next_page_params) def link_for(rel, next_page_params)
%(<#{page_href(next_page_params)}>; rel="#{rel}") %(<#{page_href(next_page_params)}>; rel="#{rel}")
end end
end end
class DefaultPaginationStrategy class DefaultPaginationStrategy < Base
attr_reader :request_context attr_reader :request_context
delegate :params, :header, :request, to: :request_context delegate :params, :header, :request, to: :request_context
...@@ -198,15 +211,13 @@ def paginate_with_limit_optimization(relation) ...@@ -198,15 +211,13 @@ def paginate_with_limit_optimization(relation)
end end
end end
# rubocop: disable CodeReuse/ActiveRecord
def add_default_order(relation) def add_default_order(relation)
if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty? if relation.is_a?(ActiveRecord::Relation) && relation.order_values.empty?
relation = relation.order(:id) relation = relation.order(:id) # rubocop: disable CodeReuse/ActiveRecord
end end
relation relation
end end
# rubocop: enable CodeReuse/ActiveRecord
def add_pagination_headers(paginated_data) def add_pagination_headers(paginated_data)
header 'X-Per-Page', paginated_data.limit_value.to_s header 'X-Per-Page', paginated_data.limit_value.to_s
...@@ -222,27 +233,13 @@ def add_pagination_headers(paginated_data) ...@@ -222,27 +233,13 @@ def add_pagination_headers(paginated_data)
end end
def pagination_links(paginated_data) def pagination_links(paginated_data)
request_url = request.url.split('?').first [].tap do |links|
request_params = params.clone links << %(<#{page_href(page: paginated_data.prev_page)}>; rel="prev") if paginated_data.prev_page
request_params[:per_page] = paginated_data.limit_value links << %(<#{page_href(page: paginated_data.next_page)}>; rel="next") if paginated_data.next_page
links << %(<#{page_href(page: 1)}>; rel="first")
links = []
request_params[:page] = paginated_data.prev_page
links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") if request_params[:page]
request_params[:page] = paginated_data.next_page
links << %(<#{request_url}?#{request_params.to_query}>; rel="next") if request_params[:page]
request_params[:page] = 1
links << %(<#{request_url}?#{request_params.to_query}>; rel="first")
unless data_without_counts?(paginated_data)
request_params[:page] = total_pages(paginated_data)
links << %(<#{request_url}?#{request_params.to_query}>; rel="last")
end
links.join(', ') links << %(<#{page_href(page: total_pages(paginated_data))}>; rel="last") unless data_without_counts?(paginated_data)
end.join(', ')
end end
def total_pages(paginated_data) def total_pages(paginated_data)
......
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
describe API::Helpers::Pagination do describe API::Helpers::Pagination do
let(:resource) { Project.all } let(:resource) { Project.all }
let(:incoming_api_projects_url) { "#{Gitlab.config.gitlab.url}:8080/api/v4/projects" }
let(:canonical_api_projects_url) { "#{Gitlab.config.gitlab.url}/api/v4/projects" }
subject do subject do
Class.new.include(described_class).new Class.new.include(described_class).new
...@@ -9,13 +11,19 @@ ...@@ -9,13 +11,19 @@
describe '#paginate (keyset pagination)' do describe '#paginate (keyset pagination)' do
let(:value) { spy('return value') } let(:value) { spy('return value') }
let(:base_query) do
{
pagination: 'keyset',
foo: 'bar',
bar: 'baz'
}
end
let(:query) { base_query }
before do before do
allow(value).to receive(:to_query).and_return(value)
allow(subject).to receive(:header).and_return(value) allow(subject).to receive(:header).and_return(value)
allow(subject).to receive(:params).and_return(value) allow(subject).to receive(:params).and_return(query)
allow(subject).to receive(:request).and_return(value) allow(subject).to receive(:request).and_return(double(url: "#{incoming_api_projects_url}?#{query.to_query}"))
end end
context 'when resource can be paginated' do context 'when resource can be paginated' do
...@@ -28,10 +36,7 @@ ...@@ -28,10 +36,7 @@
end end
describe 'first page' do describe 'first page' do
before do let(:query) { base_query.merge(per_page: 2) }
allow(subject).to receive(:params)
.and_return({ pagination: 'keyset', per_page: 2 })
end
it 'returns appropriate amount of resources' do it 'returns appropriate amount of resources' do
expect(subject.paginate(resource).count).to eq 2 expect(subject.paginate(resource).count).to eq 2
...@@ -43,7 +48,7 @@ ...@@ -43,7 +48,7 @@
it 'adds appropriate headers' do it 'adds appropriate headers' do
expect_header('X-Per-Page', '2') expect_header('X-Per-Page', '2')
expect_header('X-Next-Page', "#{value}?ks_prev_id=#{projects[1].id}&pagination=keyset&per_page=2") expect_header('X-Next-Page', "#{canonical_api_projects_url}?#{query.merge(ks_prev_id: projects[1].id).to_query}")
expect_header('Link', anything) do |_key, val| expect_header('Link', anything) do |_key, val|
expect(val).to include('rel="next"') expect(val).to include('rel="next"')
...@@ -54,10 +59,7 @@ ...@@ -54,10 +59,7 @@
end end
describe 'second page' do describe 'second page' do
before do let(:query) { base_query.merge(per_page: 2, ks_prev_id: projects[1].id) }
allow(subject).to receive(:params)
.and_return({ pagination: 'keyset', per_page: 2, ks_prev_id: projects[1].id })
end
it 'returns appropriate amount of resources' do it 'returns appropriate amount of resources' do
expect(subject.paginate(resource).count).to eq 1 expect(subject.paginate(resource).count).to eq 1
...@@ -69,7 +71,7 @@ ...@@ -69,7 +71,7 @@
it 'adds appropriate headers' do it 'adds appropriate headers' do
expect_header('X-Per-Page', '2') expect_header('X-Per-Page', '2')
expect_header('X-Next-Page', "#{value}?ks_prev_id=#{projects[2].id}&pagination=keyset&per_page=2") expect_header('X-Next-Page', "#{canonical_api_projects_url}?#{query.merge(ks_prev_id: projects[2].id).to_query}")
expect_header('Link', anything) do |_key, val| expect_header('Link', anything) do |_key, val|
expect(val).to include('rel="next"') expect(val).to include('rel="next"')
...@@ -80,10 +82,7 @@ ...@@ -80,10 +82,7 @@
end end
describe 'third page' do describe 'third page' do
before do let(:query) { base_query.merge(per_page: 2, ks_prev_id: projects[2].id) }
allow(subject).to receive(:params)
.and_return({ pagination: 'keyset', per_page: 2, ks_prev_id: projects[2].id })
end
it 'returns appropriate amount of resources' do it 'returns appropriate amount of resources' do
expect(subject.paginate(resource).count).to eq 0 expect(subject.paginate(resource).count).to eq 0
...@@ -91,6 +90,7 @@ ...@@ -91,6 +90,7 @@
it 'adds appropriate headers' do it 'adds appropriate headers' do
expect_header('X-Per-Page', '2') expect_header('X-Per-Page', '2')
expect_no_header('X-Next-Page')
expect(subject).not_to receive(:header).with('Link') expect(subject).not_to receive(:header).with('Link')
subject.paginate(resource) subject.paginate(resource)
...@@ -99,10 +99,7 @@ ...@@ -99,10 +99,7 @@
context 'if order' do context 'if order' do
context 'is not present' do context 'is not present' do
before do let(:query) { base_query.merge(per_page: 2) }
allow(subject).to receive(:params)
.and_return({ pagination: 'keyset', per_page: 2 })
end
it 'is not present it adds default order(:id) desc' do it 'is not present it adds default order(:id) desc' do
resource.order_values = [] resource.order_values = []
...@@ -144,9 +141,7 @@ ...@@ -144,9 +141,7 @@
# (key is the id) # (key is the id)
end end
it 'it also orders by primary key' do it 'also orders by primary key' do
allow(subject).to receive(:params)
.and_return({ pagination: 'keyset', per_page: 2 })
paginated_relation = subject.paginate(resource) paginated_relation = subject.paginate(resource)
expect(paginated_relation.order_values).to be_present expect(paginated_relation.order_values).to be_present
...@@ -157,46 +152,45 @@ ...@@ -157,46 +152,45 @@
expect(paginated_relation.order_values.second.expr.name).to eq :id expect(paginated_relation.order_values.second.expr.name).to eq :id
end end
it 'it returns the right records (first page)' do it 'returns the right records (first page)' do
allow(subject).to receive(:params)
.and_return({ pagination: 'keyset', per_page: 2 })
result = subject.paginate(resource) result = subject.paginate(resource)
expect(result.first).to eq(projects[1]) expect(result.first).to eq(projects[1])
expect(result.second).to eq(projects[3]) expect(result.second).to eq(projects[3])
end end
it 'it returns the right records (second page)' do describe 'second page' do
allow(subject).to receive(:params) let(:query) { base_query.merge(ks_prev_id: projects[3].id, ks_prev_name: projects[3].name, per_page: 2) }
.and_return({ pagination: 'keyset', ks_prev_id: projects[3].id, ks_prev_name: projects[3].name, per_page: 2 })
result = subject.paginate(resource)
expect(result.first).to eq(projects[2]) it 'returns the right records (second page)' do
expect(result.second).to eq(projects[6]) result = subject.paginate(resource)
end
it 'it returns the right records (third page), note increased per_page' do expect(result.first).to eq(projects[2])
allow(subject).to receive(:params) expect(result.second).to eq(projects[6])
.and_return({ pagination: 'keyset', ks_prev_id: projects[6].id, ks_prev_name: projects[6].name, per_page: 5 }) end
result = subject.paginate(resource)
expect(result.size).to eq(3) it 'returns the right link to the next page' do
expect(result.first).to eq(projects[0]) expect_header('X-Per-Page', '2')
expect(result.second).to eq(projects[4]) expect_header('X-Next-Page', "#{canonical_api_projects_url}?#{query.merge(ks_prev_id: projects[6].id, ks_prev_name: projects[6].name).to_query}")
expect(result.last).to eq(projects[5]) expect_header('Link', anything) do |_key, val|
expect(val).to include('rel="next"')
end
subject.paginate(resource)
end
end end
it 'it returns the right link to the next page' do describe 'third page' do
allow(subject).to receive(:params) let(:query) { base_query.merge(ks_prev_id: projects[6].id, ks_prev_name: projects[6].name, per_page: 5) }
.and_return({ pagination: 'keyset', ks_prev_id: projects[3].id, ks_prev_name: projects[3].name, per_page: 2 })
expect_header('X-Per-Page', '2') it 'returns the right records (third page), note increased per_page' do
expect_header('X-Next-Page', "#{value}?ks_prev_id=#{projects[6].id}&ks_prev_name=#{projects[6].name}&pagination=keyset&per_page=2") result = subject.paginate(resource)
expect_header('Link', anything) do |_key, val|
expect(val).to include('rel="next"')
end
subject.paginate(resource) expect(result.size).to eq(3)
expect(result.first).to eq(projects[0])
expect(result.second).to eq(projects[4])
expect(result.last).to eq(projects[5])
end
end end
end end
end end
...@@ -205,25 +199,13 @@ ...@@ -205,25 +199,13 @@
describe '#paginate (default offset-based pagination)' do describe '#paginate (default offset-based pagination)' do
let(:value) { spy('return value') } let(:value) { spy('return value') }
let(:base_query) { { foo: 'bar', bar: 'baz' } }
let(:query) { base_query }
before do before do
allow(value).to receive(:to_query).and_return(value)
allow(subject).to receive(:header).and_return(value) allow(subject).to receive(:header).and_return(value)
allow(subject).to receive(:params).and_return(value) allow(subject).to receive(:params).and_return(query)
allow(subject).to receive(:request).and_return(value) allow(subject).to receive(:request).and_return(double(url: "#{incoming_api_projects_url}?#{query.to_query}"))
end
describe 'required instance methods' do
let(:return_spy) { spy }
it 'requires some instance methods' do
expect_message(:header)
expect_message(:params)
expect_message(:request)
subject.paginate(resource)
end
end end
context 'when resource can be paginated' do context 'when resource can be paginated' do
...@@ -232,11 +214,6 @@ ...@@ -232,11 +214,6 @@
end end
describe 'first page' do describe 'first page' do
before do
allow(subject).to receive(:params)
.and_return({ page: 1, per_page: 2 })
end
shared_examples 'response with pagination headers' do shared_examples 'response with pagination headers' do
it 'adds appropriate headers' do it 'adds appropriate headers' do
expect_header('X-Total', '3') expect_header('X-Total', '3')
...@@ -247,9 +224,9 @@ ...@@ -247,9 +224,9 @@
expect_header('X-Prev-Page', '') expect_header('X-Prev-Page', '')
expect_header('Link', anything) do |_key, val| expect_header('Link', anything) do |_key, val|
expect(val).to include('rel="first"') expect(val).to include(%Q(<#{canonical_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first"))
expect(val).to include('rel="last"') expect(val).to include(%Q(<#{canonical_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="last"))
expect(val).to include('rel="next"') expect(val).to include(%Q(<#{canonical_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="next"))
expect(val).not_to include('rel="prev"') expect(val).not_to include('rel="prev"')
end end
...@@ -267,6 +244,8 @@ ...@@ -267,6 +244,8 @@
end end
end end
let(:query) { base_query.merge(page: 1, per_page: 2) }
context 'when the api_kaminari_count_with_limit feature flag is unset' do context 'when the api_kaminari_count_with_limit feature flag is unset' do
it_behaves_like 'paginated response' it_behaves_like 'paginated response'
it_behaves_like 'response with pagination headers' it_behaves_like 'response with pagination headers'
...@@ -311,9 +290,9 @@ ...@@ -311,9 +290,9 @@
expect_header('X-Prev-Page', '') expect_header('X-Prev-Page', '')
expect_header('Link', anything) do |_key, val| expect_header('Link', anything) do |_key, val|
expect(val).to include('rel="first"') expect(val).to include(%Q(<#{canonical_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first"))
expect(val).to include(%Q(<#{canonical_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="next"))
expect(val).not_to include('rel="last"') expect(val).not_to include('rel="last"')
expect(val).to include('rel="next"')
expect(val).not_to include('rel="prev"') expect(val).not_to include('rel="prev"')
end end
...@@ -324,10 +303,7 @@ ...@@ -324,10 +303,7 @@
end end
describe 'second page' do describe 'second page' do
before do let(:query) { base_query.merge(page: 2, per_page: 2) }
allow(subject).to receive(:params)
.and_return({ page: 2, per_page: 2 })
end
it 'returns appropriate amount of resources' do it 'returns appropriate amount of resources' do
expect(subject.paginate(resource).count).to eq 1 expect(subject.paginate(resource).count).to eq 1
...@@ -342,9 +318,9 @@ ...@@ -342,9 +318,9 @@
expect_header('X-Prev-Page', '1') expect_header('X-Prev-Page', '1')
expect_header('Link', anything) do |_key, val| expect_header('Link', anything) do |_key, val|
expect(val).to include('rel="first"') expect(val).to include(%Q(<#{canonical_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first"))
expect(val).to include('rel="last"') expect(val).to include(%Q(<#{canonical_api_projects_url}?#{query.merge(page: 2).to_query}>; rel="last"))
expect(val).to include('rel="prev"') expect(val).to include(%Q(<#{canonical_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="prev"))
expect(val).not_to include('rel="next"') expect(val).not_to include('rel="next"')
end end
...@@ -376,10 +352,7 @@ ...@@ -376,10 +352,7 @@
context 'when resource empty' do context 'when resource empty' do
describe 'first page' do describe 'first page' do
before do let(:query) { base_query.merge(page: 1, per_page: 2) }
allow(subject).to receive(:params)
.and_return({ page: 1, per_page: 2 })
end
it 'returns appropriate amount of resources' do it 'returns appropriate amount of resources' do
expect(subject.paginate(resource).count).to eq 0 expect(subject.paginate(resource).count).to eq 0
...@@ -394,8 +367,8 @@ ...@@ -394,8 +367,8 @@
expect_header('X-Prev-Page', '') expect_header('X-Prev-Page', '')
expect_header('Link', anything) do |_key, val| expect_header('Link', anything) do |_key, val|
expect(val).to include('rel="first"') expect(val).to include(%Q(<#{canonical_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="first"))
expect(val).to include('rel="last"') expect(val).to include(%Q(<#{canonical_api_projects_url}?#{query.merge(page: 1).to_query}>; rel="last"))
expect(val).not_to include('rel="prev"') expect(val).not_to include('rel="prev"')
expect(val).not_to include('rel="next"') expect(val).not_to include('rel="next"')
expect(val).not_to include('page=0') expect(val).not_to include('page=0')
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Serializer::Pagination do describe Gitlab::Serializer::Pagination do
let(:request) { spy('request') } let(:request) { double(url: "#{Gitlab.config.gitlab.url}:8080/api/v4/projects?#{query.to_query}", query_parameters: query) }
let(:response) { spy('response') } let(:response) { spy('response') }
let(:headers) { spy('headers') } let(:headers) { spy('headers') }
before do before do
allow(request).to receive(:query_parameters) allow(response).to receive(:headers).and_return(headers)
.and_return(params)
allow(response).to receive(:headers)
.and_return(headers)
end end
let(:pagination) { described_class.new(request, response) } let(:pagination) { described_class.new(request, response) }
...@@ -19,7 +15,7 @@ ...@@ -19,7 +15,7 @@