Commit c3d972f4 authored by Nick Thomas's avatar Nick Thomas

Add terminals to the Kubernetes deployment service

parent 53783027
......@@ -9,7 +9,7 @@ class Projects::EnvironmentsController < Projects::ApplicationController
def index
@scope = params[:scope]
@environments = project.environments
respond_to do |format|
format.html
format.json do
......@@ -56,6 +56,29 @@ class Projects::EnvironmentsController < Projects::ApplicationController
redirect_to polymorphic_path([project.namespace.becomes(Namespace), project, new_action])
end
def terminal
# Currently, this acts as a hint to load the terminal details into the cache
# if they aren't there already. In the future, users will need these details
# to choose between terminals to connect to.
@terminals = environment.terminals
end
# GET .../terminal.ws : implemented in gitlab-workhorse
def terminal_websocket_authorize
Gitlab::Workhorse.verify_api_request!(request.headers)
# Just return the first terminal for now. If the list is in the process of
# being looked up, this may result in a 404 response, so the frontend
# should retry
terminal = environment.terminals.try(:first)
if terminal
set_workhorse_internal_api_content_type
render json: Gitlab::Workhorse.terminal_websocket(terminal)
else
render text: 'Not found', status: 404
end
end
private
def environment_params
......
......@@ -55,6 +55,10 @@ module ReactiveCaching
self.reactive_cache_refresh_interval = 1.minute
self.reactive_cache_lifetime = 10.minutes
def calculate_reactive_cache
raise NotImplementedError
end
def with_reactive_cache(&blk)
within_reactive_cache_lifetime do
data = Rails.cache.read(full_reactive_cache_key)
......
......@@ -128,6 +128,14 @@ class Environment < ActiveRecord::Base
end
end
def has_terminals?
project.deployment_service.present? && available? && last_deployment.present?
end
def terminals
project.deployment_service.terminals(self) if has_terminals?
end
# An environment name is not necessarily suitable for use in URLs, DNS
# or other third-party contexts, so provide a slugified version. A slug has
# the following properties:
......
......@@ -12,4 +12,22 @@ class DeploymentService < Service
def predefined_variables
[]
end
# Environments may have a number of terminals. Should return an array of
# hashes describing them, e.g.:
#
# [{
# :selectors => {"a" => "b", "foo" => "bar"},
# :url => "wss://external.example.com/exec",
# :headers => {"Authorization" => "Token xxx"},
# :subprotocols => ["foo"],
# :ca_pem => "----BEGIN CERTIFICATE...", # optional
# :created_at => Time.now.utc
# }]
#
# Selectors should be a set of values that uniquely identify a particular
# terminal
def terminals(environment)
raise NotImplementedError
end
end
class KubernetesService < DeploymentService
include Gitlab::Kubernetes
include ReactiveCaching
self.reactive_cache_key = ->(service) { [ service.class.model_name.singular, service.project_id ] }
# Namespace defaults to the project path, but can be overridden in case that
# is an invalid or inappropriate name
prop_accessor :namespace
......@@ -25,6 +30,8 @@ class KubernetesService < DeploymentService
length: 1..63
end
after_save :clear_reactive_cache!
def initialize_properties
if properties.nil?
self.properties = {}
......@@ -41,7 +48,8 @@ class KubernetesService < DeploymentService
end
def help
''
'To enable terminal access to Kubernetes environments, label your ' \
'deployments with `app=$CI_ENVIRONMENT_SLUG`'
end
def to_param
......@@ -75,9 +83,9 @@ class KubernetesService < DeploymentService
# Check we can connect to the Kubernetes API
def test(*args)
kubeclient = build_kubeclient
kubeclient.discover
kubeclient = build_kubeclient!
kubeclient.discover
{ success: kubeclient.discovered, result: "Checked API discovery endpoint" }
rescue => err
{ success: false, result: err }
......@@ -93,20 +101,48 @@ class KubernetesService < DeploymentService
variables
end
private
# Constructs a list of terminals from the reactive cache
#
# Returns nil if the cache is empty, in which case you should try again a
# short time later
def terminals(environment)
with_reactive_cache do |data|
pods = data.fetch(:pods, nil)
filter_pods(pods, app: environment.slug).
flat_map { |pod| terminals_for_pod(api_url, namespace, pod) }.
map { |terminal| add_terminal_auth(terminal, token, ca_pem) }
end
end
def build_kubeclient(api_path = '/api', api_version = 'v1')
return nil unless api_url && namespace && token
# Caches all pods in the namespace so other calls don't need to block on
# network access.
def calculate_reactive_cache
return unless active? && project && !project.pending_delete?
url = URI.parse(api_url)
url.path = url.path[0..-2] if url.path[-1] == "/"
url.path += api_path
kubeclient = build_kubeclient!
# Store as hashes, rather than as third-party types
pods = begin
kubeclient.get_pods(namespace: namespace).as_json
rescue KubeException => err
raise err unless err.error_code == 404
[]
end
# We may want to cache extra things in the future
{ pods: pods }
end
private
def build_kubeclient!(api_path: 'api', api_version: 'v1')
raise "Incomplete settings" unless api_url && namespace && token
::Kubeclient::Client.new(
url,
join_api_url(api_path),
api_version,
ssl_options: kubeclient_ssl_options,
auth_options: kubeclient_auth_options,
ssl_options: kubeclient_ssl_options,
http_proxy_uri: ENV['http_proxy']
)
end
......@@ -125,4 +161,13 @@ class KubernetesService < DeploymentService
def kubeclient_auth_options
{ bearer_token: token }
end
def join_api_url(*parts)
url = URI.parse(api_url)
prefix = url.path.sub(%r{/+\z}, '')
url.path = [ prefix, *parts ].join("/")
url.to_s
end
end
......@@ -8,6 +8,7 @@ class EnvironmentEntity < Grape::Entity
expose :environment_type
expose :last_deployment, using: DeploymentEntity
expose :stoppable?
expose :has_terminals?, as: :has_terminals
expose :environment_path do |environment|
namespace_project_environment_path(
......@@ -23,5 +24,12 @@ class EnvironmentEntity < Grape::Entity
environment)
end
expose :terminal_path, if: ->(environment, _) { environment.has_terminals? } do |environment|
terminal_namespace_project_environment_path(
environment.project.namespace,
environment.project,
environment)
end
expose :created_at, :updated_at
end
module Gitlab
# Helper methods to do with Kubernetes network services & resources
module Kubernetes
# This is the comand that is run to start a terminal session. Kubernetes
# expects `command=foo&command=bar, not `command[]=foo&command[]=bar`
EXEC_COMMAND = URI.encode_www_form(
['sh', '-c', 'bash || sh'].map { |value| ['command', value] }
)
# Filters an array of pods (as returned by the kubernetes API) by their labels
def filter_pods(pods, labels = {})
pods.select do |pod|
metadata = pod.fetch("metadata", {})
pod_labels = metadata.fetch("labels", nil)
next unless pod_labels
labels.all? { |k, v| pod_labels[k.to_s] == v }
end
end
# Converts a pod (as returned by the kubernetes API) into a terminal
def terminals_for_pod(api_url, namespace, pod)
metadata = pod.fetch("metadata", {})
status = pod.fetch("status", {})
spec = pod.fetch("spec", {})
containers = spec["containers"]
pod_name = metadata["name"]
phase = status["phase"]
return unless containers.present? && pod_name.present? && phase == "Running"
created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil
containers.map do |container|
{
selectors: { pod: pod_name, container: container["name"] },
url: container_exec_url(api_url, namespace, pod_name, container["name"]),
subprotocols: ['channel.k8s.io'],
headers: Hash.new { |h, k| h[k] = [] },
created_at: created_at,
}
end
end
def add_terminal_auth(terminal, token, ca_pem = nil)
terminal[:headers]['Authorization'] << "Bearer #{token}"
terminal[:ca_pem] = ca_pem if ca_pem.present?
terminal
end
def container_exec_url(api_url, namespace, pod_name, container_name)
url = URI.parse(api_url)
url.path = [
url.path.sub(%r{/+\z}, ''),
'api', 'v1',
'namespaces', ERB::Util.url_encode(namespace),
'pods', ERB::Util.url_encode(pod_name),
'exec'
].join('/')
url.query = {
container: container_name,
tty: true,
stdin: true,
stdout: true,
stderr: true,
}.to_query + '&' + EXEC_COMMAND
case url.scheme
when 'http'
url.scheme = 'ws'
when 'https'
url.scheme = 'wss'
end
url.to_s
end
end
end
......@@ -95,6 +95,19 @@ module Gitlab
]
end
def terminal_websocket(terminal)
details = {
'Terminal' => {
'Subprotocols' => terminal[:subprotocols],
'Url' => terminal[:url],
'Header' => terminal[:headers]
}
}
details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem)
details
end
def version
path = Rails.root.join(VERSION_FILE)
path.readable? ? path.read.chomp : 'unknown'
......
......@@ -71,6 +71,74 @@ describe Projects::EnvironmentsController do
end
end
describe 'GET #terminal' do
context 'with valid id' do
it 'responds with a status code 200' do
get :terminal, environment_params
expect(response).to have_http_status(200)
end
it 'loads the terminals for the enviroment' do
expect_any_instance_of(Environment).to receive(:terminals)
get :terminal, environment_params
end
end
context 'with invalid id' do
it 'responds with a status code 404' do
get :terminal, environment_params(id: 666)
expect(response).to have_http_status(404)
end
end
end
describe 'GET #terminal_websocket_authorize' do
context 'with valid workhorse signature' do
before do
allow(Gitlab::Workhorse).to receive(:verify_api_request!).and_return(nil)
end
context 'and valid id' do
it 'returns the first terminal for the environment' do
expect_any_instance_of(Environment).
to receive(:terminals).
and_return([:fake_terminal])
expect(Gitlab::Workhorse).
to receive(:terminal_websocket).
with(:fake_terminal).
and_return(workhorse: :response)
get :terminal_websocket_authorize, environment_params
expect(response).to have_http_status(200)
expect(response.headers["Content-Type"]).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE)
expect(response.body).to eq('{"workhorse":"response"}')
end
end
context 'and invalid id' do
it 'returns 404' do
get :terminal_websocket_authorize, environment_params(id: 666)
expect(response).to have_http_status(404)
end
end
end
context 'with invalid workhorse signature' do
it 'aborts with an exception' do
allow(Gitlab::Workhorse).to receive(:verify_api_request!).and_raise(JWT::DecodeError)
expect { get :terminal_websocket_authorize, environment_params }.to raise_error(JWT::DecodeError)
# controller tests don't set the response status correctly. It's enough
# to check that the action raised an exception
end
end
end
def environment_params(opts = {})
opts.reverse_merge(namespace_id: project.namespace,
project_id: project,
......
......@@ -140,7 +140,7 @@ FactoryGirl.define do
active: true,
properties: {
namespace: project.path,
api_url: 'https://kubernetes.example.com/api',
api_url: 'https://kubernetes.example.com',
token: 'a' * 40,
}
)
......
require 'spec_helper'
describe Gitlab::Kubernetes do
include described_class
describe '#container_exec_url' do
let(:api_url) { 'https://example.com' }
let(:namespace) { 'default' }
let(:pod_name) { 'pod1' }
let(:container_name) { 'container1' }
subject(:result) { URI::parse(container_exec_url(api_url, namespace, pod_name, container_name)) }
it { expect(result.scheme).to eq('wss') }
it { expect(result.host).to eq('example.com') }
it { expect(result.path).to eq('/api/v1/namespaces/default/pods/pod1/exec') }
it { expect(result.query).to eq('container=container1&stderr=true&stdin=true&stdout=true&tty=true&command=sh&command=-c&command=bash+%7C%7C+sh') }
context 'with a HTTP API URL' do
let(:api_url) { 'http://example.com' }
it { expect(result.scheme).to eq('ws') }
end
context 'with a path prefix in the API URL' do
let(:api_url) { 'https://example.com/prefix/' }
it { expect(result.path).to eq('/prefix/api/v1/namespaces/default/pods/pod1/exec') }
end
context 'with arguments that need urlencoding' do
let(:namespace) { 'default namespace' }
let(:pod_name) { 'pod 1' }
let(:container_name) { 'container 1' }
it { expect(result.path).to eq('/api/v1/namespaces/default%20namespace/pods/pod%201/exec') }
it { expect(result.query).to match(/\Acontainer=container\+1&/) }
end
end
end
......@@ -37,6 +37,42 @@ describe Gitlab::Workhorse, lib: true do
end
end
describe '.terminal_websocket' do
def terminal(ca_pem: nil)
out = {
subprotocols: ['foo'],
url: 'wss://example.com/terminal.ws',
headers: { 'Authorization' => ['Token x'] }
}
out[:ca_pem] = ca_pem if ca_pem
out
end
def workhorse(ca_pem: nil)
out = {
'Terminal' => {
'Subprotocols' => ['foo'],
'Url' => 'wss://example.com/terminal.ws',
'Header' => { 'Authorization' => ['Token x'] }
}
}
out['Terminal']['CAPem'] = ca_pem if ca_pem
out
end
context 'without ca_pem' do
subject { Gitlab::Workhorse.terminal_websocket(terminal) }
it { is_expected.to eq(workhorse) }
end
context 'with ca_pem' do
subject { Gitlab::Workhorse.terminal_websocket(terminal(ca_pem: "foo")) }
it { is_expected.to eq(workhorse(ca_pem: "foo")) }
end
end
describe '.send_git_diff' do
let(:diff_refs) { double(base_sha: "base", head_sha: "head") }
subject { described_class.send_git_patch(repository, diff_refs) }
......
require 'spec_helper'
describe Environment, models: true do
subject(:environment) { create(:environment) }
let(:project) { create(:empty_project) }
subject(:environment) { create(:environment, project: project) }
it { is_expected.to belong_to(:project) }
it { is_expected.to have_many(:deployments) }
......@@ -31,6 +32,8 @@ describe Environment, models: true do
end
describe '#includes_commit?' do
let(:project) { create(:project) }
context 'without a last deployment' do
it "returns false" do
expect(environment.includes_commit?('HEAD')).to be false
......@@ -38,9 +41,6 @@ describe Environment, models: true do
end
context 'with a last deployment' do
let(:project) { create(:project) }
let(:environment) { create(:environment, project: project) }
let!(:deployment) do
create(:deployment, environment: environment, sha: project.commit('master').id)
end
......@@ -65,7 +65,6 @@ describe Environment, models: true do
describe '#first_deployment_for' do
let(:project) { create(:project) }
let!(:environment) { create(:environment, project: project) }
let!(:deployment) { create(:deployment, environment: environment, ref: commit.parent.id) }
let!(:deployment1) { create(:deployment, environment: environment, ref: commit.id) }
let(:head_commit) { project.commit }
......@@ -196,6 +195,57 @@ describe Environment, models: true do
end
end
describe '#has_terminals?' do
subject { environment.has_terminals? }
context 'when the enviroment is available' do
context 'with a deployment service' do
let(:project) { create(:kubernetes_project) }
context 'and a deployment' do
let!(:deployment) { create(:deployment, environment: environment) }
it { is_expected.to be_truthy }
end
context 'but no deployments' do
it { is_expected.to be_falsy }
end
end
context 'without a deployment service' do
it { is_expected.to be_falsy }
end
end
context 'when the environment is unavailable' do
let(:project) { create(:kubernetes_project) }
before { environment.stop }
it { is_expected.to be_falsy }
end
end
describe '#terminals' do
let(:project) { create(:kubernetes_project) }
subject { environment.terminals }
context 'when the environment has terminals' do
before { allow(environment).to receive(:has_terminals?).and_return(true) }
it 'returns the terminals from the deployment service' do
expect(project.deployment_service).
to receive(:terminals).with(environment).
and_return(:fake_terminals)
is_expected.to eq(:fake_terminals)
end
end
context 'when the environment does not have terminals' do
before { allow(environment).to receive(:has_terminals?).and_return(false) }
it { is_expected.to eq(nil) }
end
end
describe '#slug' do
it "is automatically generated" do
expect(environment.slug).not_to be_nil
......
require 'spec_helper'
describe KubernetesService, models: true do
let(:project) { create(:empty_project) }
describe KubernetesService, models: true, caching: true do
include KubernetesHelpers
include ReactiveCachingHelpers
let(:project) { create(:kubernetes_project) }
let(:service) { project.kubernetes_service }
# We use Kubeclient to interactive with the Kubernetes API. It will
# GET /api/v1 for a list of resources the API supports. This must be stubbed
# in addition to any other HTTP requests we expect it to perform.
let(:discovery_url) { service.api_url + '/api/v1' }
let(:discovery_response) { { body: kube_discovery_body.to_json } }
let(:pods_url) { service.api_url + "/api/v1/namespaces/#{service.namespace}/pods" }
let(:pods_response) { { body: kube_pods_body(kube_pod).to_json } }
def stub_kubeclient_discover
WebMock.stub_request(:get, discovery_url).to_return(discovery_response)
end
def stub_kubeclient_pods
stub_kubeclient_discover
WebMock.stub_request(:get, pods_url).to_return(pods_response)
end
describe "Associations" do
it { is_expected.to belong_to :project }
......@@ -65,22 +87,15 @@ describe KubernetesService, models: true do
end
describe '#test' do
let(:project) { create(:kubernetes_project) }
let(:service) { project.kubernetes_service }
let(:discovery_url) { service.api_url + '/api/v1' }
# JSON response body from Kubernetes GET /api/v1 request
let(:discovery_response) { { "kind" => "APIResourceList", "groupVersion" => "v1", "resources" => [] }.to_json }
before do
stub_kubeclient_discover
end
context 'with path prefix in api_url' do
let(:discovery_url) { 'https://kubernetes.example.com/prefix/api/v1' }
before do
service.api_url = 'https://kubernetes.example.com/prefix/'
end
it 'tests with the prefix' do
WebMock.stub_request(:get, discovery_url).to_return(body: discovery_response)
service.api_url = 'https://kubernetes.example.com/prefix/'
expect(service.test[:success]).to be_truthy
expect(WebMock).to have_requested(:get, discovery_url).once
......@@ -88,17 +103,12 @@ describe KubernetesService, models: true do
end
context 'with custom CA certificate' do
let(:certificate) { "CA PEM DATA" }
before do
service.update_attributes!(ca_pem: certificate)
end
it 'is added to the certificate store' do
cert = double("certificate")
service.ca_pem = "CA PEM DATA"
expect(OpenSSL::X509::Certificate).to receive(:new).with(certificate).and_return(cert)
cert = double("certificate")
expect(OpenSSL::X509::Certificate).to receive(:new).with(service.ca_pem).and_return(cert)
expect_any_instance_of(OpenSSL::X509::Store).to receive(:add_cert).with(cert)
WebMock.stub_request(:get, discovery_url).to_return(body: discovery_response)
expect(service.test[:success]).to be_truthy
expect(WebMock).to have_requested(:get, discovery_url).once
......@@ -107,17 +117,15 @@ describe KubernetesService, models: true do
context 'success' do
it 'reads the discovery endpoint' do
WebMock.stub_request(:get, discovery_url).to_return(body: discovery_response)
expect(service.test[:success]).to be_truthy
expect(WebMock).to have_requested(:get, discovery_url).once
end
end
context 'failure' do
it 'fails to read the discovery endpoint' do
WebMock.stub_request(:get, discovery_url).to_return(status: 404)
let(:discovery_response) { { status: 404 } }
it 'fails to read the discovery endpoint' do
expect(service.test[:success]).to be_falsy
expect(WebMock).to have_requested(:get, discovery_url).once
end
......@@ -156,4 +164,55 @@ describe KubernetesService, models: true do
)
end
end
describe '#terminals' do
let(:environment) { build(:environment, project: project, name: "env", slug: "env-000000") }
subject { service.terminals(environment) }
context 'with invalid pods' do
it 'returns no terminals' do
stub_reactive_cache(service, pods: [ { "bad" => "pod" } ])
is_expected.to be_empty
end
end
context 'with valid pods' do
let(:pod) { kube_pod(app: environment.slug) }
let(:terminals) { kube_terminals(service, pod) }
it 'returns terminals' do
stub_reactive_cache(service, pods: [ pod, pod, kube_pod(app: "should-be-filtered-out") ])
is_expected.to eq(terminals + terminals)
end
end
end
describe '#calculate_reactive_cache' do
before { stub_kubeclient_pods }
subject { service.calculate_reactive_cache }
context 'when service is inactive' do
before { service.active = false }