Commit c239bfcb authored by Dylan Griffith's avatar Dylan Griffith Committed by James Lopez

Add more info logging to cluster apps

Log events so that it's easy to see
when different requests are starting.
parent b5af30bb
......@@ -16,6 +16,7 @@ module Clusters
error_code: error.respond_to?(:error_code) ? error.error_code : nil,
service: self.class.name,
app_id: app.id,
app_name: app.name,
project_ids: app.cluster.project_ids,
group_ids: app.cluster.group_ids
}
......@@ -30,6 +31,19 @@ module Clusters
Gitlab::Sentry.track_acceptable_exception(error, extra: meta)
end
def log_event(event)
meta = {
service: self.class.name,
app_id: app.id,
app_name: app.name,
project_ids: app.cluster.project_ids,
group_ids: app.cluster.group_ids,
event: event
}
logger.info(meta)
end
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
......
......@@ -7,8 +7,10 @@ module Clusters
return unless app.scheduled?
app.make_installing!
log_event(:begin_install)
helm_api.install(install_command)
log_event(:schedule_wait_for_installation)
ClusterWaitForAppInstallationWorker.perform_in(
ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => e
......
......@@ -8,8 +8,10 @@ module Clusters
app.make_updating!
log_event(:begin_patch)
helm_api.update(update_command)
log_event(:schedule_wait_for_patch)
ClusterWaitForAppInstallationWorker.perform_in(
ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => e
......
......@@ -9,10 +9,12 @@ module Clusters
begin
app.make_updating!
log_event(:begin_upgrade)
# install_command works with upgrades too
# as it basically does `helm upgrade --install`
helm_api.update(install_command)
log_event(:schedule_wait_for_upgrade)
ClusterWaitForAppInstallationWorker.perform_in(
ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => e
......
......@@ -573,7 +573,7 @@ However, sometimes GitLab can not create them. In such instances, your job will
This job failed because the necessary resources were not successfully created.
```
To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#sidekiqlog).
To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#kuberneteslog).
Common reasons for failure include:
......
......@@ -19,11 +19,40 @@ module Gitlab
def create!
resource = ::Kubeclient::Resource.new(metadata: { name: name })
log_event(:begin_create)
@client.create_namespace(resource)
end
def ensure_exists!
exists? || create!
rescue ::Kubeclient::HttpError => error
log_create_failed(error)
raise
end
private
def log_create_failed(error)
logger.error({
exception: error.class.name,
status_code: error.error_code,
namespace: name,
class_name: self.class.name,
event: :failed_to_create_namespace,
message: error.message
})
end
def log_event(event)
logger.info(
namespace: name,
class_name: self.class.name,
event: event
)
end
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
end
end
......
......@@ -62,5 +62,32 @@ describe Gitlab::Kubernetes::Namespace do
subject.ensure_exists!
end
context 'when client errors' do
let(:exception) { Kubeclient::HttpError.new(500, 'system failure', nil) }
before do
allow(client).to receive(:get_namespace).with(name).once.and_raise(exception)
end
it 'raises the exception' do
expect { subject.ensure_exists! }.to raise_error(exception)
end
it 'logs the error' do
expect(subject.send(:logger)).to receive(:error).with(
hash_including(
exception: 'Kubeclient::HttpError',
status_code: 500,
namespace: 'a_namespace',
class_name: 'Gitlab::Kubernetes::Namespace',
event: :failed_to_create_namespace,
message: 'system failure'
)
)
expect { subject.ensure_exists! }.to raise_error(exception)
end
end
end
end
......@@ -20,7 +20,7 @@ shared_examples 'logs kubernetes errors' do
end
it 'logs into kubernetes.log and Sentry' do
expect(service.send(:logger)).to receive(:error).with(logger_hash)
expect(service.send(:logger)).to receive(:error).with(hash_including(logger_hash))
expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
error,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment