Commit 112741ef authored by James Lopez's avatar James Lopez

Merge branch '59276-more-k8s-logging' into 'master'

Resolve "Improvements of kubernetes.log"

Closes #59276

See merge request gitlab-org/gitlab-ce!26895
parents b5af30bb c239bfcb
......@@ -16,6 +16,7 @@ module Clusters
error_code: error.respond_to?(:error_code) ? error.error_code : nil,
service: self.class.name,
app_id: app.id,
app_name: app.name,
project_ids: app.cluster.project_ids,
group_ids: app.cluster.group_ids
}
......@@ -30,6 +31,19 @@ module Clusters
Gitlab::Sentry.track_acceptable_exception(error, extra: meta)
end
def log_event(event)
meta = {
service: self.class.name,
app_id: app.id,
app_name: app.name,
project_ids: app.cluster.project_ids,
group_ids: app.cluster.group_ids,
event: event
}
logger.info(meta)
end
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
......
......@@ -7,8 +7,10 @@ module Clusters
return unless app.scheduled?
app.make_installing!
log_event(:begin_install)
helm_api.install(install_command)
log_event(:schedule_wait_for_installation)
ClusterWaitForAppInstallationWorker.perform_in(
ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => e
......
......@@ -8,8 +8,10 @@ module Clusters
app.make_updating!
log_event(:begin_patch)
helm_api.update(update_command)
log_event(:schedule_wait_for_patch)
ClusterWaitForAppInstallationWorker.perform_in(
ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => e
......
......@@ -9,10 +9,12 @@ module Clusters
begin
app.make_updating!
log_event(:begin_upgrade)
# install_command works with upgrades too
# as it basically does `helm upgrade --install`
helm_api.update(install_command)
log_event(:schedule_wait_for_upgrade)
ClusterWaitForAppInstallationWorker.perform_in(
ClusterWaitForAppInstallationWorker::INTERVAL, app.name, app.id)
rescue Kubeclient::HttpError => e
......
......@@ -573,7 +573,7 @@ However, sometimes GitLab can not create them. In such instances, your job will
This job failed because the necessary resources were not successfully created.
```
To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#sidekiqlog).
To find the cause of this error when creating a namespace and service account, check the [logs](../../../administration/logs.md#kuberneteslog).
Common reasons for failure include:
......
......@@ -19,11 +19,40 @@ module Gitlab
def create!
resource = ::Kubeclient::Resource.new(metadata: { name: name })
log_event(:begin_create)
@client.create_namespace(resource)
end
def ensure_exists!
exists? || create!
rescue ::Kubeclient::HttpError => error
log_create_failed(error)
raise
end
private
def log_create_failed(error)
logger.error({
exception: error.class.name,
status_code: error.error_code,
namespace: name,
class_name: self.class.name,
event: :failed_to_create_namespace,
message: error.message
})
end
def log_event(event)
logger.info(
namespace: name,
class_name: self.class.name,
event: event
)
end
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
end
end
......
......@@ -62,5 +62,32 @@ describe Gitlab::Kubernetes::Namespace do
subject.ensure_exists!
end
context 'when client errors' do
let(:exception) { Kubeclient::HttpError.new(500, 'system failure', nil) }
before do
allow(client).to receive(:get_namespace).with(name).once.and_raise(exception)
end
it 'raises the exception' do
expect { subject.ensure_exists! }.to raise_error(exception)
end
it 'logs the error' do
expect(subject.send(:logger)).to receive(:error).with(
hash_including(
exception: 'Kubeclient::HttpError',
status_code: 500,
namespace: 'a_namespace',
class_name: 'Gitlab::Kubernetes::Namespace',
event: :failed_to_create_namespace,
message: 'system failure'
)
)
expect { subject.ensure_exists! }.to raise_error(exception)
end
end
end
end
......@@ -20,7 +20,7 @@ shared_examples 'logs kubernetes errors' do
end
it 'logs into kubernetes.log and Sentry' do
expect(service.send(:logger)).to receive(:error).with(logger_hash)
expect(service.send(:logger)).to receive(:error).with(hash_including(logger_hash))
expect(Gitlab::Sentry).to receive(:track_acceptable_exception).with(
error,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment