Add a manual job to run QA against a Review App

1. Renames review to review-deploy.
2. Renames stop_review to review-stop.
3. Adds a build-qa-image job to the prepare stage and save the QA
  Docker image as artifact (example:
  https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/120967019).
4. Adds a manual review-qa job to the test stage to run QA tests
  against the Review App (example:
  https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/120967238).
5. Make the review-deploy job plays the review-qa job as soon as the
  Review App is deployed (example:
  https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/120988775).
6. Updates the Review Apps documentation accordingly.
Signed-off-by: 's avatarRémy Coutable <remy@rymai.me>
parent 99757d96
......@@ -439,6 +439,50 @@ setup-test-env:
- config/secrets.yml
- vendor/gitaly-ruby
# GitLab Review apps
.review-base: &review-base
<<: *dedicated-no-docs-no-db-pull-cache-job
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
stage: test
cache: {}
dependencies: []
environment: &review-environment
name: review/${CI_COMMIT_REF_NAME}
url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}
only:
refs:
- branches@gitlab-org/gitlab-ce
- branches@gitlab-org/gitlab-ee
kubernetes: active
except:
refs:
- master
- /(^docs[\/-].*|.*-docs$)/
before_script: []
.review-docker: &review-docker
<<: *review-base
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine
services:
- docker:stable-dind
tags:
- gitlab-org
- docker
variables: &review-docker-variables
GIT_DEPTH: "1"
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375
LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly"
QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}"
build-qa-image:
<<: *review-docker
stage: prepare
script:
- time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} ./qa/
- echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY}
- time docker push ${QA_IMAGE}
danger-review:
<<: *pull-cache
image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
......@@ -875,85 +919,97 @@ no_ee_check:
- //@gitlab-org/gitlab-ce
# GitLab Review apps
review:
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
stage: test
allow_failure: true
before_script:
- gem install gitlab --no-document
review-deploy:
<<: *review-base
variables:
GIT_DEPTH: "1"
HOST_SUFFIX: "$CI_ENVIRONMENT_SLUG"
DOMAIN: "-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN"
HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
GITLAB_HELM_CHART_REF: "master"
API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
environment:
<<: *review-environment
on_stop: review-stop
before_script:
- apk update && apk add jq
- gem install gitlab --no-document
script:
- export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
- export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
- export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
- source ./scripts/review_apps/review-apps.sh
- wait_for_job_to_be_done "gitlab:assets:compile"
- BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
- check_kube_domain
- download_gitlab_chart
- ensure_namespace
- install_tiller
- install_external_dns
- deploy
environment:
name: review/$CI_COMMIT_REF_NAME
url: https://gitlab-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN
on_stop: stop_review
only:
refs:
- branches@gitlab-org/gitlab-ce
- branches@gitlab-org/gitlab-ee
kubernetes: active
except:
refs:
- master
- /(^docs[\/-].*|.*-docs$)/
- time deploy
- add_license
.review-qa-base: &review-qa-base
<<: *review-docker
variables:
<<: *review-docker-variables
API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
GITLAB_USERNAME: "root"
GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
GITLAB_ADMIN_USERNAME: "root"
GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}"
EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}"
artifacts:
paths:
- ./qa/gitlab-qa-run-*
expire_in: 7 days
when: always
before_script:
- echo "${QA_IMAGE}"
- echo "${CI_ENVIRONMENT_URL}"
- apk update && apk add curl jq
- source ./scripts/review_apps/review-apps.sh
- gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}}
- wait_for_job_to_be_done "review-deploy"
review-qa-smoke:
<<: *review-qa-base
script:
- gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
review-qa-all:
<<: *review-qa-base
script:
- gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
when: manual
stop_review:
review-stop:
<<: *review-base
<<: *single-script-job
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
stage: test
allow_failure: true
cache: {}
dependencies: []
variables:
<<: *single-script-job-variables
SCRIPT_NAME: "review_apps/review-apps.sh"
when: manual
environment:
<<: *review-environment
action: stop
script:
- source $(basename "${SCRIPT_NAME}")
- delete
- cleanup
when: manual
environment:
name: review/$CI_COMMIT_REF_NAME
action: stop
only:
refs:
- branches@gitlab-org/gitlab-ce
- branches@gitlab-org/gitlab-ee
kubernetes: active
except:
- master
- /(^docs[\/-].*|.*-docs$)/
schedule:review_apps_cleanup:
<<: *dedicated-no-docs-pull-cache-job
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
schedule:review-cleanup:
<<: *review-base
stage: build
allow_failure: true
cache: {}
dependencies: []
before_script:
- gem install gitlab --no-document
variables:
GIT_DEPTH: "1"
script:
- ruby -rrubygems scripts/review_apps/automated_cleanup.rb
environment:
name: review/auto-cleanup
action: stop
only:
refs:
- schedules@gitlab-org/gitlab-ce
......@@ -962,3 +1018,7 @@ schedule:review_apps_cleanup:
except:
- tags
- /(^docs[\/-].*|.*-docs$)/
before_script:
- gem install gitlab --no-document
script:
- ruby -rrubygems scripts/review_apps/automated_cleanup.rb
......@@ -7,31 +7,36 @@ Review Apps are automatically deployed by each pipeline, both in
## How does it work?
1. On every [pipeline][gitlab-pipeline] during the `test` stage, the
[`review` job][review-job] is automatically started.
1. The `review` job [triggers a pipeline][cng-pipeline] in the
[`CNG-mirror`][cng-mirror] project.
- We use the `CNG-mirror` project so that the `CNG`, (**C**loud **N**ative
**G**itLab), project's registry is not overloaded with a lot of transient
Docker images.
1. The `CNG-mirror` pipeline creates the Docker images of each component (e.g.
`gitlab-rails-ee`, `gitlab-shell`, `gitaly` etc.) based on the commit from the
[GitLab pipeline][gitlab-pipeline] and store them in its
[registry][cng-mirror-registry].
1. Once all images are built, the Review App is deployed using
[the official GitLab Helm chart][helm-chart] to the
[`review-apps-ee` Kubernetes cluster on GCP][review-apps-ee]
- The actual scripts used to deploy the Review App can be found at
[`scripts/review_apps/review-apps.sh`][review-apps.sh]
- These scripts are basically
[our official Auto DevOps scripts][Auto-DevOps.gitlab-ci.yml] where the
default CNG images are overridden with the images built and stored in the
[`CNG-mirror` project's registry][cng-mirror-registry].
- Since we're using [the official GitLab Helm chart][helm-chart], this means
you get a dedicated environment for your branch that's very close to what it
would look in production.
1. Once the `review` job succeeds, you should be able to use your Review App
thanks to the direct link to it from the MR widget. The default username is
`root` and its password can be found in the 1Password secure note named
[`review-deploy`][review-deploy-job] job is automatically started.
1. The `review-deploy` job:
1. Waits for the `gitlab:assets:compile` job to finish since the
[`CNG-mirror`][cng-mirror] pipeline triggerred in the following step
depends on it.
1. [Triggers a pipeline][cng-pipeline] in the [`CNG-mirror`][cng-mirror]
project.
- We use the `CNG-mirror` project so that the `CNG`, (**C**loud
**N**ative **G**itLab), project's registry is not overloaded with a
lot of transient Docker images.
- The `CNG-mirror` pipeline creates the Docker images of each component
(e.g. `gitlab-rails-ee`, `gitlab-shell`, `gitaly` etc.) based on the
commit from the [GitLab pipeline][gitlab-pipeline] and store them in
its [registry][cng-mirror-registry].
1. Once all images are built by [`CNG-mirror`][cng-mirror], the Review App
is deployed using [the official GitLab Helm chart][helm-chart] to the
[`review-apps-ce`][review-apps-ce] / [`review-apps-ee`][review-apps-ee]
Kubernetes cluster on GCP.
- The actual scripts used to deploy the Review App can be found at
[`scripts/review_apps/review-apps.sh`][review-apps.sh].
- These scripts are basically
[our official Auto DevOps scripts][Auto-DevOps.gitlab-ci.yml] where the
default CNG images are overridden with the images built and stored in the
[`CNG-mirror` project's registry][cng-mirror-registry].
- Since we're using [the official GitLab Helm chart][helm-chart], this means
you get a dedicated environment for your branch that's very close to what
it would look in production.
1. Once the `review-deploy` job succeeds, you should be able to use your Review
App thanks to the direct link to it from the MR widget. The default username
is `root` and its password can be found in the 1Password secure note named
**gitlab-{ce,ee} Review App's root password** (note that there's currently
[a bug where the default password seems to be overridden][password-bug]).
......@@ -39,16 +44,23 @@ Review Apps are automatically deployed by each pipeline, both in
- The Kubernetes cluster is connected to the `gitlab-{ce,ee}` projects using
[GitLab's Kubernetes integration][gitlab-k8s-integration]. This basically
allows to have a link to the Review App directly from the merge request widget.
- The manual `stop_review` in the `test` stage can be used to stop a Review App
manually, and is also started by GitLab once a branch is deleted.
- Review Apps are cleaned up regularly using a pipeline schedule that runs
the [`scripts/review_apps/automated_cleanup.rb`][automated_cleanup.rb] script.
allows to have a link to the Review App directly from the merge request
widget.
- If the Review App deployment fails, you can simply retry it (there's no need
to run the `stop_review` job first).
- If you're unable to log in using the `root` username and password, you may
encounter [this bug][password-bug]. Stop the Review App via the `stop_review`
manual job and then retry the `review` job to redeploy the Review App.
to run the [`review-stop`][gitlab-ci-yml] job first).
- The manual [`review-stop`][gitlab-ci-yml] in the `test` stage can be used to
stop a Review App manually, and is also started by GitLab once a branch is
deleted.
- Review Apps are cleaned up regularly using a pipeline schedule that runs
the [`schedule:review-cleanup`][gitlab-ci-yml] job.
## QA runs
On every [pipeline][gitlab-pipeline] during the `test` stage, the
`review-qa-smoke` job is automatically started: it runs the smoke QA suite.
You can also manually start the `review-qa-all`: it runs the full QA suite.
Note that both jobs first wait for the `review-deploy` job to be finished.
## Frequently Asked Questions
......@@ -74,15 +86,17 @@ find a way to limit it to only us.**
> This isn't enabled for forks.
[gitlab-pipeline]: https://gitlab.com/gitlab-org/gitlab-ce/pipelines/35850709
[review-job]: https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/118076368
[review-deploy-job]: https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/118076368
[cng-mirror]: https://gitlab.com/gitlab-org/build/CNG-mirror
[cng-pipeline]: https://gitlab.com/gitlab-org/build/CNG-mirror/pipelines/35883435
[cng-mirror-registry]: https://gitlab.com/gitlab-org/build/CNG-mirror/container_registry
[helm-chart]: https://gitlab.com/charts/gitlab/
[review-apps-ce]: https://console.cloud.google.com/kubernetes/clusters/details/us-central1-a/review-apps-ce?project=gitlab-review-apps
[review-apps-ee]: https://console.cloud.google.com/kubernetes/clusters/details/us-central1-b/review-apps-ee?project=gitlab-review-apps
[review-apps.sh]: https://gitlab.com/gitlab-org/gitlab-ee/blob/master/scripts/review_apps/review-apps.sh
[automated_cleanup.rb]: https://gitlab.com/gitlab-org/gitlab-ee/blob/master/scripts/review_apps/automated_cleanup.rb
[Auto-DevOps.gitlab-ci.yml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
[gitlab-ci-yml]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.gitlab-ci.yml
[gitlab-k8s-integration]: https://docs.gitlab.com/ee/user/project/clusters/index.html
[password-bug]: https://gitlab.com/gitlab-org/gitlab-ce/issues/53621
......
[[ "$TRACE" ]] && set -x
export TILLER_NAMESPACE="$KUBE_NAMESPACE"
function echoerr() { printf "\033[0;31m%s\n\033[0m" "$*" >&2; }
function check_kube_domain() {
if [ -z ${REVIEW_APPS_DOMAIN+x} ]; then
echo "In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set"
......@@ -88,19 +90,16 @@ function deploy() {
replicas="1"
service_enabled="false"
postgres_enabled="$POSTGRES_ENABLED"
gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ce"
gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ce"
gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ce"
gitlab_gitaly_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly"
gitlab_shell_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell"
gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ce"
if [[ "$CI_PROJECT_NAME" == "gitlab-ee" ]]; then
gitlab_migrations_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-rails-ee"
gitlab_sidekiq_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-sidekiq-ee"
gitlab_unicorn_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-unicorn-ee"
gitlab_workhorse_image_repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-workhorse-ee"
fi
IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror"
IMAGE_VERSION="${CI_PROJECT_NAME#gitlab-}"
gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-${IMAGE_VERSION}"
gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-${IMAGE_VERSION}"
gitlab_unicorn_image_repository="${IMAGE_REPOSITORY}/gitlab-unicorn-${IMAGE_VERSION}"
gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-${IMAGE_VERSION}"
gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly"
gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${IMAGE_VERSION}"
# canary uses stable db
[[ "$track" == "canary" ]] && postgres_enabled="false"
......@@ -155,6 +154,8 @@ HELM_CMD=$(cat << EOF
--set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_NAME" \
--set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \
--set gitlab.unicorn.image.tag="$CI_COMMIT_REF_NAME" \
--set gitlab.task-runner.image.repository="$gitlab_task_runner_image_repository" \
--set gitlab.task-runner.image.tag="$CI_COMMIT_REF_NAME" \
--set gitlab.gitaly.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitaly" \
--set gitlab.gitaly.image.tag="v$GITALY_VERSION" \
--set gitlab.gitlab-shell.image.repository="registry.gitlab.com/gitlab-org/build/cng-mirror/gitlab-shell" \
......@@ -229,3 +230,121 @@ function install_external_dns() {
--set rbac.create="true"
fi
}
function get_pod() {
local app_name="${1}"
local status="${2-Running}"
get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name"
echoerr "Running '${get_pod_cmd}'"
while true; do
local pod_name="$(eval $get_pod_cmd)"
[[ "${pod_name}" == "" ]] || break
echoerr "Waiting till '${app_name}' pod is ready";
sleep 5;
done
echoerr "The pod name is '${pod_name}'."
echo "${pod_name}"
}
function add_license() {
if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi
task_runner_pod=$(get_pod "task-runner");
if [ -z "${task_runner_pod}" ]; then echo "Task runner pod not found" && return; fi
echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab
kubectl -n "$KUBE_NAMESPACE" cp /tmp/license.gitlab ${task_runner_pod}:/tmp/license.gitlab
rm /tmp/license.gitlab
kubectl -n "$KUBE_NAMESPACE" exec -it ${task_runner_pod} -- /srv/gitlab/bin/rails runner -e production \
'
content = File.read("/tmp/license.gitlab").strip;
FileUtils.rm_f("/tmp/license.gitlab");
unless License.where(data:content).empty?
puts "License already exists";
Kernel.exit 0;
end
unless License.new(data: content).save
puts "Could not add license";
Kernel.exit 0;
end
puts "License added";
'
}
function get_job_id() {
local job_name="${1}"
local query_string="${2:+&${2}}"
local max_page=3
local page=1
while true; do
local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}"
echoerr "GET ${url}"
local job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".[] | select(.name == \"${job_name}\") | .id")
[[ "${job_id}" == "" && "${page}" -lt "$max_page" ]] || break
((page++))
done
if [[ "${job_id}" == "" ]]; then
echoerr "The '${job_name}' job ID couldn't be retrieved!"
else
echoerr "The '${job_name}' job ID is ${job_id}"
echo "${job_id}"
fi
}
function play_job() {
local job_name="${1}"
local job_id=$(get_job_id "${job_name}" "scope=manual");
if [ -z "${job_id}" ]; then return; fi
local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play"
echoerr "POST ${url}"
local job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".web_url")
echo "Manual job '${job_name}' started at: ${job_url}"
}
function wait_for_job_to_be_done() {
local job_name="${1}"
local query_string="${2}"
local job_id=$(get_job_id "${job_name}" "${query_string}");
if [ -z "${job_id}" ]; then return; fi
echoerr "Waiting for the '${job_name}' job to finish..."
local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}"
echo "GET ${url}"
# In case the job hasn't finished yet. Keep trying until the job times out.
local interval=30
local elapsed=0
while true; do
local job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${API_TOKEN}" "${url}" | jq ".status" | sed -e s/\"//g)
[[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break
printf "."
((elapsed+=$interval))
sleep ${interval}
done
echoerr "Waited '${job_name}' for ${elapsed} seconds."
if [[ "${job_status}" == "failed" ]]; then
echo "The '${job_name}' failed."
elif [[ "${job_status}" == "manual" ]]; then
echo "The '${job_name}' is manual."
else
echo "The '${job_name}' passed."
fi
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment