.gitlab-ci.yml 30.5 KB
Newer Older
1
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.7-golang-1.9-git-2.17-chrome-65.0-node-8.x-yarn-1.2-postgresql-9.6"
2

3 4 5 6 7
.dedicated-runner: &dedicated-runner
  retry: 1
  tags:
    - gitlab-org

8
.default-cache: &default-cache
9
  key: "ruby-2.3.7-debian-stretch-with-yarn"
10
  paths:
11 12
    - vendor/ruby
    - .yarn-cache/
13
    - vendor/gitaly-ruby
14 15 16 17 18 19 20 21 22 23

.push-cache: &push-cache
  cache:
    <<: *default-cache
    policy: push

.pull-cache: &pull-cache
  cache:
    <<: *default-cache
    policy: pull
24

25 26
variables:
  MYSQL_ALLOW_EMPTY_PASSWORD: "1"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
27
  RAILS_ENV: "test"
28
  NODE_ENV: "test"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
29
  SIMPLECOV: "true"
30
  GIT_DEPTH: "20"
31
  GIT_SUBMODULE_STRATEGY: "none"
32
  GET_SOURCES_ATTEMPTS: "3"
33
  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
34
  FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json
35
  BUILD_ASSETS_IMAGE: "false"
36

Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
37
before_script:
Kamil Trzcinski's avatar
Kamil Trzcinski committed
38
  - bundle --version
39
  - date
40
  - source scripts/utils.sh
41
  - date
42
  - source scripts/prepare_build.sh
43 44 45 46
  - date

after_script:
  - date
47

Kamil Trzcinski's avatar
Kamil Trzcinski committed
48
stages:
49 50
  - build
  - prepare
51
  - merge
52 53 54
  - test
  - post-test
  - pages
55
  - post-cleanup
56

57
# Predefined scopes
58
.tests-metadata-state: &tests-metadata-state
59
  <<: *dedicated-runner
60
  variables:
61
    TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
62 63
  before_script:
    - source scripts/utils.sh
64 65 66
  artifacts:
    expire_in: 31d
    paths:
67
      - knapsack/
68
      - rspec_flaky/
69

70 71
.use-pg: &use-pg
  services:
72
    - postgres:9.6
73 74 75
    - redis:alpine

.use-mysql: &use-mysql
76
  services:
77
    - mysql:5.7
78
    - redis:alpine
79

80 81
# Skip all jobs except the ones that begin with 'docs/'.
# Used for commits including ONLY documentation changes.
82
# https://docs.gitlab.com/ce/development/documentation/#testing
83 84
.except-docs: &except-docs
  except:
85
    - /(^docs[\/-].*|.*-docs$)/
86

87 88 89 90
.except-qa: &except-qa
  except:
    - /(^qa[\/-].*|.*-qa$)/

91 92 93 94 95
.except-docs-and-qa: &except-docs-and-qa
  except:
    - /(^docs[\/-].*|.*-docs$)/
    - /(^qa[\/-].*|.*-qa$)/

96 97 98
# Jobs that only need to pull cache
.dedicated-no-docs-pull-cache-job: &dedicated-no-docs-pull-cache-job
  <<: *dedicated-runner
99
  <<: *except-docs
100 101 102 103 104 105 106 107 108 109 110
  <<: *pull-cache
  dependencies:
    - setup-test-env
  stage: test

# Jobs that do not need a DB
.dedicated-no-docs-no-db-pull-cache-job: &dedicated-no-docs-no-db-pull-cache-job
  <<: *dedicated-no-docs-pull-cache-job
  variables:
    SETUP_DB: "false"

111 112 113 114
.dedicated-no-docs-and-no-qa-pull-cache-job: &dedicated-no-docs-and-no-qa-pull-cache-job
  <<: *dedicated-no-docs-pull-cache-job
  <<: *except-docs-and-qa

115
.single-script-job: &single-script-job
Stan Hu's avatar
Stan Hu committed
116
  image: ruby:2.5-alpine
117
  stage: test
118 119 120 121 122
  cache: {}
  dependencies: []
  variables: &single-script-job-variables
    GIT_STRATEGY: none
  before_script:
123 124
    # We don't clone the repo by using GIT_STRATEGY: none and only download the
    # single script we need here so it's much faster than cloning.
125 126 127
    - export SCRIPT_NAME="${SCRIPT_NAME:-$CI_JOB_NAME}"
    - apk add --update openssl
    - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/$SCRIPT_NAME
Ian Baum's avatar
Ian Baum committed
128
    - chmod 755 $(basename $SCRIPT_NAME)
129

130 131 132 133 134
.rake-exec: &rake-exec
  <<: *dedicated-no-docs-no-db-pull-cache-job
  script:
    - bundle exec rake $CI_JOB_NAME

135
.rspec-metadata: &rspec-metadata
136
  <<: *dedicated-runner
137
  <<: *except-docs-and-qa
138
  <<: *pull-cache
139
  stage: test
140
  script:
141
    - JOB_NAME=( $CI_JOB_NAME )
142 143
    - TEST_TOOL=${JOB_NAME[0]}
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
144
    - export KNAPSACK_GENERATE_REPORT=true
145
    - export SUITE_FLAKY_RSPEC_REPORT_PATH=${FLAKY_RSPEC_SUITE_REPORT_PATH}
146 147
    - export FLAKY_RSPEC_REPORT_PATH=rspec_flaky/all_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
    - export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/new_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
148
    - export FLAKY_RSPEC_GENERATE_REPORT=true
149
    - export CACHE_CLASSES=true
150
    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
151
    - '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}'
152
    - '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}'
Jacob Vosmaer's avatar
Jacob Vosmaer committed
153
    - scripts/gitaly-test-spawn
154
    - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml"
155 156
  artifacts:
    expire_in: 31d
157
    when: always
158
    paths:
159 160
      - coverage/
      - knapsack/
161
      - rspec_flaky/
162
      - tmp/capybara/
163 164
    reports:
      junit: junit_rspec.xml
165

166 167
.rspec-metadata-pg: &rspec-metadata-pg
  <<: *rspec-metadata
168 169
  <<: *use-pg

170 171
.rspec-metadata-mysql: &rspec-metadata-mysql
  <<: *rspec-metadata
172 173
  <<: *use-mysql

174 175 176 177 178 179 180
.only-canonical-masters: &only-canonical-masters
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee

181
.gitlab-setup: &gitlab-setup
182
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
183 184
  <<: *use-pg
  variables:
185
    SETUP_DB: "false"
186
  script:
187 188 189 190 191 192 193 194 195 196 197
    # Manually clone gitlab-test and only seed this project in
    # db/fixtures/development/04_project.rb thanks to SIZE=1 below
    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
       /home/git/repositories/gitlab-org/gitlab-test.git
    - scripts/gitaly-test-spawn
    - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
  artifacts:
    when: on_failure
    expire_in: 1d
    paths:
      - log/development.log
198

199 200
# DB migration, rollback, and seed jobs
.db-migrate-reset: &db-migrate-reset
201
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
202 203 204 205
  script:
    - bundle exec rake db:migrate:reset

.migration-paths: &migration-paths
206
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
207
  variables:
208
    SETUP_DB: "false"
209 210 211
  script:
    - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0
    - git checkout -f FETCH_HEAD
Stan Hu's avatar
Stan Hu committed
212 213
    - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile
    - bundle update google-protobuf grpc
214 215 216 217 218
    - bundle install $BUNDLE_INSTALL_FLAGS
    - date
    - cp config/gitlab.yml.example config/gitlab.yml
    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
    - date
219
    - git checkout -f $CI_COMMIT_SHA
220 221 222 223 224 225 226 227 228
    - bundle install $BUNDLE_INSTALL_FLAGS
    - date
    - . scripts/prepare_build.sh
    - date
    - bundle exec rake db:migrate

##
# Trigger a package build in omnibus-gitlab repository
#
229
package-and-qa:
230 231 232 233 234
  image: ruby:2.5-alpine
  stage: test
  before_script: []
  dependencies: []
  cache: {}
235
  variables:
236
    GIT_DEPTH: "1"
237
    API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
238
  retry: 0
239
  script:
240
    - apk add --update openssl curl jq
241 242
    - gem install gitlab --no-document
    - source ./scripts/review_apps/review-apps.sh
243
    - wait_for_job_to_be_done "gitlab:assets:compile"
244
    - ./scripts/trigger-build omnibus
245
  when: manual
246
  only:
247 248
    - /.+/@gitlab-org/gitlab-ce
    - /.+/@gitlab-org/gitlab-ee
249

250 251 252 253 254 255 256
# Review docs base
.review-docs: &review-docs
  <<: *dedicated-runner
  <<: *single-script-job
  variables:
    <<: *single-script-job-variables
    SCRIPT_NAME: trigger-build-docs
257
  environment:
258
    name: review-docs/$CI_COMMIT_REF_SLUG
259
    # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables
260
    # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693
261
    url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX
262
    on_stop: review-docs-cleanup
263 264 265 266 267 268

# Trigger a manual docs build in gitlab-docs only on non docs-only branches.
# Useful to preview the docs changes live.
review-docs-deploy-manual:
  <<: *review-docs
  stage: build
269
  script:
270
    - gem install gitlab --no-document
271
    - ./$SCRIPT_NAME deploy
272 273
  when: manual
  only:
274 275
    - branches@gitlab-org/gitlab-ce
    - branches@gitlab-org/gitlab-ee
276 277 278 279 280 281 282 283
  <<: *except-docs-and-qa

# Always trigger a docs build in gitlab-docs only on docs-only branches.
# Useful to preview the docs changes live.
review-docs-deploy:
  <<: *review-docs
  stage: post-test
  script:
284
    - gem install gitlab --no-document
285 286
    - ./$SCRIPT_NAME deploy
  only:
287 288
    - /(^docs[\/-].*|.*-docs$)/@gitlab-org/gitlab-ce
    - /(^docs[\/-].*|.*-docs$)/@gitlab-org/gitlab-ee
289
  <<: *except-qa
290 291 292 293 294 295

# Cleanup remote environment of gitlab-docs
review-docs-cleanup:
  <<: *review-docs
  stage: post-cleanup
  environment:
296
    name: review-docs/$CI_COMMIT_REF_SLUG
297 298
    action: stop
  script:
299
    - gem install gitlab --no-document
300
    - ./$SCRIPT_NAME cleanup
301 302 303 304
  when: manual
  only:
    - branches@gitlab-org/gitlab-ce
    - branches@gitlab-org/gitlab-ee
305

306 307 308 309
##
# Trigger a docker image build in CNG (Cloud Native GitLab) repository
#
cloud-native-image:
Stan Hu's avatar
Stan Hu committed
310
  image: ruby:2.5-alpine
311
  before_script: []
312
  dependencies: []
313
  stage: post-test
314
  allow_failure: true
315 316
  variables:
    GIT_DEPTH: "1"
317
  cache: {}
318
  when: manual
319
  script:
320 321
    - gem install gitlab --no-document
    - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng
322 323 324 325
  only:
    - tags@gitlab-org/gitlab-ce
    - tags@gitlab-org/gitlab-ee

326 327 328
# Retrieve knapsack and rspec_flaky reports
retrieve-tests-metadata:
  <<: *tests-metadata-state
329
  <<: *except-docs-and-qa
330
  stage: prepare
331
  cache:
332
    key: tests_metadata
333
    policy: pull
334
  script:
335
    - mkdir -p knapsack/${CI_PROJECT_NAME}/
336
    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
337
    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
338
    - mkdir -p rspec_flaky/
339 340
    - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH
    - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
341

342 343
update-tests-metadata:
  <<: *tests-metadata-state
344
  <<: *only-canonical-masters
345
  stage: post-test
346
  cache:
347
    key: tests_metadata
348 349
    paths:
      - knapsack/
350
      - rspec_flaky/
351
    policy: push
352
  script:
353
    - retry gem install fog-aws mime-types activesupport --no-document
354
    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
355
    - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
356
    - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
blackst0ne's avatar
blackst0ne committed
357
    - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH'
358
    - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
359
    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
360
    - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
361 362 363

flaky-examples-check:
  <<: *dedicated-runner
Stan Hu's avatar
Stan Hu committed
364
  image: ruby:2.5-alpine
365 366 367 368 369
  services: []
  before_script: []
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
370
    NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json
371
  stage: post-test
372
  allow_failure: true
373
  retry: 0
374 375 376 377
  only:
    - branches
  except:
    - master
378
    - /(^docs[\/-].*|.*-docs$)/
379
    - /(^qa[\/-].*|.*-qa$)/
380 381 382 383 384 385
  artifacts:
    expire_in: 30d
    paths:
      - rspec_flaky/
  script:
    - '[[ -f $NEW_FLAKY_SPECS_REPORT ]] || echo "{}" > ${NEW_FLAKY_SPECS_REPORT}'
386
    - scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json
387
    - scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT
388

389 390
.assets-compile-cache: &assets-compile-cache
  cache:
391
    key: "assets-compile:vendor_ruby:.yarn-cache:tmp_cache_assets_sprockets:v5"
392 393 394
    paths:
      - vendor/ruby/
      - .yarn-cache/
395
      - tmp/cache/assets/sprockets
396

397
compile-assets:
398
  <<: *dedicated-runner
399
  <<: *except-docs
400
  <<: *use-pg
401 402
  stage: prepare
  script:
403
    - node --version
404
    - yarn install --frozen-lockfile --cache-folder .yarn-cache
405
    - free -m
Mike Greiling's avatar
Mike Greiling committed
406
    - bundle exec rake gitlab:assets:compile
407
    - scripts/clean-old-cached-assets
408 409 410
  variables:
    # we override the max_old_space_size to prevent OOM errors
    NODE_OPTIONS: --max_old_space_size=3584
411 412 413
  artifacts:
    expire_in: 7d
    paths:
414
      - node_modules
415
      - public/assets
416
  <<: *assets-compile-cache
417 418 419 420 421 422 423 424 425 426 427 428 429 430

setup-test-env:
  <<: *dedicated-runner
  <<: *except-docs
  <<: *use-pg
  stage: prepare
  cache:
    <<: *default-cache
  script:
    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
    - scripts/gitaly-test-build # Do not use 'bundle exec' here
  artifacts:
    expire_in: 7d
    paths:
431
      - tmp/tests
432
      - config/secrets.yml
433
      - vendor/gitaly-ruby
434

435
# GitLab Review apps
436
.review-only: &review-only
437 438
  only:
    refs:
439 440
      - branches@gitlab-org/gitlab-ce
      - branches@gitlab-org/gitlab-ee
441 442 443 444 445
    kubernetes: active
  except:
    refs:
      - master
      - /(^docs[\/-].*|.*-docs$)/
446

447 448 449 450 451 452 453 454 455 456 457
.review-schedules-only: &review-schedules-only
  only:
    refs:
      - schedules@gitlab-org/gitlab-ce
      - schedules@gitlab-org/gitlab-ee
    kubernetes: active
  except:
    refs:
      - tags
      - /(^docs[\/-].*|.*-docs$)/

458 459 460 461 462 463 464 465 466 467
.review-base: &review-base
  <<: *dedicated-no-docs-no-db-pull-cache-job
  <<: *review-only
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
  stage: test
  cache: {}
  dependencies: []
  environment: &review-environment
    name: review/${CI_COMMIT_REF_NAME}
    url: https://gitlab-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}
468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
  before_script: []

.review-docker: &review-docker
  <<: *review-base
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine
  services:
    - docker:stable-dind
  tags:
    - gitlab-org
    - docker
  variables: &review-docker-variables
    GIT_DEPTH: "1"
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375
    LATEST_QA_IMAGE: "gitlab/${CI_PROJECT_NAME}-qa:nightly"
    QA_IMAGE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab/${CI_PROJECT_NAME}-qa:${CI_COMMIT_REF_SLUG}"

build-qa-image:
  <<: *review-docker
  stage: prepare
  script:
    - time docker build --cache-from ${LATEST_QA_IMAGE} --tag ${QA_IMAGE} ./qa/
    - echo "${CI_JOB_TOKEN}" | docker login --username gitlab-ci-token --password-stdin ${CI_REGISTRY}
    - time docker push ${QA_IMAGE}

493
danger-review:
494
  <<: *pull-cache
495
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
496
  stage: test
497 498
  dependencies: []
  before_script: []
499
  only:
500 501
    variables:
      - $DANGER_GITLAB_API_TOKEN
502 503 504 505 506
  except:
    refs:
      - master
    variables:
      - $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/
507
      - $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/
508 509
  script:
    - git version
510
    - node --version
511
    - yarn install --frozen-lockfile --cache-folder .yarn-cache
512 513
    - danger --fail-on-errors=true

514 515
rspec-pg:
  <<: *rspec-metadata-pg
516
  parallel: 50
517 518 519

rspec-mysql:
  <<: *rspec-metadata-mysql
520
  parallel: 50
521

522
.rspec-quarantine: &rspec-quarantine
523
  retry: 0
524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
  script:
    - export CACHE_CLASSES=true
    - scripts/gitaly-test-spawn
    - bin/rspec --color --format documentation --tag quarantine spec/

rspec-pg-quarantine:
  <<: *rspec-metadata-pg
  <<: *rspec-quarantine
  allow_failure: true

rspec-mysql-quarantine:
  <<: *rspec-metadata-mysql
  <<: *rspec-quarantine
  allow_failure: true

539
static-analysis:
540 541 542 543
  <<: *dedicated-no-docs-no-db-pull-cache-job
  dependencies:
    - compile-assets
    - setup-test-env
544
  script:
545
    - scripts/static-analysis
546
  cache:
547
    key: "ruby-2.3.7-debian-stretch-with-yarn-and-rubocop"
548 549 550 551
    paths:
      - vendor/ruby
      - .yarn-cache/
      - tmp/rubocop_cache
552

553 554 555 556
# Documentation checks:
# - Check validity of relative links
# - Make sure cURL examples in API docs use the full switches
docs lint:
557
  <<: *dedicated-runner
558
  <<: *except-qa
559
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
560 561 562 563 564
  stage: test
  cache: {}
  dependencies: []
  before_script: []
  script:
565
    - scripts/lint-doc.sh
566
    - scripts/lint-changelog-yaml
567
    - mv doc/ /tmp/gitlab-docs/content/$DOCS_GITLAB_REPO_SUFFIX
568
    - cd /tmp/gitlab-docs
569 570 571
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
572
    - bundle exec nanoc check internal_links
573

574 575
downtime_check:
  <<: *rake-exec
576 577 578 579
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
580
    - /(^docs[\/-].*|.*-docs$)/
581
    - /(^qa[\/-].*|.*-qa$)/
582

583 584
ee_compat_check:
  <<: *rake-exec
585
  dependencies: []
586 587 588
  except:
    - master
    - tags
589
    - /[\d-]+-stable(-ee)?/
590
    - /^security-/
591 592
    - branches@gitlab-org/gitlab-ee
    - branches@gitlab/gitlab-ee
593
  retry: 0
594
  artifacts:
595
    name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
596
    when: always
597 598 599 600
    expire_in: 10d
    paths:
      - ee_compat_check/patches/*.patch

601
db:migrate:reset-pg:
602 603 604
  <<: *db-migrate-reset
  <<: *use-pg

605
db:migrate:reset-mysql:
606 607 608
  <<: *db-migrate-reset
  <<: *use-mysql

609 610 611 612 613 614
db:check-schema-pg:
  <<: *db-migrate-reset
  <<: *use-pg
  script:
    - source scripts/schema_changed.sh

615 616 617 618 619 620 621 622
migration:path-pg:
  <<: *migration-paths
  <<: *use-pg

migration:path-mysql:
  <<: *migration-paths
  <<: *use-mysql

623
.db-rollback: &db-rollback
624
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
625
  script:
626
    - bundle exec rake db:migrate VERSION=20170523121229
627
    - bundle exec rake db:migrate
628

629
db:rollback-pg:
630 631 632
  <<: *db-rollback
  <<: *use-pg

633
db:rollback-mysql:
634 635 636
  <<: *db-rollback
  <<: *use-mysql

637 638
gitlab:setup-pg:
  <<: *gitlab-setup
639 640
  <<: *use-pg

641 642
gitlab:setup-mysql:
  <<: *gitlab-setup
643 644
  <<: *use-mysql

645
# Frontend-related jobs
646
gitlab:assets:compile:
647
  <<: *dedicated-no-docs-pull-cache-job
648
  image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-git-2.18-chrome-71.0-node-8.x-yarn-1.12-graphicsmagick-1.3.29-docker-18.06.1
649 650
  dependencies:
    - setup-test-env
651 652
  services:
    - docker:stable-dind
653 654 655 656 657
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"
    SETUP_DB: "false"
    SKIP_STORAGE_VALIDATION: "true"
658
    WEBPACK_REPORT: "true"
659 660
    # we override the max_old_space_size to prevent OOM errors
    NODE_OPTIONS: --max_old_space_size=3584
661 662
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375
663
  script:
664
    - node --version
665
    - yarn install --frozen-lockfile --production --cache-folder .yarn-cache
666
    - free -m
667
    - bundle exec rake gitlab:assets:compile
668 669
    - time scripts/build_assets_image
    - scripts/clean-old-cached-assets
670 671 672 673
  artifacts:
    name: webpack-report
    expire_in: 31d
    paths:
674
      - webpack-report/
675
      - public/assets/
676
  <<: *assets-compile-cache
677
  only:
678 679 680 681
    - /.+/@gitlab-org/gitlab-ce
    - /.+/@gitlab-org/gitlab-ee
    - /.+/@gitlab/gitlabhq
    - /.+/@gitlab/gitlab-ee
682 683 684
  tags:
    - docker
    - gitlab-org
685

686
gitlab:ui:visual:
687
  <<: *except-docs
688 689 690
  tags:
    - gitlab-org
  before_script: []
691
  allow_failure: true
692 693 694 695 696 697 698 699 700 701 702 703 704 705 706
  dependencies:
    - compile-assets
  script:
    # Remove node modules from GitLab that may conflict with gitlab-ui
    - rm -r node_modules
    - git clone https://gitlab.com/gitlab-org/gitlab-ui.git
    - cp public/assets/application-*.css gitlab-ui/styles/application.css
    - cd gitlab-ui
    - yarn install
    - CSS_URL=./application.css yarn test
  only:
    changes:
      - app/assets/stylesheets/*.scss
      - app/assets/stylesheets/**/*.scss
      - app/assets/stylesheets/**/**/*.scss
707
  except:
708 709
    refs:
      - master
710 711
    variables:
      - $CI_COMMIT_MESSAGE =~ /\[skip visual\]/i
712 713 714 715
  artifacts:
    paths:
      - tests/__image_snapshots__/

716
karma:
717
  <<: *dedicated-no-docs-pull-cache-job
718
  <<: *use-pg
719 720 721
  dependencies:
    - compile-assets
    - setup-test-env
722 723 724
  variables:
    # we override the max_old_space_size to prevent OOM errors
    NODE_OPTIONS: --max_old_space_size=3584
725
  script:
726
    - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log
727
    - date
Jacob Vosmaer's avatar
Jacob Vosmaer committed
728
    - scripts/gitaly-test-spawn
729
    - date
730
    - bundle exec rake karma
731
  coverage: '/^Statements *: (\d+\.\d+%)/'
732 733 734
  artifacts:
    name: coverage-javascript
    expire_in: 31d
735
    when: always
736
    paths:
737 738
      - chrome_debug.log
      - coverage-javascript/
739 740
    reports:
      junit: junit_karma.xml
741

742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
jest:
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
  <<: *use-pg
  dependencies:
  - compile-assets
  - setup-test-env
  script:
    - scripts/gitaly-test-spawn
    - date
    - bundle exec rake karma:fixtures
    - date
    - yarn jest --ci --coverage
  artifacts:
    name: coverage-frontend
    expire_in: 31d
    when: always
    paths:
    - coverage-frontend/
    - junit_jest.xml
    reports:
      junit: junit_jest.xml
  cache:
    key: jest
    paths:
      - tmp/jest/jest/

Shinya Maeda's avatar
Shinya Maeda committed
768
code_quality:
769
  <<: *dedicated-no-docs-no-db-pull-cache-job
770 771
  image: docker:stable
  allow_failure: true
772 773 774
  # gitlab-org runners set `privileged: false` but we need to have it set to true
  # since we're using Docker in Docker
  tags: []
775
  before_script: []
776
  services:
777
    - docker:stable-dind
778 779 780 781 782
  variables:
    SETUP_DB: "false"
    DOCKER_DRIVER: overlay2
  cache: {}
  dependencies: []
783
  script:
784 785
    # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products
    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
Shinya Maeda's avatar
Shinya Maeda committed
786 787 788 789 790
    - docker run
        --env SOURCE_CODE="$PWD"
        --volume "$PWD":/code
        --volume /var/run/docker.sock:/var/run/docker.sock
        "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
791
  artifacts:
792 793
    reports:
      codequality: gl-code-quality-report.json
794
    expire_in: 1 week
795

796
sast:
797 798
  <<: *dedicated-no-docs-no-db-pull-cache-job
  image: docker:stable
799
  variables:
800 801 802 803
    SAST_CONFIDENCE_LEVEL: 2
    DOCKER_DRIVER: overlay2
  allow_failure: true
  tags: []
804
  before_script: []
805 806 807 808
  cache: {}
  dependencies: []
  services:
    - docker:stable-dind
809
  script:
810 811 812 813 814 815
    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
    - docker run
        --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}"
        --volume "$PWD:/code"
        --volume /var/run/docker.sock:/var/run/docker.sock
        "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code
816
  artifacts:
817 818
    reports:
      sast: gl-sast-report.json
819

820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839
dependency_scanning:
  <<: *dedicated-no-docs-no-db-pull-cache-job
  image: docker:stable
  variables:
    DOCKER_DRIVER: overlay2
  allow_failure: true
  tags: []
  before_script: []
  cache: {}
  dependencies: []
  services:
    - docker:stable-dind
  script:
    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
    - docker run
        --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}"
        --volume "$PWD:/code"
        --volume /var/run/docker.sock:/var/run/docker.sock
        "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
  artifacts:
840 841
    reports:
      dependency_scanning: gl-dependency-scanning-report.json
842

843
qa:internal:
844
  <<: *dedicated-no-docs-no-db-pull-cache-job
845 846 847 848 849 850
  services: []
  script:
    - cd qa/
    - bundle install
    - bundle exec rspec

851
qa:selectors:
852
  <<: *dedicated-no-docs-no-db-pull-cache-job
853 854 855 856
  services: []
  script:
    - cd qa/
    - bundle install
857
    - bundle exec bin/qa Test::Sanity::Selectors
858

859
.qa-frontend-node: &qa-frontend-node
860
  <<: *dedicated-no-docs-no-db-pull-cache-job
861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888
  stage: test
  variables:
    NODE_OPTIONS: --max_old_space_size=3584
  cache:
    key: "$CI_JOB_NAME"
    paths:
      - .yarn-cache/
  dependencies: []
  before_script: []
  script:
    - date
    - yarn install --frozen-lockfile --cache-folder .yarn-cache
    - date
    - yarn run webpack-prod

qa-frontend-node:8:
  <<: *qa-frontend-node
  image: node:8-alpine

qa-frontend-node:10:
  <<: *qa-frontend-node
  image: node:10-alpine

qa-frontend-node:latest:
  <<: *qa-frontend-node
  image: node:alpine
  allow_failure: true

889
coverage:
890 891 892 893 894 895 896
  # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to
  # download artifacts from all the rspec jobs instead of from setup-test-env only
  <<: *dedicated-runner
  <<: *except-docs-and-qa
  <<: *pull-cache
  variables:
    SETUP_DB: "false"
897
  stage: post-test
898 899
  script:
    - bundle exec scripts/merge-simplecov
900
  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
901 902 903 904 905 906 907
  artifacts:
    name: coverage
    expire_in: 31d
    paths:
    - coverage/index.html
    - coverage/assets/

908
lint:javascript:report:
909
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
910
  stage: post-test
911
  dependencies: []
912
  before_script: []
913
  script:
914
    - date
915
    - yarn run eslint-report || true # ignore exit code
916 917 918 919
  artifacts:
    name: eslint-report
    expire_in: 31d
    paths:
920
      - eslint-report.html
921

922 923 924 925 926 927 928 929 930 931 932 933 934 935 936
jsdoc:
  <<: *dedicated-no-docs-pull-cache-job
  stage: post-test
  dependencies:
    - compile-assets
  before_script: []
  script:
    - date
    - yarn run jsdoc || true # ignore exit code
  artifacts:
    name: jsdoc
    expire_in: 31d
    paths:
      - jsdoc/

937
pages:
938
  <<: *dedicated-no-docs-no-db-pull-cache-job
939 940 941 942
  before_script: []
  stage: pages
  dependencies:
    - coverage
943 944
    - karma
    - gitlab:assets:compile
945
    - lint:javascript:report
946
    - jsdoc
947 948 949
  script:
    - mv public/ .public/
    - mkdir public/
950
    - mv coverage/ public/coverage-ruby/ || true
951
    - mv coverage-javascript/ public/coverage-javascript/ || true
952
    - mv eslint-report.html public/ || true
953
    - mv webpack-report/ public/webpack-report/ || true
954 955
    - cp .public/assets/application-*.css public/application.css || true
    - cp .public/assets/application-*.css.gz public/application.css.gz || true
956
    - mv jsdoc/ public/jsdoc/ || true
957 958 959 960
  artifacts:
    paths:
      - public
  only:
961
    - master@gitlab-org/gitlab-ce
962
    - master@gitlab-org/gitlab-ee
963 964 965 966

# Insurance in case a gem needed by one of our releases gets yanked from
# rubygems.org in the future.
cache gems:
967
  <<: *dedicated-no-docs-no-db-pull-cache-job
968 969 970 971 972
  script:
    - bundle package --all --all-platforms
  artifacts:
    paths:
      - vendor/cache
973 974
  only:
    - master@gitlab-org/gitlab-ce
975
    - master@gitlab-org/gitlab-ee
976
    - tags
977 978

gitlab_git_test:
979
  <<: *dedicated-runner
980
  <<: *except-docs-and-qa
981 982
  variables:
    SETUP_DB: "false"
983
  before_script: []
984
  dependencies: []
985
  cache: {}
986 987
  script:
    - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes
988 989 990 991 992 993 994

no_ee_check:
  <<: *dedicated-runner
  <<: *except-docs-and-qa
  variables:
    SETUP_DB: "false"
  before_script: []
995
  dependencies: []
996 997 998 999
  cache: {}
  script:
    - scripts/no-ee-check
  only:
1000
    - /.+/@gitlab-org/gitlab-ce
1001

Ian Baum's avatar
Ian Baum committed
1002
# GitLab Review apps
1003
.review-build-cng-base: &review-build-cng-base
1004 1005 1006 1007 1008
  image: ruby:2.5-alpine
  stage: test
  before_script: []
  dependencies: []
  cache: {}
1009
  variables:
1010
    GIT_DEPTH: "1"
1011
    API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
1012 1013
  script:
    - apk add --update openssl curl jq
1014 1015
    - gem install gitlab --no-document
    - source ./scripts/review_apps/review-apps.sh
1016
    - wait_for_job_to_be_done "gitlab:assets:compile"
1017
    - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
1018

1019 1020 1021 1022 1023 1024 1025 1026 1027
review-build-cng:
  <<: *review-only
  <<: *review-build-cng-base

schedule:review-build-cng:
  <<: *review-schedules-only
  <<: *review-build-cng-base

.review-deploy-base: &review-deploy-base
1028
  <<: *review-base
1029 1030
  retry: 2
  allow_failure: true
1031 1032
  variables:
    GIT_DEPTH: "1"
1033 1034
    HOST_SUFFIX: "${CI_ENVIRONMENT_SLUG}"
    DOMAIN: "-${CI_ENVIRONMENT_SLUG}.${REVIEW_APPS_DOMAIN}"
1035
    GITLAB_HELM_CHART_REF: "master"
1036 1037 1038 1039 1040
    API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
  environment:
    <<: *review-environment
    on_stop: review-stop
  before_script:
1041 1042 1043
    - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
    - export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
    - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
1044 1045
    - apk update && apk add jq
    - gem install gitlab --no-document
1046
    - source ./scripts/review_apps/review-apps.sh
1047 1048
  script:
    - wait_for_job_to_be_done "review-build-cng"
1049
  after_script:
1050
    - source ./scripts/review_apps/review-apps.sh
1051 1052 1053 1054
    - check_kube_domain
    - download_gitlab_chart
    - ensure_namespace
    - install_tiller
Ian Baum's avatar
Ian Baum committed
1055
    - install_external_dns
1056 1057 1058
    - time deploy
    - add_license

1059 1060 1061 1062 1063 1064 1065
review-deploy:
  <<: *review-deploy-base

schedule:review-deploy:
  <<: *review-deploy-base
  <<: *review-schedules-only
  script:
1066
    - wait_for_job_to_be_done "schedule:review-build-cng"
1067

1068 1069
.review-qa-base: &review-qa-base
  <<: *review-docker
1070
  allow_failure: true
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
  variables:
    <<: *review-docker-variables
    API_TOKEN: "${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}"
    QA_ARTIFACTS_DIR: "${CI_PROJECT_DIR}/qa"
    QA_CAN_TEST_GIT_PROTOCOL_V2: "false"
    GITLAB_USERNAME: "root"
    GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
    GITLAB_ADMIN_USERNAME: "root"
    GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
    GITHUB_ACCESS_TOKEN: "${REVIEW_APPS_QA_GITHUB_ACCESS_TOKEN}"
    EE_LICENSE: "${REVIEW_APPS_EE_LICENSE}"
1082
    QA_DEBUG: "true"
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
  artifacts:
    paths:
      - ./qa/gitlab-qa-run-*
    expire_in: 7 days
    when: always
  before_script:
    - echo "${QA_IMAGE}"
    - echo "${CI_ENVIRONMENT_URL}"
    - apk update && apk add curl jq
    - source ./scripts/review_apps/review-apps.sh
    - gem install gitlab-qa --no-document ${GITLAB_QA_VERSION:+ --version ${GITLAB_QA_VERSION}}

review-qa-smoke:
  <<: *review-qa-base
  script:
1098
    - wait_for_job_to_be_done "review-deploy"
1099 1100 1101 1102 1103
    - gitlab-qa Test::Instance::Smoke "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"

review-qa-all:
  <<: *review-qa-base
  script:
1104
    - wait_for_job_to_be_done "review-deploy"
1105 1106
    - gitlab-qa Test::Instance::Any "${QA_IMAGE}" "${CI_ENVIRONMENT_URL}"
  when: manual
1107

1108 1109

.review-performance-base: &review-performance-base
1110 1111
  <<: *review-qa-base
  script:
1112 1113
    - wait_for_job_to_be_done "review-deploy"
  after_script:
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
    - mkdir gitlab-exporter
    - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
    - mkdir sitespeed-results
    - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
    - mv sitespeed-results/data/performance.json performance.json
  artifacts:
    paths:
      - sitespeed-results/
    reports:
      performance: performance.json

1125 1126 1127
review-performance:
  <<: *review-performance-base

1128 1129
review-stop:
  <<: *review-base
1130 1131 1132
  <<: *single-script-job
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
  allow_failure: true
Ian Baum's avatar
Ian Baum committed
1133
  variables:
1134
    <<: *single-script-job-variables
Ian Baum's avatar
Ian Baum committed
1135
    SCRIPT_NAME: "review_apps/review-apps.sh"
1136 1137 1138 1139
  when: manual
  environment:
    <<: *review-environment
    action: stop
1140
  script:
Ian Baum's avatar
Ian Baum committed
1141
    - source $(basename "${SCRIPT_NAME}")
1142 1143 1144
    - delete
    - cleanup

1145 1146
schedule:review-cleanup:
  <<: *review-base
1147
  <<: *review-schedules-only
1148
  stage: build
1149 1150 1151
  allow_failure: true
  variables:
    GIT_DEPTH: "1"
1152 1153
  environment:
    name: review/auto-cleanup
1154 1155 1156 1157
  before_script:
    - gem install gitlab --no-document
  script:
    - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
1158 1159 1160 1161 1162 1163

schedule:review-performance:
  <<: *review-performance-base
  <<: *review-schedules-only
  script:
    - wait_for_job_to_be_done "schedule:review-deploy"