.gitlab-ci.yml 24.1 KB
Newer Older
Stan Hu's avatar
Stan Hu committed
1
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-golang-1.9-git-2.18-chrome-69.0-node-10.x-yarn-1.12-postgresql-9.6-graphicsmagick-1.3.29"
2

3 4 5 6 7
.dedicated-runner: &dedicated-runner
  retry: 1
  tags:
    - gitlab-org

8
.default-cache: &default-cache
Stan Hu's avatar
Stan Hu committed
9
  key: "debian-stretch-ruby-2.5.3-node-10.x"
10
  paths:
11 12
    - vendor/ruby
    - .yarn-cache/
13
    - vendor/gitaly-ruby
14 15 16 17 18 19 20 21 22 23

.push-cache: &push-cache
  cache:
    <<: *default-cache
    policy: push

.pull-cache: &pull-cache
  cache:
    <<: *default-cache
    policy: pull
24

25 26
variables:
  MYSQL_ALLOW_EMPTY_PASSWORD: "1"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
27
  RAILS_ENV: "test"
Mike Greiling's avatar
Mike Greiling committed
28
  NODE_ENV: "test"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
29
  SIMPLECOV: "true"
30
  GIT_DEPTH: "20"
31
  GIT_SUBMODULE_STRATEGY: "none"
32
  GET_SOURCES_ATTEMPTS: "3"
33
  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
34
  FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json
35
  BUILD_ASSETS_IMAGE: "false"
36

Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
37
before_script:
Kamil Trzcinski's avatar
Test  
Kamil Trzcinski committed
38
  - bundle --version
39
  - date
40
  - source scripts/utils.sh
41
  - date
42
  - source scripts/prepare_build.sh
43 44 45 46
  - date

after_script:
  - date
47

Kamil Trzcinski's avatar
Kamil Trzcinski committed
48
stages:
49 50 51 52 53
  - build
  - prepare
  - test
  - post-test
  - pages
54
  - post-cleanup
55

56
# Predefined scopes
57
.tests-metadata-state: &tests-metadata-state
58
  <<: *dedicated-runner
59
  variables:
60
    TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
61 62
  before_script:
    - source scripts/utils.sh
63 64 65
  artifacts:
    expire_in: 31d
    paths:
66
      - knapsack/
67
      - rspec_flaky/
68

69 70
.use-pg: &use-pg
  services:
71
    - postgres:9.6
72 73 74
    - redis:alpine

.use-mysql: &use-mysql
75
  services:
76
    - mysql:5.7
77
    - redis:alpine
78

79 80
.rails4: &rails4
  allow_failure: false
81
  except:
82
    variables:
83 84
      - $CI_COMMIT_REF_NAME =~ /(^docs[\/-].*|.*-docs$)/
      - $CI_COMMIT_REF_NAME =~ /(^qa[\/-].*|.*-qa$)/
85 86
      - $CI_COMMIT_REF_NAME =~ /norails4/
      - $RAILS5_DISABLED
87
  variables:
88 89
    BUNDLE_GEMFILE: "Gemfile.rails4"
    RAILS5: "false"
90

91 92
# Skip all jobs except the ones that begin with 'docs/'.
# Used for commits including ONLY documentation changes.
93
# https://docs.gitlab.com/ce/development/documentation/#testing
94 95
.except-docs: &except-docs
  except:
96
    - /(^docs[\/-].*|.*-docs$)/
97

98 99 100 101
.except-qa: &except-qa
  except:
    - /(^qa[\/-].*|.*-qa$)/

102 103 104 105 106
.except-docs-and-qa: &except-docs-and-qa
  except:
    - /(^docs[\/-].*|.*-docs$)/
    - /(^qa[\/-].*|.*-qa$)/

107 108 109
# Jobs that only need to pull cache
.dedicated-no-docs-pull-cache-job: &dedicated-no-docs-pull-cache-job
  <<: *dedicated-runner
110
  <<: *except-docs
111 112 113 114 115 116 117 118 119 120 121
  <<: *pull-cache
  dependencies:
    - setup-test-env
  stage: test

# Jobs that do not need a DB
.dedicated-no-docs-no-db-pull-cache-job: &dedicated-no-docs-no-db-pull-cache-job
  <<: *dedicated-no-docs-pull-cache-job
  variables:
    SETUP_DB: "false"

122 123 124 125
.dedicated-no-docs-and-no-qa-pull-cache-job: &dedicated-no-docs-and-no-qa-pull-cache-job
  <<: *dedicated-no-docs-pull-cache-job
  <<: *except-docs-and-qa

126
.single-script-job: &single-script-job
Stan Hu's avatar
Stan Hu committed
127
  image: ruby:2.5-alpine
128
  stage: test
129 130 131 132 133 134 135 136 137 138 139
  cache: {}
  dependencies: []
  variables: &single-script-job-variables
    GIT_STRATEGY: none
  before_script:
    # We need to download the script rather than clone the repo since the
    # package-and-qa job will not be able to run when the branch gets
    # deleted (when merging the MR).
    - export SCRIPT_NAME="${SCRIPT_NAME:-$CI_JOB_NAME}"
    - apk add --update openssl
    - wget $CI_PROJECT_URL/raw/$CI_COMMIT_SHA/scripts/$SCRIPT_NAME
Ian Baum's avatar
Ian Baum committed
140
    - chmod 755 $(basename $SCRIPT_NAME)
141

142 143 144 145 146
.rake-exec: &rake-exec
  <<: *dedicated-no-docs-no-db-pull-cache-job
  script:
    - bundle exec rake $CI_JOB_NAME

147
.rspec-metadata: &rspec-metadata
148
  <<: *dedicated-runner
149
  <<: *except-docs-and-qa
150
  <<: *pull-cache
151
  stage: test
152
  script:
153
    - JOB_NAME=( $CI_JOB_NAME )
154 155
    - TEST_TOOL=${JOB_NAME[0]}
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${TEST_TOOL}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
156
    - export KNAPSACK_GENERATE_REPORT=true
157
    - export SUITE_FLAKY_RSPEC_REPORT_PATH=${FLAKY_RSPEC_SUITE_REPORT_PATH}
158 159
    - export FLAKY_RSPEC_REPORT_PATH=rspec_flaky/all_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
    - export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/new_${TEST_TOOL}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
160
    - export FLAKY_RSPEC_GENERATE_REPORT=true
161
    - export CACHE_CLASSES=true
162
    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
163
    - '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}'
164
    - '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}'
Jacob Vosmaer's avatar
Jacob Vosmaer committed
165
    - scripts/gitaly-test-spawn
166
    - knapsack rspec "--color --format documentation --format RspecJunitFormatter --out junit_rspec.xml"
167 168
  artifacts:
    expire_in: 31d
169
    when: always
170
    paths:
171 172
      - coverage/
      - knapsack/
173
      - rspec_flaky/
174
      - tmp/capybara/
175 176
    reports:
      junit: junit_rspec.xml
177

178 179
.rspec-metadata-pg: &rspec-metadata-pg
  <<: *rspec-metadata
180 181
  <<: *use-pg

182
.rspec-metadata-pg-rails4: &rspec-metadata-pg-rails4
183
  <<: *rspec-metadata-pg
184
  <<: *rails4
185

186 187
.rspec-metadata-mysql: &rspec-metadata-mysql
  <<: *rspec-metadata
188 189
  <<: *use-mysql

190
.rspec-metadata-mysql-rails4: &rspec-metadata-mysql-rails4
191
  <<: *rspec-metadata-mysql
192
  <<: *rails4
193

194 195 196 197 198 199 200
.only-canonical-masters: &only-canonical-masters
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee

201
.gitlab-setup: &gitlab-setup
202
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
203 204
  <<: *use-pg
  variables:
205
    SETUP_DB: "false"
206
  script:
207 208 209 210 211 212 213 214 215 216 217
    # Manually clone gitlab-test and only seed this project in
    # db/fixtures/development/04_project.rb thanks to SIZE=1 below
    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
       /home/git/repositories/gitlab-org/gitlab-test.git
    - scripts/gitaly-test-spawn
    - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup
  artifacts:
    when: on_failure
    expire_in: 1d
    paths:
      - log/development.log
218

219 220
# DB migration, rollback, and seed jobs
.db-migrate-reset: &db-migrate-reset
221
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
222 223 224 225
  script:
    - bundle exec rake db:migrate:reset

.migration-paths: &migration-paths
226
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
227
  variables:
228
    SETUP_DB: "false"
229 230 231
  script:
    - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0
    - git checkout -f FETCH_HEAD
Stan Hu's avatar
Stan Hu committed
232 233
    - sed -i "s/gem 'oj', '~> 2.17.4'//" Gemfile
    - bundle update google-protobuf grpc
234 235 236 237 238
    - bundle install $BUNDLE_INSTALL_FLAGS
    - date
    - cp config/gitlab.yml.example config/gitlab.yml
    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
    - date
239
    - git checkout -f $CI_COMMIT_SHA
240 241 242 243 244 245 246 247 248
    - bundle install $BUNDLE_INSTALL_FLAGS
    - date
    - . scripts/prepare_build.sh
    - date
    - bundle exec rake db:migrate

##
# Trigger a package build in omnibus-gitlab repository
#
249
package-and-qa:
250
  <<: *single-script-job
251
  variables:
252
    <<: *single-script-job-variables
253
    SCRIPT_NAME: trigger-build
254
  retry: 0
255
  script:
256
    - gem install gitlab --no-document
257
    - ./$SCRIPT_NAME omnibus
258
  when: manual
259 260 261 262
  only:
    - //@gitlab-org/gitlab-ce
    - //@gitlab-org/gitlab-ee

263 264 265 266 267 268 269
# Review docs base
.review-docs: &review-docs
  <<: *dedicated-runner
  <<: *single-script-job
  variables:
    <<: *single-script-job-variables
    SCRIPT_NAME: trigger-build-docs
270
  environment:
271
    name: review-docs/$CI_COMMIT_REF_SLUG
272
    # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables
273
    # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693
274
    url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX
275
    on_stop: review-docs-cleanup
276 277 278 279 280 281

# Trigger a manual docs build in gitlab-docs only on non docs-only branches.
# Useful to preview the docs changes live.
review-docs-deploy-manual:
  <<: *review-docs
  stage: build
282
  script:
283
    - gem install gitlab --no-document
284
    - ./$SCRIPT_NAME deploy
285 286
  when: manual
  only:
287 288
    - branches@gitlab-org/gitlab-ce
    - branches@gitlab-org/gitlab-ee
289 290 291 292 293 294 295 296
  <<: *except-docs-and-qa

# Always trigger a docs build in gitlab-docs only on docs-only branches.
# Useful to preview the docs changes live.
review-docs-deploy:
  <<: *review-docs
  stage: post-test
  script:
297
    - gem install gitlab --no-document
298 299
    - ./$SCRIPT_NAME deploy
  only:
300 301
    - /(^docs[\/-].*|.*-docs$)/@gitlab-org/gitlab-ce
    - /(^docs[\/-].*|.*-docs$)/@gitlab-org/gitlab-ee
302
  <<: *except-qa
303 304 305 306 307 308

# Cleanup remote environment of gitlab-docs
review-docs-cleanup:
  <<: *review-docs
  stage: post-cleanup
  environment:
309
    name: review-docs/$CI_COMMIT_REF_SLUG
310 311
    action: stop
  script:
312
    - gem install gitlab --no-document
313
    - ./$SCRIPT_NAME cleanup
314 315 316 317
  when: manual
  only:
    - branches@gitlab-org/gitlab-ce
    - branches@gitlab-org/gitlab-ee
318

319 320 321 322
##
# Trigger a docker image build in CNG (Cloud Native GitLab) repository
#
cloud-native-image:
Stan Hu's avatar
Stan Hu committed
323
  image: ruby:2.5-alpine
324
  before_script: []
325
  dependencies: []
326
  stage: post-test
327
  allow_failure: true
328 329
  variables:
    GIT_DEPTH: "1"
330 331
  cache: {}
  script:
332 333
    - gem install gitlab --no-document
    - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng
334 335 336 337
  only:
    - tags@gitlab-org/gitlab-ce
    - tags@gitlab-org/gitlab-ee

338 339 340
# Retrieve knapsack and rspec_flaky reports
retrieve-tests-metadata:
  <<: *tests-metadata-state
341
  <<: *except-docs-and-qa
342
  stage: prepare
343
  cache:
344
    key: tests_metadata
345
    policy: pull
346
  script:
347
    - mkdir -p knapsack/${CI_PROJECT_NAME}/
348
    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
349
    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
350
    - mkdir -p rspec_flaky/
351 352
    - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH
    - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}'
353

354 355
update-tests-metadata:
  <<: *tests-metadata-state
356
  <<: *only-canonical-masters
357
  stage: post-test
358
  cache:
359
    key: tests_metadata
360 361
    paths:
      - knapsack/
362
      - rspec_flaky/
363
    policy: push
364
  script:
365
    - retry gem install fog-aws mime-types activesupport --no-document
366
    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
367
    - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
368
    - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
blackst0ne's avatar
blackst0ne committed
369
    - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH'
370
    - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH'
371
    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
372
    - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
373 374 375

flaky-examples-check:
  <<: *dedicated-runner
Stan Hu's avatar
Stan Hu committed
376
  image: ruby:2.5-alpine
377 378 379 380 381
  services: []
  before_script: []
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
382
    NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json
383
  stage: post-test
384
  allow_failure: true
385
  retry: 0
386 387 388 389
  only:
    - branches
  except:
    - master
390
    - /(^docs[\/-].*|.*-docs$)/
391
    - /(^qa[\/-].*|.*-qa$)/
392 393 394 395 396 397
  artifacts:
    expire_in: 30d
    paths:
      - rspec_flaky/
  script:
    - '[[ -f $NEW_FLAKY_SPECS_REPORT ]] || echo "{}" > ${NEW_FLAKY_SPECS_REPORT}'
398
    - scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json
399
    - scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT
400

401
compile-assets:
402
  <<: *dedicated-runner
403
  <<: *except-docs
404
  <<: *use-pg
405
  stage: prepare
406 407
  cache:
    <<: *default-cache
408
  script:
409
    - node --version
410
    - date
411
    - yarn install --frozen-lockfile --cache-folder .yarn-cache
412
    - date
413
    - free -m
Mike Greiling's avatar
Mike Greiling committed
414
    - bundle exec rake gitlab:assets:compile
415 416 417
  variables:
    # we override the max_old_space_size to prevent OOM errors
    NODE_OPTIONS: --max_old_space_size=3584
418 419 420
  artifacts:
    expire_in: 7d
    paths:
421
      - node_modules
422
      - public/assets
423 424 425 426 427 428 429 430 431 432 433

setup-test-env:
  <<: *dedicated-runner
  <<: *except-docs
  <<: *use-pg
  stage: prepare
  cache:
    <<: *default-cache
  script:
    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
    - scripts/gitaly-test-build # Do not use 'bundle exec' here
434
    - BUNDLE_GEMFILE=Gemfile.rails4 bundle install $BUNDLE_INSTALL_FLAGS
435 436 437
  artifacts:
    expire_in: 7d
    paths:
438
      - tmp/tests
439
      - config/secrets.yml
440
      - vendor/gitaly-ruby
441

442
danger-review:
443
  <<: *pull-cache
444
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
445 446
  stage: test
  allow_failure: true
447 448
  dependencies: []
  before_script: []
449
  only:
450 451
    variables:
      - $DANGER_GITLAB_API_TOKEN
452 453 454 455 456
  except:
    refs:
      - master
    variables:
      - $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/
457
      - $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/
458 459
  script:
    - git version
460
    - node --version
461
    - yarn install --frozen-lockfile --cache-folder .yarn-cache
462 463
    - danger --fail-on-errors=true

464 465 466 467 468 469 470 471 472 473 474 475 476 477 478
rspec-pg:
  <<: *rspec-metadata-pg
  parallel: 30

rspec-mysql:
  <<: *rspec-metadata-mysql
  parallel: 30

rspec-pg-rails4:
  <<: *rspec-metadata-pg-rails4
  parallel: 30

rspec-mysql-rails4:
  <<: *rspec-metadata-mysql-rails4
  parallel: 30
479

480
static-analysis:
481 482 483 484
  <<: *dedicated-no-docs-no-db-pull-cache-job
  dependencies:
    - compile-assets
    - setup-test-env
485
  script:
486
    - scripts/static-analysis
Lin Jen-Shin's avatar
Lin Jen-Shin committed
487
  cache:
Stan Hu's avatar
Stan Hu committed
488
    key: "debian-stretch-ruby-2.5.3-node-10.x-and-rubocop"
Lin Jen-Shin's avatar
Lin Jen-Shin committed
489 490 491 492
    paths:
      - vendor/ruby
      - .yarn-cache/
      - tmp/rubocop_cache
493

494 495 496 497
# Documentation checks:
# - Check validity of relative links
# - Make sure cURL examples in API docs use the full switches
docs lint:
498
  <<: *dedicated-runner
499
  <<: *except-qa
Evan Read's avatar
Evan Read committed
500
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
501 502 503 504 505
  stage: test
  cache: {}
  dependencies: []
  before_script: []
  script:
506
    - scripts/lint-doc.sh
507
    - scripts/lint-changelog-yaml
Evan Read's avatar
Evan Read committed
508 509
    - mv doc/ /tmp/gitlab-docs/content/
    - cd /tmp/gitlab-docs
510 511 512 513 514
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links

515 516
downtime_check:
  <<: *rake-exec
517 518 519 520
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
521
    - /(^docs[\/-].*|.*-docs$)/
522
    - /(^qa[\/-].*|.*-qa$)/
523

524
rails4_gemfile_lock_check:
525 526 527
  <<: *dedicated-no-docs-no-db-pull-cache-job
  <<: *except-docs-and-qa
  script:
528
    - scripts/rails4-gemfile-lock-check
529

530 531
ee_compat_check:
  <<: *rake-exec
532
  dependencies: []
533 534 535
  except:
    - master
    - tags
536
    - /[\d-]+-stable(-ee)?/
537
    - /^security-/
538 539
    - branches@gitlab-org/gitlab-ee
    - branches@gitlab/gitlab-ee
540
  retry: 0
541
  artifacts:
542
    name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
543
    when: always
544 545 546 547
    expire_in: 10d
    paths:
      - ee_compat_check/patches/*.patch

548
db:migrate:reset-pg:
549 550 551
  <<: *db-migrate-reset
  <<: *use-pg

552
db:migrate:reset-mysql:
553 554 555
  <<: *db-migrate-reset
  <<: *use-mysql

556 557 558 559 560 561
db:check-schema-pg:
  <<: *db-migrate-reset
  <<: *use-pg
  script:
    - source scripts/schema_changed.sh

562 563 564 565 566 567 568 569
migration:path-pg:
  <<: *migration-paths
  <<: *use-pg

migration:path-mysql:
  <<: *migration-paths
  <<: *use-mysql

570
.db-rollback: &db-rollback
571
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
572
  script:
573
    - bundle exec rake db:migrate VERSION=20170523121229
574
    - bundle exec rake db:migrate
575

576
db:rollback-pg:
577 578 579
  <<: *db-rollback
  <<: *use-pg

580
db:rollback-mysql:
581 582 583
  <<: *db-rollback
  <<: *use-mysql

584 585
gitlab:setup-pg:
  <<: *gitlab-setup
586 587
  <<: *use-pg

588 589
gitlab:setup-mysql:
  <<: *gitlab-setup
590 591
  <<: *use-mysql

592
# Frontend-related jobs
593
gitlab:assets:compile:
594
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
Stan Hu's avatar
Stan Hu committed
595
  image: dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.5.3-git-2.18-chrome-69.0-node-8.x-yarn-1.2-graphicsmagick-1.3.29-docker-18.06.1
596
  dependencies: []
597 598
  services:
    - docker:stable-dind
599 600 601 602 603
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"
    SETUP_DB: "false"
    SKIP_STORAGE_VALIDATION: "true"
604
    WEBPACK_REPORT: "true"
605 606
    # we override the max_old_space_size to prevent OOM errors
    NODE_OPTIONS: --max_old_space_size=3584
607 608
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375
609
  script:
610
    - date
611
    - yarn install --frozen-lockfile --production --cache-folder .yarn-cache
612
    - date
613
    - free -m
614
    - bundle exec rake gitlab:assets:compile
615
    - scripts/build_assets_image
616 617 618 619
  artifacts:
    name: webpack-report
    expire_in: 31d
    paths:
620
      - webpack-report/
621
      - public/assets/
622 623
  tags:
    - docker
624

625
karma:
Rémy Coutable's avatar
Rémy Coutable committed
626
  <<: *dedicated-no-docs-pull-cache-job
627
  <<: *use-pg
628 629 630
  dependencies:
    - compile-assets
    - setup-test-env
631 632 633
  variables:
    # we override the max_old_space_size to prevent OOM errors
    NODE_OPTIONS: --max_old_space_size=3584
634
  script:
635
    - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log
636
    - date
Jacob Vosmaer's avatar
Jacob Vosmaer committed
637
    - scripts/gitaly-test-spawn
638
    - date
639
    - bundle exec rake karma
640
  coverage: '/^Statements *: (\d+\.\d+%)/'
641 642 643
  artifacts:
    name: coverage-javascript
    expire_in: 31d
644
    when: always
645
    paths:
646 647
      - chrome_debug.log
      - coverage-javascript/
648 649
    reports:
      junit: junit_karma.xml
650

Shinya Maeda's avatar
Shinya Maeda committed
651
code_quality:
652
  <<: *dedicated-no-docs-no-db-pull-cache-job
653 654
  image: docker:stable
  allow_failure: true
655 656 657
  # gitlab-org runners set `privileged: false` but we need to have it set to true
  # since we're using Docker in Docker
  tags: []
658
  before_script: []
659
  services:
660
    - docker:stable-dind
661 662 663 664 665
  variables:
    SETUP_DB: "false"
    DOCKER_DRIVER: overlay2
  cache: {}
  dependencies: []
666
  script:
667 668
    # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products
    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
Shinya Maeda's avatar
Shinya Maeda committed
669 670 671 672 673
    - docker run
        --env SOURCE_CODE="$PWD"
        --volume "$PWD":/code
        --volume /var/run/docker.sock:/var/run/docker.sock
        "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
674
  artifacts:
675 676
    reports:
      codequality: gl-code-quality-report.json
677
    expire_in: 1 week
678

679
sast:
680 681
  <<: *dedicated-no-docs-no-db-pull-cache-job
  image: docker:stable
682
  variables:
683 684 685 686
    SAST_CONFIDENCE_LEVEL: 2
    DOCKER_DRIVER: overlay2
  allow_failure: true
  tags: []
687
  before_script: []
688 689 690 691
  cache: {}
  dependencies: []
  services:
    - docker:stable-dind
692
  script:
693 694 695 696 697 698
    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
    - docker run
        --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}"
        --volume "$PWD:/code"
        --volume /var/run/docker.sock:/var/run/docker.sock
        "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code
699
  artifacts:
700 701
    reports:
      sast: gl-sast-report.json
702

703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722
dependency_scanning:
  <<: *dedicated-no-docs-no-db-pull-cache-job
  image: docker:stable
  variables:
    DOCKER_DRIVER: overlay2
  allow_failure: true
  tags: []
  before_script: []
  cache: {}
  dependencies: []
  services:
    - docker:stable-dind
  script:
    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
    - docker run
        --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}"
        --volume "$PWD:/code"
        --volume /var/run/docker.sock:/var/run/docker.sock
        "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
  artifacts:
723 724
    reports:
      dependency_scanning: gl-dependency-scanning-report.json
725

726
qa:internal:
727
  <<: *dedicated-no-docs-no-db-pull-cache-job
728 729 730 731 732 733
  services: []
  script:
    - cd qa/
    - bundle install
    - bundle exec rspec

734
qa:selectors:
735
  <<: *dedicated-no-docs-no-db-pull-cache-job
736 737 738 739
  services: []
  script:
    - cd qa/
    - bundle install
740
    - bundle exec bin/qa Test::Sanity::Selectors
741

742
coverage:
743 744 745 746 747 748 749
  # Don't include dedicated-no-docs-no-db-pull-cache-job here since we need to
  # download artifacts from all the rspec jobs instead of from setup-test-env only
  <<: *dedicated-runner
  <<: *except-docs-and-qa
  <<: *pull-cache
  variables:
    SETUP_DB: "false"
750
  stage: post-test
751 752
  script:
    - bundle exec scripts/merge-simplecov
753
  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
754 755 756 757 758 759 760
  artifacts:
    name: coverage
    expire_in: 31d
    paths:
    - coverage/index.html
    - coverage/assets/

761
lint:javascript:report:
762
  <<: *dedicated-no-docs-and-no-qa-pull-cache-job
763
  stage: post-test
764
  dependencies: []
765
  before_script: []
766
  script:
767
    - date
768
    - yarn run eslint-report || true # ignore exit code
769 770 771 772
  artifacts:
    name: eslint-report
    expire_in: 31d
    paths:
773
      - eslint-report.html
774 775

pages:
776
  <<: *dedicated-no-docs-no-db-pull-cache-job
777 778 779 780
  before_script: []
  stage: pages
  dependencies:
    - coverage
781 782
    - karma
    - gitlab:assets:compile
783
    - lint:javascript:report
784 785 786
  script:
    - mv public/ .public/
    - mkdir public/
787
    - mv coverage/ public/coverage-ruby/ || true
788
    - mv coverage-javascript/ public/coverage-javascript/ || true
789
    - mv eslint-report.html public/ || true
790
    - mv webpack-report/ public/webpack-report/ || true
791 792 793 794
  artifacts:
    paths:
      - public
  only:
795
    - master@gitlab-org/gitlab-ce
796
    - master@gitlab-org/gitlab-ee
797 798 799 800

# Insurance in case a gem needed by one of our releases gets yanked from
# rubygems.org in the future.
cache gems:
801
  <<: *dedicated-no-docs-no-db-pull-cache-job
802 803 804 805 806
  script:
    - bundle package --all --all-platforms
  artifacts:
    paths:
      - vendor/cache
807 808
  only:
    - master@gitlab-org/gitlab-ce
809
    - master@gitlab-org/gitlab-ee
810
    - tags
811 812

gitlab_git_test:
813
  <<: *dedicated-runner
814
  <<: *except-docs-and-qa
815 816
  variables:
    SETUP_DB: "false"
817
  before_script: []
818
  dependencies: []
819
  cache: {}
820 821
  script:
    - spec/support/prepare-gitlab-git-test-for-commit --check-for-changes
822 823 824 825 826 827 828

no_ee_check:
  <<: *dedicated-runner
  <<: *except-docs-and-qa
  variables:
    SETUP_DB: "false"
  before_script: []
829
  dependencies: []
830 831 832 833 834
  cache: {}
  script:
    - scripts/no-ee-check
  only:
    - //@gitlab-org/gitlab-ce
835

Ian Baum's avatar
Ian Baum committed
836
# GitLab Review apps
837 838 839 840
review:
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
  stage: test
  allow_failure: true
Ian Baum's avatar
Ian Baum committed
841 842
  before_script:
    - gem install gitlab --no-document
843 844 845 846 847 848 849 850 851 852 853 854 855 856 857
  variables:
    GIT_DEPTH: "1"
    HOST_SUFFIX: "$CI_ENVIRONMENT_SLUG"
    DOMAIN: "-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN"
    GITLAB_HELM_CHART_REF: "master"
  script:
    - export GITLAB_SHELL_VERSION=$(<GITLAB_SHELL_VERSION)
    - export GITALY_VERSION=$(<GITALY_SERVER_VERSION)
    - export GITLAB_WORKHORSE_VERSION=$(<GITLAB_WORKHORSE_VERSION)
    - source ./scripts/review_apps/review-apps.sh
    - BUILD_TRIGGER_TOKEN=$REVIEW_APPS_BUILD_TRIGGER_TOKEN ./scripts/trigger-build cng
    - check_kube_domain
    - download_gitlab_chart
    - ensure_namespace
    - install_tiller
Ian Baum's avatar
Ian Baum committed
858
    - install_external_dns
859 860 861 862 863 864 865 866
    - deploy
  environment:
    name: review/$CI_COMMIT_REF_NAME
    url: https://gitlab-$CI_ENVIRONMENT_SLUG.$REVIEW_APPS_DOMAIN
    on_stop: stop_review
  only:
    refs:
      - branches@gitlab-org/gitlab-ce
Ian Baum's avatar
Ian Baum committed
867
      - branches@gitlab-org/gitlab-ee
868 869 870 871 872 873 874 875 876
    kubernetes: active
  except:
    refs:
      - master
      - /(^docs[\/-].*|.*-docs$)/

stop_review:
  <<: *single-script-job
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
Ian Baum's avatar
Ian Baum committed
877
  stage: test
878 879 880
  allow_failure: true
  cache: {}
  dependencies: []
Ian Baum's avatar
Ian Baum committed
881 882
  variables:
    SCRIPT_NAME: "review_apps/review-apps.sh"
883
  script:
Ian Baum's avatar
Ian Baum committed
884
    - source $(basename "${SCRIPT_NAME}")
885 886 887 888 889 890 891 892
    - delete
    - cleanup
  when: manual
  environment:
    name: review/$CI_COMMIT_REF_NAME
    action: stop
  only:
    refs:
Ian Baum's avatar
Ian Baum committed
893
      - branches@gitlab-org/gitlab-ce
894 895 896 897 898 899
      - branches@gitlab-org/gitlab-ee
    kubernetes: active
  except:
    - master
    - /(^docs[\/-].*|.*-docs$)/

900
schedule:review_apps_cleanup:
901 902
  <<: *dedicated-no-docs-pull-cache-job
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
903
  stage: build
904 905 906 907 908 909 910 911 912
  allow_failure: true
  cache: {}
  dependencies: []
  before_script:
    - gem install gitlab --no-document
  variables:
    GIT_DEPTH: "1"
  script:
    - ruby -rrubygems scripts/review_apps/automated_cleanup.rb
913 914 915
  environment:
    name: review/auto-cleanup
    action: stop
916 917
  only:
    refs:
Ian Baum's avatar
Ian Baum committed
918
      - schedules@gitlab-org/gitlab-ce
919
      - schedules@gitlab-org/gitlab-ee
920
    kubernetes: active
921
  except:
922
    - tags
923
    - /(^docs[\/-].*|.*-docs$)/