.gitlab-ci.yml 15 KB
Newer Older
1
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
2

3
cache:
Z.J. van de Weg's avatar
Z.J. van de Weg committed
4
  key: "ruby-233"
5
  paths:
6
  - vendor/ruby
7

8 9
variables:
  MYSQL_ALLOW_EMPTY_PASSWORD: "1"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
10
  RAILS_ENV: "test"
Mike Greiling's avatar
Mike Greiling committed
11
  NODE_ENV: "test"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
12
  SIMPLECOV: "true"
13
  GIT_DEPTH: "20"
14
  GIT_SUBMODULE_STRATEGY: "none"
15
  PHANTOMJS_VERSION: "2.1.1"
16
  GET_SOURCES_ATTEMPTS: "3"
17 18
  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
  KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json
19

Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
20
before_script:
Kamil Trzcinski's avatar
Test  
Kamil Trzcinski committed
21
  - bundle --version
22 23
  - source scripts/utils.sh
  - source scripts/prepare_build.sh
24

Kamil Trzcinski's avatar
Kamil Trzcinski committed
25
stages:
Kamil Trzcinski's avatar
Kamil Trzcinski committed
26
- prepare
Kamil Trzcinski's avatar
Kamil Trzcinski committed
27
- test
28
- post-test
29
- pages
30

31 32 33 34 35
# Predefined scopes
.dedicated-runner: &dedicated-runner
  tags:
    - gitlab-org

36 37 38 39 40
.knapsack-state: &knapsack-state
  services: []
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
41
    KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
42 43 44
  cache:
    key: "knapsack"
    paths:
45
      - knapsack/
46 47 48
  artifacts:
    expire_in: 31d
    paths:
49
      - knapsack/
50

51 52 53 54 55 56
.use-pg: &use-pg
  services:
    - postgres:latest
    - redis:alpine

.use-mysql: &use-mysql
57 58 59
  services:
    - mysql:latest
    - redis:alpine
60

61 62
.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
  only:
63
    - /mysql/
64 65 66 67 68 69 70
    - master@gitlab-org/gitlab-ce
    - master@gitlab/gitlabhq
    - tags@gitlab-org/gitlab-ce
    - tags@gitlab/gitlabhq
    - //@gitlab-org/gitlab-ee
    - //@gitlab/gitlab-ee

71 72 73
# Skip all jobs except the ones that begin with 'docs/'.
# Used for commits including ONLY documentation changes.
# https://docs.gitlab.com/ce/development/writing_documentation.html#testing
74 75
.except-docs: &except-docs
  except:
76
    - /^docs\/.*/
77

78 79
.rspec-knapsack: &rspec-knapsack
  stage: test
80
  <<: *dedicated-runner
81
  script:
82
    - JOB_NAME=( $CI_JOB_NAME )
83 84
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
85
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
86
    - export KNAPSACK_GENERATE_REPORT=true
87
    - export CACHE_CLASSES=true
88
    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
89 90 91
    - knapsack rspec "--color --format documentation"
  artifacts:
    expire_in: 31d
92
    when: always
93
    paths:
94 95 96
      - coverage/
      - knapsack/
      - tmp/capybara/
97

98 99 100
.rspec-knapsack-pg: &rspec-knapsack-pg
  <<: *rspec-knapsack
  <<: *use-pg
101
  <<: *except-docs
102 103 104 105

.rspec-knapsack-mysql: &rspec-knapsack-mysql
  <<: *rspec-knapsack
  <<: *use-mysql
106
  <<: *only-master-and-ee-or-mysql
107
  <<: *except-docs
108

109 110
.spinach-knapsack: &spinach-knapsack
  stage: test
111
  <<: *dedicated-runner
112
  script:
113
    - JOB_NAME=( $CI_JOB_NAME )
114 115
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
116
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
117
    - export KNAPSACK_GENERATE_REPORT=true
118
    - export CACHE_CLASSES=true
119
    - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
120 121 122
    - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
  artifacts:
    expire_in: 31d
123
    when: always
124
    paths:
125 126 127
      - coverage/
      - knapsack/
      - tmp/capybara/
128

129 130 131
.spinach-knapsack-pg: &spinach-knapsack-pg
  <<: *spinach-knapsack
  <<: *use-pg
132
  <<: *except-docs
133 134 135 136

.spinach-knapsack-mysql: &spinach-knapsack-mysql
  <<: *spinach-knapsack
  <<: *use-mysql
137
  <<: *only-master-and-ee-or-mysql
138
  <<: *except-docs
139

140 141 142 143
# Prepare and merge knapsack tests
knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
144
  <<: *except-docs
145 146
  stage: prepare
  script:
147
    - mkdir -p knapsack/${CI_PROJECT_NAME}/
148 149
    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
    - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH
150 151 152 153 154 155 156 157
    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
    - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'

update-knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: post-test
  script:
158 159
    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json
    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json
160 161 162 163 164 165 166
    - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
167 168

setup-test-env:
169
  <<: *use-pg
170
  <<: *dedicated-runner
171
  <<: *except-docs
172 173
  stage: prepare
  script:
174
    - node --version
175
    - yarn install --pure-lockfile
Mike Greiling's avatar
Mike Greiling committed
176
    - bundle exec rake gitlab:assets:compile
177 178 179 180
    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
  artifacts:
    expire_in: 7d
    paths:
181
      - node_modules
182 183 184
      - public/assets
      - tmp/tests

185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
rspec pg 0 20: *rspec-knapsack-pg
rspec pg 1 20: *rspec-knapsack-pg
rspec pg 2 20: *rspec-knapsack-pg
rspec pg 3 20: *rspec-knapsack-pg
rspec pg 4 20: *rspec-knapsack-pg
rspec pg 5 20: *rspec-knapsack-pg
rspec pg 6 20: *rspec-knapsack-pg
rspec pg 7 20: *rspec-knapsack-pg
rspec pg 8 20: *rspec-knapsack-pg
rspec pg 9 20: *rspec-knapsack-pg
rspec pg 10 20: *rspec-knapsack-pg
rspec pg 11 20: *rspec-knapsack-pg
rspec pg 12 20: *rspec-knapsack-pg
rspec pg 13 20: *rspec-knapsack-pg
rspec pg 14 20: *rspec-knapsack-pg
rspec pg 15 20: *rspec-knapsack-pg
rspec pg 16 20: *rspec-knapsack-pg
rspec pg 17 20: *rspec-knapsack-pg
rspec pg 18 20: *rspec-knapsack-pg
rspec pg 19 20: *rspec-knapsack-pg

rspec mysql 0 20: *rspec-knapsack-mysql
rspec mysql 1 20: *rspec-knapsack-mysql
rspec mysql 2 20: *rspec-knapsack-mysql
rspec mysql 3 20: *rspec-knapsack-mysql
rspec mysql 4 20: *rspec-knapsack-mysql
rspec mysql 5 20: *rspec-knapsack-mysql
rspec mysql 6 20: *rspec-knapsack-mysql
rspec mysql 7 20: *rspec-knapsack-mysql
rspec mysql 8 20: *rspec-knapsack-mysql
rspec mysql 9 20: *rspec-knapsack-mysql
rspec mysql 10 20: *rspec-knapsack-mysql
rspec mysql 11 20: *rspec-knapsack-mysql
rspec mysql 12 20: *rspec-knapsack-mysql
rspec mysql 13 20: *rspec-knapsack-mysql
rspec mysql 14 20: *rspec-knapsack-mysql
rspec mysql 15 20: *rspec-knapsack-mysql
rspec mysql 16 20: *rspec-knapsack-mysql
rspec mysql 17 20: *rspec-knapsack-mysql
rspec mysql 18 20: *rspec-knapsack-mysql
rspec mysql 19 20: *rspec-knapsack-mysql

spinach pg 0 10: *spinach-knapsack-pg
spinach pg 1 10: *spinach-knapsack-pg
spinach pg 2 10: *spinach-knapsack-pg
spinach pg 3 10: *spinach-knapsack-pg
spinach pg 4 10: *spinach-knapsack-pg
spinach pg 5 10: *spinach-knapsack-pg
spinach pg 6 10: *spinach-knapsack-pg
spinach pg 7 10: *spinach-knapsack-pg
spinach pg 8 10: *spinach-knapsack-pg
spinach pg 9 10: *spinach-knapsack-pg

spinach mysql 0 10: *spinach-knapsack-mysql
spinach mysql 1 10: *spinach-knapsack-mysql
spinach mysql 2 10: *spinach-knapsack-mysql
spinach mysql 3 10: *spinach-knapsack-mysql
spinach mysql 4 10: *spinach-knapsack-mysql
spinach mysql 5 10: *spinach-knapsack-mysql
spinach mysql 6 10: *spinach-knapsack-mysql
spinach mysql 7 10: *spinach-knapsack-mysql
spinach mysql 8 10: *spinach-knapsack-mysql
spinach mysql 9 10: *spinach-knapsack-mysql
248 249 250 251 252 253 254 255 256 257

# Other generic tests
.ruby-static-analysis: &ruby-static-analysis
  variables:
    SIMPLECOV: "false"
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"

.exec: &exec
  <<: *ruby-static-analysis
258
  <<: *dedicated-runner
259
  <<: *except-docs
260 261
  stage: test
  script:
262
    - bundle exec $CI_JOB_NAME
263

264
rubocop:
265 266
  <<: *ruby-static-analysis
  <<: *dedicated-runner
267
  <<: *except-docs
268 269 270 271
  stage: test
  script:
    - bundle exec "rubocop --require rubocop-rspec"

272 273
rake haml_lint: *exec
rake scss_lint: *exec
274
rake config_lint: *exec
275 276 277
rake brakeman: *exec
rake flay: *exec
license_finder: *exec
278 279 280 281 282 283
rake downtime_check:
  <<: *exec
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
284
    - /^docs\/*/
285

286 287 288 289 290 291 292 293 294 295
rake ee_compat_check:
  <<: *exec
  only:
    - branches@gitlab-org/gitlab-ce
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
  allow_failure: yes
  cache:
296
    key: "ee_compat_check_repo"
297
    paths:
298
      - ee_compat_check/ee-repo/
299
  artifacts:
300
    name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
301 302 303 304 305
    when: on_failure
    expire_in: 10d
    paths:
      - ee_compat_check/patches/*.patch

306
.db-migrate-reset: &db-migrate-reset
307
  stage: test
308
  <<: *dedicated-runner
309
  script:
310
    - bundle exec rake db:migrate:reset
311

312
rake pg db:migrate:reset:
313 314
  <<: *db-migrate-reset
  <<: *use-pg
315
  <<: *except-docs
316

317
rake mysql db:migrate:reset:
318 319
  <<: *db-migrate-reset
  <<: *use-mysql
320
  <<: *except-docs
321 322

.db-rollback: &db-rollback
323 324 325
  stage: test
  <<: *dedicated-runner
  script:
326 327
    - bundle exec rake db:rollback STEP=120
    - bundle exec rake db:migrate
328

329
rake pg db:rollback:
330 331
  <<: *db-rollback
  <<: *use-pg
332
  <<: *except-docs
333

334
rake mysql db:rollback:
335 336
  <<: *db-rollback
  <<: *use-mysql
337
  <<: *except-docs
338 339

.db-seed_fu: &db-seed_fu
340
  stage: test
341
  <<: *dedicated-runner
342 343 344 345 346 347 348 349 350 351 352 353 354 355
  variables:
    SIZE: "1"
    SETUP_DB: "false"
    RAILS_ENV: "development"
  script:
    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
       /home/git/repositories/gitlab-org/gitlab-test.git
    - bundle exec rake db:setup db:seed_fu
  artifacts:
    when: on_failure
    expire_in: 1d
    paths:
      - log/development.log

356
rake pg db:seed_fu:
357 358
  <<: *db-seed_fu
  <<: *use-pg
359
  <<: *except-docs
360

361
rake mysql db:seed_fu:
362 363
  <<: *db-seed_fu
  <<: *use-mysql
364
  <<: *except-docs
365

366 367 368
rake gitlab:assets:compile:
  stage: test
  <<: *dedicated-runner
369
  <<: *except-docs
370 371 372 373 374 375 376
  dependencies: []
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"
    SETUP_DB: "false"
    USE_DB: "false"
    SKIP_STORAGE_VALIDATION: "true"
377 378 379 380 381 382 383 384
    WEBPACK_REPORT: "true"
  script:
    - bundle exec rake yarn:install gitlab:assets:compile
  artifacts:
    name: webpack-report
    expire_in: 31d
    paths:
    - webpack-report/
385

Clement Ho's avatar
Clement Ho committed
386
rake karma:
387 388 389
  cache:
    paths:
      - vendor/ruby
390
  stage: test
391
  <<: *use-pg
392
  <<: *dedicated-runner
393
  <<: *except-docs
394 395
  variables:
    BABEL_ENV: "coverage"
396
  script:
397
    - bundle exec rake karma
398
  coverage: '/^Statements *: (\d+\.\d+%)/'
399 400 401 402
  artifacts:
    name: coverage-javascript
    expire_in: 31d
    paths:
403
    - coverage-javascript/
404

405 406
docs:check:apilint:
  image: "phusion/baseimage"
407
  stage: test
408
  <<: *dedicated-runner
409 410
  cache: {}
  dependencies: []
411 412 413 414
  before_script: []
  script:
    - scripts/lint-doc.sh

415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
docs:check:links:
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
  stage: test
  <<: *dedicated-runner
  cache: {}
  dependencies: []
  before_script: []
  script:
    - mv doc/ /nanoc/content/
    - cd /nanoc
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links

430 431 432
bundler:audit:
  stage: test
  <<: *ruby-static-analysis
433
  <<: *dedicated-runner
434 435 436 437 438 439
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
  script:
440
    - "bundle exec bundle-audit check --update --ignore CVE-2016-4658"
441

442
.migration-paths: &migration-paths
443
  stage: test
444
  <<: *dedicated-runner
445 446
  variables:
    SETUP_DB: "false"
447 448 449 450 451
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
452
  script:
453
    - git fetch origin v8.14.10
Kamil Trzcinski's avatar
Kamil Trzcinski committed
454
    - git checkout -f FETCH_HEAD
455
    - bundle install $BUNDLE_INSTALL_FLAGS
456
    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
457
    - git checkout $CI_COMMIT_SHA
458
    - bundle install $BUNDLE_INSTALL_FLAGS
459
    - . scripts/prepare_build.sh
460
    - bundle exec rake db:migrate
461

462
migration pg paths:
463 464 465
  <<: *migration-paths
  <<: *use-pg

466
migration mysql paths:
467 468 469
  <<: *migration-paths
  <<: *use-mysql

470 471 472
coverage:
  stage: post-test
  services: []
473
  <<: *dedicated-runner
474
  <<: *except-docs
475 476 477 478 479
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"
  script:
    - bundle exec scripts/merge-simplecov
480
  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
481 482 483 484 485 486 487
  artifacts:
    name: coverage
    expire_in: 31d
    paths:
    - coverage/index.html
    - coverage/assets/

488
lint:javascript:
489
  <<: *dedicated-runner
490
  <<: *except-docs
491
  stage: test
492
  before_script: []
493
  script:
494
    - yarn run eslint
495 496

lint:javascript:report:
497
  <<: *dedicated-runner
498
  <<: *except-docs
499
  stage: post-test
500
  before_script: []
501
  script:
502
    - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files
503
    - yarn run eslint-report || true # ignore exit code
504 505 506 507 508
  artifacts:
    name: eslint-report
    expire_in: 31d
    paths:
    - eslint-report.html
509 510 511 512 513 514

# Trigger docs build
# https://gitlab.com/gitlab-com/doc-gitlab-com/blob/master/README.md#deployment-process
trigger_docs:
  stage: post-test
  image: "alpine"
515
  <<: *dedicated-runner
516 517 518
  before_script:
    - apk update && apk add curl
  variables:
519
    GIT_STRATEGY: "none"
520 521 522
  cache: {}
  artifacts: {}
  script:
523 524
    - "HTTP_STATUS=$(curl -X POST -F token=${DOCS_TRIGGER_TOKEN} -F ref=master -F variables[PROJECT]=${CI_PROJECT_NAME} --silent --output curl.log --write-out '%{http_code}' https://gitlab.com/api/v3/projects/1794617/trigger/builds)"
    - if [ "${HTTP_STATUS}" -ne "201" ]; then echo "Error ${HTTP_STATUS}"; cat curl.log; echo; exit 1; fi
525 526
  only:
    - master@gitlab-org/gitlab-ce
527
    - master@gitlab-org/gitlab-ee
528 529 530 531

# Notify slack in the end
notify:slack:
  stage: post-test
532
  <<: *dedicated-runner
533 534 535 536
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
  script:
537
    - ./scripts/notify_slack.sh "#development" "Build on \`$CI_COMMIT_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_COMMIT_SHA"/pipelines>"
538 539 540 541 542 543 544 545 546 547
  when: on_failure
  only:
    - master@gitlab-org/gitlab-ce
    - tags@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - tags@gitlab-org/gitlab-ee

pages:
  before_script: []
  stage: pages
548
  <<: *dedicated-runner
549 550
  dependencies:
    - coverage
Clement Ho's avatar
Clement Ho committed
551
    - rake karma
552
    - rake gitlab:assets:compile
553
    - lint:javascript:report
554 555 556
  script:
    - mv public/ .public/
    - mkdir public/
557
    - mv coverage/ public/coverage-ruby/ || true
558
    - mv coverage-javascript/ public/coverage-javascript/ || true
559
    - mv eslint-report.html public/ || true
560
    - mv webpack-report/ public/webpack-report/ || true
561 562 563 564
  artifacts:
    paths:
      - public
  only:
565
    - master@gitlab-org/gitlab-ce
566
    - master@gitlab-org/gitlab-ee
567 568 569 570

# Insurance in case a gem needed by one of our releases gets yanked from
# rubygems.org in the future.
cache gems:
571
  <<: *dedicated-runner
572 573 574 575 576 577 578 579 580
  only:
    - tags
  variables:
    SETUP_DB: "false"
  script:
    - bundle package --all --all-platforms
  artifacts:
    paths:
      - vendor/cache
581 582
  only:
    - master@gitlab-org/gitlab-ce
583
    - master@gitlab-org/gitlab-ee