.gitlab-ci.yml 15 KB
Newer Older
1
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
2

3
cache:
Z.J. van de Weg's avatar
Z.J. van de Weg committed
4
  key: "ruby-233"
5
  paths:
6
  - vendor/ruby
7

8 9
variables:
  MYSQL_ALLOW_EMPTY_PASSWORD: "1"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
10
  RAILS_ENV: "test"
Mike Greiling's avatar
Mike Greiling committed
11
  NODE_ENV: "test"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
12
  SIMPLECOV: "true"
13
  GIT_DEPTH: "20"
14
  GIT_SUBMODULE_STRATEGY: "none"
15
  PHANTOMJS_VERSION: "2.1.1"
16
  GET_SOURCES_ATTEMPTS: "3"
17 18
  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
  KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json
19

Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
20
before_script:
Kamil Trzcinski's avatar
Test  
Kamil Trzcinski committed
21
  - bundle --version
22 23
  - . scripts/utils.sh
  - ./scripts/prepare_build.sh
24

Kamil Trzcinski's avatar
Kamil Trzcinski committed
25
stages:
Kamil Trzcinski's avatar
Kamil Trzcinski committed
26
- prepare
Kamil Trzcinski's avatar
Kamil Trzcinski committed
27
- test
28
- post-test
29
- pages
30

31 32 33 34 35
# Predefined scopes
.dedicated-runner: &dedicated-runner
  tags:
    - gitlab-org

36 37 38 39 40
.knapsack-state: &knapsack-state
  services: []
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
41
    KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
42 43 44
  cache:
    key: "knapsack"
    paths:
45
      - knapsack/
46 47 48
  artifacts:
    expire_in: 31d
    paths:
49
      - knapsack/
50

51 52 53 54 55 56
.use-pg: &use-pg
  services:
    - postgres:latest
    - redis:alpine

.use-mysql: &use-mysql
57 58 59
  services:
    - mysql:latest
    - redis:alpine
60

61 62
.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
  only:
63
    - /mysql/
64 65 66 67 68 69 70
    - master@gitlab-org/gitlab-ce
    - master@gitlab/gitlabhq
    - tags@gitlab-org/gitlab-ce
    - tags@gitlab/gitlabhq
    - //@gitlab-org/gitlab-ee
    - //@gitlab/gitlab-ee

71 72 73 74 75 76
# Skip all jobs except the ones that begin with 'docs/', for commits
# including ONLY doc changes inside the 'doc/` directory.
.except-docs: &except-docs
  except:
    - /^docs\/*/

77 78
.rspec-knapsack: &rspec-knapsack
  stage: test
79
  <<: *dedicated-runner
80
  script:
81
    - JOB_NAME=( $CI_JOB_NAME )
82 83
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
84
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
85
    - export KNAPSACK_GENERATE_REPORT=true
86
    - export CACHE_CLASSES=true
87
    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
88 89 90
    - knapsack rspec "--color --format documentation"
  artifacts:
    expire_in: 31d
91
    when: always
92
    paths:
93 94 95
      - coverage/
      - knapsack/
      - tmp/capybara/
96

97 98 99
.rspec-knapsack-pg: &rspec-knapsack-pg
  <<: *rspec-knapsack
  <<: *use-pg
100
  <<: *except-docs
101 102 103 104

.rspec-knapsack-mysql: &rspec-knapsack-mysql
  <<: *rspec-knapsack
  <<: *use-mysql
105
  <<: *only-master-and-ee-or-mysql
106
  <<: *except-docs
107

108 109
.spinach-knapsack: &spinach-knapsack
  stage: test
110
  <<: *dedicated-runner
111
  script:
112
    - JOB_NAME=( $CI_JOB_NAME )
113 114
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
115
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
116
    - export KNAPSACK_GENERATE_REPORT=true
117
    - export CACHE_CLASSES=true
118
    - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
119 120 121
    - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
  artifacts:
    expire_in: 31d
122
    when: always
123
    paths:
124 125 126
      - coverage/
      - knapsack/
      - tmp/capybara/
127

128 129 130
.spinach-knapsack-pg: &spinach-knapsack-pg
  <<: *spinach-knapsack
  <<: *use-pg
131
  <<: *except-docs
132 133 134 135

.spinach-knapsack-mysql: &spinach-knapsack-mysql
  <<: *spinach-knapsack
  <<: *use-mysql
136
  <<: *only-master-and-ee-or-mysql
137
  <<: *except-docs
138

139 140 141 142
# Prepare and merge knapsack tests
knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
143
  <<: *except-docs
144 145
  stage: prepare
  script:
146
    - mkdir -p knapsack/${CI_PROJECT_NAME}/
147 148
    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
    - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH
149 150 151 152 153 154 155 156
    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
    - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'

update-knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: post-test
  script:
157 158
    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json
    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json
159 160 161 162 163 164 165
    - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
166 167

setup-test-env:
168
  <<: *use-pg
169
  <<: *dedicated-runner
170
  <<: *except-docs
171 172
  stage: prepare
  script:
173
    - node --version
174
    - yarn install --pure-lockfile
Mike Greiling's avatar
Mike Greiling committed
175
    - bundle exec rake gitlab:assets:compile
176 177 178 179
    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
  artifacts:
    expire_in: 7d
    paths:
180
      - node_modules
181 182 183
      - public/assets
      - tmp/tests

184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
rspec pg 0 20: *rspec-knapsack-pg
rspec pg 1 20: *rspec-knapsack-pg
rspec pg 2 20: *rspec-knapsack-pg
rspec pg 3 20: *rspec-knapsack-pg
rspec pg 4 20: *rspec-knapsack-pg
rspec pg 5 20: *rspec-knapsack-pg
rspec pg 6 20: *rspec-knapsack-pg
rspec pg 7 20: *rspec-knapsack-pg
rspec pg 8 20: *rspec-knapsack-pg
rspec pg 9 20: *rspec-knapsack-pg
rspec pg 10 20: *rspec-knapsack-pg
rspec pg 11 20: *rspec-knapsack-pg
rspec pg 12 20: *rspec-knapsack-pg
rspec pg 13 20: *rspec-knapsack-pg
rspec pg 14 20: *rspec-knapsack-pg
rspec pg 15 20: *rspec-knapsack-pg
rspec pg 16 20: *rspec-knapsack-pg
rspec pg 17 20: *rspec-knapsack-pg
rspec pg 18 20: *rspec-knapsack-pg
rspec pg 19 20: *rspec-knapsack-pg

rspec mysql 0 20: *rspec-knapsack-mysql
rspec mysql 1 20: *rspec-knapsack-mysql
rspec mysql 2 20: *rspec-knapsack-mysql
rspec mysql 3 20: *rspec-knapsack-mysql
rspec mysql 4 20: *rspec-knapsack-mysql
rspec mysql 5 20: *rspec-knapsack-mysql
rspec mysql 6 20: *rspec-knapsack-mysql
rspec mysql 7 20: *rspec-knapsack-mysql
rspec mysql 8 20: *rspec-knapsack-mysql
rspec mysql 9 20: *rspec-knapsack-mysql
rspec mysql 10 20: *rspec-knapsack-mysql
rspec mysql 11 20: *rspec-knapsack-mysql
rspec mysql 12 20: *rspec-knapsack-mysql
rspec mysql 13 20: *rspec-knapsack-mysql
rspec mysql 14 20: *rspec-knapsack-mysql
rspec mysql 15 20: *rspec-knapsack-mysql
rspec mysql 16 20: *rspec-knapsack-mysql
rspec mysql 17 20: *rspec-knapsack-mysql
rspec mysql 18 20: *rspec-knapsack-mysql
rspec mysql 19 20: *rspec-knapsack-mysql

spinach pg 0 10: *spinach-knapsack-pg
spinach pg 1 10: *spinach-knapsack-pg
spinach pg 2 10: *spinach-knapsack-pg
spinach pg 3 10: *spinach-knapsack-pg
spinach pg 4 10: *spinach-knapsack-pg
spinach pg 5 10: *spinach-knapsack-pg
spinach pg 6 10: *spinach-knapsack-pg
spinach pg 7 10: *spinach-knapsack-pg
spinach pg 8 10: *spinach-knapsack-pg
spinach pg 9 10: *spinach-knapsack-pg

spinach mysql 0 10: *spinach-knapsack-mysql
spinach mysql 1 10: *spinach-knapsack-mysql
spinach mysql 2 10: *spinach-knapsack-mysql
spinach mysql 3 10: *spinach-knapsack-mysql
spinach mysql 4 10: *spinach-knapsack-mysql
spinach mysql 5 10: *spinach-knapsack-mysql
spinach mysql 6 10: *spinach-knapsack-mysql
spinach mysql 7 10: *spinach-knapsack-mysql
spinach mysql 8 10: *spinach-knapsack-mysql
spinach mysql 9 10: *spinach-knapsack-mysql
247 248 249 250 251 252 253 254 255 256

# Other generic tests
.ruby-static-analysis: &ruby-static-analysis
  variables:
    SIMPLECOV: "false"
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"

.exec: &exec
  <<: *ruby-static-analysis
257
  <<: *dedicated-runner
258
  <<: *except-docs
259 260
  stage: test
  script:
261
    - bundle exec $CI_JOB_NAME
262

263
rubocop:
264 265
  <<: *ruby-static-analysis
  <<: *dedicated-runner
266
  <<: *except-docs
267 268 269 270
  stage: test
  script:
    - bundle exec "rubocop --require rubocop-rspec"

271 272
rake haml_lint: *exec
rake scss_lint: *exec
273
rake config_lint: *exec
274 275 276
rake brakeman: *exec
rake flay: *exec
license_finder: *exec
277 278 279 280 281 282
rake downtime_check:
  <<: *exec
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
283
    - /^docs\/*/
284

285 286 287 288 289 290 291 292 293 294
rake ee_compat_check:
  <<: *exec
  only:
    - branches@gitlab-org/gitlab-ce
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
  allow_failure: yes
  cache:
295
    key: "ee_compat_check_repo"
296
    paths:
297
      - ee_compat_check/ee-repo/
298
  artifacts:
299
    name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
300 301 302 303 304
    when: on_failure
    expire_in: 10d
    paths:
      - ee_compat_check/patches/*.patch

305
.db-migrate-reset: &db-migrate-reset
306
  stage: test
307
  <<: *dedicated-runner
308
  script:
309
    - bundle exec rake db:migrate:reset
310

311
rake pg db:migrate:reset:
312 313
  <<: *db-migrate-reset
  <<: *use-pg
314
  <<: *except-docs
315

316
rake mysql db:migrate:reset:
317 318
  <<: *db-migrate-reset
  <<: *use-mysql
319
  <<: *except-docs
320 321

.db-rollback: &db-rollback
322 323 324
  stage: test
  <<: *dedicated-runner
  script:
325 326
    - bundle exec rake db:rollback STEP=120
    - bundle exec rake db:migrate
327

328
rake pg db:rollback:
329 330
  <<: *db-rollback
  <<: *use-pg
331
  <<: *except-docs
332

333
rake mysql db:rollback:
334 335
  <<: *db-rollback
  <<: *use-mysql
336
  <<: *except-docs
337 338

.db-seed_fu: &db-seed_fu
339
  stage: test
340
  <<: *dedicated-runner
341 342 343 344 345 346 347 348 349 350 351 352 353 354
  variables:
    SIZE: "1"
    SETUP_DB: "false"
    RAILS_ENV: "development"
  script:
    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
       /home/git/repositories/gitlab-org/gitlab-test.git
    - bundle exec rake db:setup db:seed_fu
  artifacts:
    when: on_failure
    expire_in: 1d
    paths:
      - log/development.log

355
rake pg db:seed_fu:
356 357
  <<: *db-seed_fu
  <<: *use-pg
358
  <<: *except-docs
359

360
rake mysql db:seed_fu:
361 362
  <<: *db-seed_fu
  <<: *use-mysql
363
  <<: *except-docs
364

365 366 367
rake gitlab:assets:compile:
  stage: test
  <<: *dedicated-runner
368
  <<: *except-docs
369 370 371 372 373 374 375
  dependencies: []
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"
    SETUP_DB: "false"
    USE_DB: "false"
    SKIP_STORAGE_VALIDATION: "true"
376 377 378 379 380 381 382 383
    WEBPACK_REPORT: "true"
  script:
    - bundle exec rake yarn:install gitlab:assets:compile
  artifacts:
    name: webpack-report
    expire_in: 31d
    paths:
    - webpack-report/
384

Clement Ho's avatar
Clement Ho committed
385
rake karma:
386 387 388
  cache:
    paths:
      - vendor/ruby
389
  stage: test
390
  <<: *use-pg
391
  <<: *dedicated-runner
392
  <<: *except-docs
393 394
  variables:
    BABEL_ENV: "coverage"
395
  script:
396
    - bundle exec rake karma
397
  coverage: '/^Statements *: (\d+\.\d+%)/'
398 399 400 401
  artifacts:
    name: coverage-javascript
    expire_in: 31d
    paths:
402
    - coverage-javascript/
403

404 405
docs:check:apilint:
  image: "phusion/baseimage"
406
  stage: test
407
  <<: *dedicated-runner
408 409
  cache: {}
  dependencies: []
410 411 412 413
  before_script: []
  script:
    - scripts/lint-doc.sh

414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
docs:check:links:
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
  stage: test
  <<: *dedicated-runner
  cache: {}
  dependencies: []
  before_script: []
  script:
    - mv doc/ /nanoc/content/
    - cd /nanoc
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links

429 430 431
bundler:audit:
  stage: test
  <<: *ruby-static-analysis
432
  <<: *dedicated-runner
433 434 435 436 437 438
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
  script:
439
    - "bundle exec bundle-audit check --update --ignore CVE-2016-4658"
440

441
.migration-paths: &migration-paths
442
  stage: test
443
  <<: *dedicated-runner
444 445
  variables:
    SETUP_DB: "false"
446 447 448 449 450
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
451
  script:
452
    - git fetch origin v8.14.10
Kamil Trzcinski's avatar
Kamil Trzcinski committed
453
    - git checkout -f FETCH_HEAD
454
    - bundle install $BUNDLE_INSTALL_FLAGS
455
    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
456
    - git checkout $CI_COMMIT_SHA
457
    - bundle install $BUNDLE_INSTALL_FLAGS
458
    - . scripts/prepare_build.sh
459
    - bundle exec rake db:migrate
460

461
migration pg paths:
462 463 464
  <<: *migration-paths
  <<: *use-pg

465
migration mysql paths:
466 467 468
  <<: *migration-paths
  <<: *use-mysql

469 470 471
coverage:
  stage: post-test
  services: []
472
  <<: *dedicated-runner
473
  <<: *except-docs
474 475 476 477 478
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"
  script:
    - bundle exec scripts/merge-simplecov
479
  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
480 481 482 483 484 485 486
  artifacts:
    name: coverage
    expire_in: 31d
    paths:
    - coverage/index.html
    - coverage/assets/

487
lint:javascript:
488
  <<: *dedicated-runner
489
  <<: *except-docs
490
  stage: test
491
  before_script: []
492
  script:
493
    - yarn run eslint
494 495

lint:javascript:report:
496
  <<: *dedicated-runner
497
  <<: *except-docs
498
  stage: post-test
499
  before_script: []
500
  script:
501
    - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files
502
    - yarn run eslint-report || true # ignore exit code
503 504 505 506 507
  artifacts:
    name: eslint-report
    expire_in: 31d
    paths:
    - eslint-report.html
508 509 510 511 512 513

# Trigger docs build
# https://gitlab.com/gitlab-com/doc-gitlab-com/blob/master/README.md#deployment-process
trigger_docs:
  stage: post-test
  image: "alpine"
514
  <<: *dedicated-runner
515 516 517
  before_script:
    - apk update && apk add curl
  variables:
518
    GIT_STRATEGY: "none"
519 520 521
  cache: {}
  artifacts: {}
  script:
522 523
    - "HTTP_STATUS=$(curl -X POST -F token=${DOCS_TRIGGER_TOKEN} -F ref=master -F variables[PROJECT]=${CI_PROJECT_NAME} --silent --output curl.log --write-out '%{http_code}' https://gitlab.com/api/v3/projects/1794617/trigger/builds)"
    - if [ "${HTTP_STATUS}" -ne "201" ]; then echo "Error ${HTTP_STATUS}"; cat curl.log; echo; exit 1; fi
524 525
  only:
    - master@gitlab-org/gitlab-ce
526
    - master@gitlab-org/gitlab-ee
527 528 529 530

# Notify slack in the end
notify:slack:
  stage: post-test
531
  <<: *dedicated-runner
532 533 534 535
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
  script:
536
    - ./scripts/notify_slack.sh "#development" "Build on \`$CI_COMMIT_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_COMMIT_SHA"/pipelines>"
537 538 539 540 541 542 543 544 545 546
  when: on_failure
  only:
    - master@gitlab-org/gitlab-ce
    - tags@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - tags@gitlab-org/gitlab-ee

pages:
  before_script: []
  stage: pages
547
  <<: *dedicated-runner
548 549
  dependencies:
    - coverage
Clement Ho's avatar
Clement Ho committed
550
    - rake karma
551
    - rake gitlab:assets:compile
552
    - lint:javascript:report
553 554 555
  script:
    - mv public/ .public/
    - mkdir public/
556
    - mv coverage/ public/coverage-ruby/ || true
557
    - mv coverage-javascript/ public/coverage-javascript/ || true
558
    - mv eslint-report.html public/ || true
559
    - mv webpack-report/ public/webpack-report/ || true
560 561 562 563
  artifacts:
    paths:
      - public
  only:
564
    - master@gitlab-org/gitlab-ce
565
    - master@gitlab-org/gitlab-ee
566 567 568 569

# Insurance in case a gem needed by one of our releases gets yanked from
# rubygems.org in the future.
cache gems:
570
  <<: *dedicated-runner
571 572 573 574 575 576 577 578 579
  only:
    - tags
  variables:
    SETUP_DB: "false"
  script:
    - bundle package --all --all-platforms
  artifacts:
    paths:
      - vendor/cache
580 581
  only:
    - master@gitlab-org/gitlab-ce
582
    - master@gitlab-org/gitlab-ee