.gitlab-ci.yml 14.8 KB
Newer Older
1
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
2

3
cache:
Z.J. van de Weg's avatar
Z.J. van de Weg committed
4
  key: "ruby-233"
5
  paths:
6
  - vendor/ruby
7

8 9
variables:
  MYSQL_ALLOW_EMPTY_PASSWORD: "1"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
10
  RAILS_ENV: "test"
Mike Greiling's avatar
Mike Greiling committed
11
  NODE_ENV: "test"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
12
  SIMPLECOV: "true"
13
  GIT_DEPTH: "20"
14
  GIT_SUBMODULE_STRATEGY: "none"
15
  PHANTOMJS_VERSION: "2.1.1"
16
  GET_SOURCES_ATTEMPTS: "3"
17 18
  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
  KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json
19

Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
20
before_script:
Kamil Trzcinski's avatar
Test  
Kamil Trzcinski committed
21
  - bundle --version
22 23
  - source scripts/utils.sh
  - source scripts/prepare_build.sh
24

Kamil Trzcinski's avatar
Kamil Trzcinski committed
25
stages:
Kamil Trzcinski's avatar
Kamil Trzcinski committed
26
- prepare
Kamil Trzcinski's avatar
Kamil Trzcinski committed
27
- test
28
- post-test
29
- pages
30

31 32 33 34 35
# Predefined scopes
.dedicated-runner: &dedicated-runner
  tags:
    - gitlab-org

36 37 38 39 40
.knapsack-state: &knapsack-state
  services: []
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
41
    KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
42 43 44
  cache:
    key: "knapsack"
    paths:
45
      - knapsack/
46 47 48
  artifacts:
    expire_in: 31d
    paths:
49
      - knapsack/
50

51 52 53 54 55 56
.use-pg: &use-pg
  services:
    - postgres:latest
    - redis:alpine

.use-mysql: &use-mysql
57 58 59
  services:
    - mysql:latest
    - redis:alpine
60

61 62
.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
  only:
63
    - /mysql/
64 65 66 67 68 69 70
    - master@gitlab-org/gitlab-ce
    - master@gitlab/gitlabhq
    - tags@gitlab-org/gitlab-ce
    - tags@gitlab/gitlabhq
    - //@gitlab-org/gitlab-ee
    - //@gitlab/gitlab-ee

71 72 73
# Skip all jobs except the ones that begin with 'docs/'.
# Used for commits including ONLY documentation changes.
# https://docs.gitlab.com/ce/development/writing_documentation.html#testing
74 75
.except-docs: &except-docs
  except:
76
    - /^docs\/.*/
77

78 79
.rspec-knapsack: &rspec-knapsack
  stage: test
80
  <<: *dedicated-runner
81
  script:
82
    - JOB_NAME=( $CI_JOB_NAME )
83 84
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
85
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
86
    - export KNAPSACK_GENERATE_REPORT=true
87
    - export CACHE_CLASSES=true
88
    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
89 90 91
    - knapsack rspec "--color --format documentation"
  artifacts:
    expire_in: 31d
92
    when: always
93
    paths:
94 95 96
      - coverage/
      - knapsack/
      - tmp/capybara/
97

98 99 100
.rspec-knapsack-pg: &rspec-knapsack-pg
  <<: *rspec-knapsack
  <<: *use-pg
101
  <<: *except-docs
102 103 104 105

.rspec-knapsack-mysql: &rspec-knapsack-mysql
  <<: *rspec-knapsack
  <<: *use-mysql
106
  <<: *only-master-and-ee-or-mysql
107
  <<: *except-docs
108

109 110
.spinach-knapsack: &spinach-knapsack
  stage: test
111
  <<: *dedicated-runner
112
  script:
113
    - JOB_NAME=( $CI_JOB_NAME )
114 115
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
116
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
117
    - export KNAPSACK_GENERATE_REPORT=true
118
    - export CACHE_CLASSES=true
119
    - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
120 121 122
    - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
  artifacts:
    expire_in: 31d
123
    when: always
124
    paths:
125 126 127
      - coverage/
      - knapsack/
      - tmp/capybara/
128

129 130 131
.spinach-knapsack-pg: &spinach-knapsack-pg
  <<: *spinach-knapsack
  <<: *use-pg
132
  <<: *except-docs
133 134 135 136

.spinach-knapsack-mysql: &spinach-knapsack-mysql
  <<: *spinach-knapsack
  <<: *use-mysql
137
  <<: *only-master-and-ee-or-mysql
138
  <<: *except-docs
139

140 141 142 143
# Prepare and merge knapsack tests
knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
144
  <<: *except-docs
145 146
  stage: prepare
  script:
147
    - mkdir -p knapsack/${CI_PROJECT_NAME}/
148 149
    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
    - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH
150 151 152 153 154 155 156 157
    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
    - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'

update-knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: post-test
  script:
158 159
    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json
    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json
160 161 162 163 164 165 166
    - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
167 168

setup-test-env:
169
  <<: *use-pg
170
  <<: *dedicated-runner
171
  <<: *except-docs
172 173
  stage: prepare
  script:
174
    - node --version
175
    - yarn install --pure-lockfile
Mike Greiling's avatar
Mike Greiling committed
176
    - bundle exec rake gitlab:assets:compile
177 178 179 180
    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
  artifacts:
    expire_in: 7d
    paths:
181
      - node_modules
182 183 184
      - public/assets
      - tmp/tests

185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247
rspec pg 0 20: *rspec-knapsack-pg
rspec pg 1 20: *rspec-knapsack-pg
rspec pg 2 20: *rspec-knapsack-pg
rspec pg 3 20: *rspec-knapsack-pg
rspec pg 4 20: *rspec-knapsack-pg
rspec pg 5 20: *rspec-knapsack-pg
rspec pg 6 20: *rspec-knapsack-pg
rspec pg 7 20: *rspec-knapsack-pg
rspec pg 8 20: *rspec-knapsack-pg
rspec pg 9 20: *rspec-knapsack-pg
rspec pg 10 20: *rspec-knapsack-pg
rspec pg 11 20: *rspec-knapsack-pg
rspec pg 12 20: *rspec-knapsack-pg
rspec pg 13 20: *rspec-knapsack-pg
rspec pg 14 20: *rspec-knapsack-pg
rspec pg 15 20: *rspec-knapsack-pg
rspec pg 16 20: *rspec-knapsack-pg
rspec pg 17 20: *rspec-knapsack-pg
rspec pg 18 20: *rspec-knapsack-pg
rspec pg 19 20: *rspec-knapsack-pg

rspec mysql 0 20: *rspec-knapsack-mysql
rspec mysql 1 20: *rspec-knapsack-mysql
rspec mysql 2 20: *rspec-knapsack-mysql
rspec mysql 3 20: *rspec-knapsack-mysql
rspec mysql 4 20: *rspec-knapsack-mysql
rspec mysql 5 20: *rspec-knapsack-mysql
rspec mysql 6 20: *rspec-knapsack-mysql
rspec mysql 7 20: *rspec-knapsack-mysql
rspec mysql 8 20: *rspec-knapsack-mysql
rspec mysql 9 20: *rspec-knapsack-mysql
rspec mysql 10 20: *rspec-knapsack-mysql
rspec mysql 11 20: *rspec-knapsack-mysql
rspec mysql 12 20: *rspec-knapsack-mysql
rspec mysql 13 20: *rspec-knapsack-mysql
rspec mysql 14 20: *rspec-knapsack-mysql
rspec mysql 15 20: *rspec-knapsack-mysql
rspec mysql 16 20: *rspec-knapsack-mysql
rspec mysql 17 20: *rspec-knapsack-mysql
rspec mysql 18 20: *rspec-knapsack-mysql
rspec mysql 19 20: *rspec-knapsack-mysql

spinach pg 0 10: *spinach-knapsack-pg
spinach pg 1 10: *spinach-knapsack-pg
spinach pg 2 10: *spinach-knapsack-pg
spinach pg 3 10: *spinach-knapsack-pg
spinach pg 4 10: *spinach-knapsack-pg
spinach pg 5 10: *spinach-knapsack-pg
spinach pg 6 10: *spinach-knapsack-pg
spinach pg 7 10: *spinach-knapsack-pg
spinach pg 8 10: *spinach-knapsack-pg
spinach pg 9 10: *spinach-knapsack-pg

spinach mysql 0 10: *spinach-knapsack-mysql
spinach mysql 1 10: *spinach-knapsack-mysql
spinach mysql 2 10: *spinach-knapsack-mysql
spinach mysql 3 10: *spinach-knapsack-mysql
spinach mysql 4 10: *spinach-knapsack-mysql
spinach mysql 5 10: *spinach-knapsack-mysql
spinach mysql 6 10: *spinach-knapsack-mysql
spinach mysql 7 10: *spinach-knapsack-mysql
spinach mysql 8 10: *spinach-knapsack-mysql
spinach mysql 9 10: *spinach-knapsack-mysql
248 249 250 251 252 253 254 255

# Other generic tests
.ruby-static-analysis: &ruby-static-analysis
  variables:
    SIMPLECOV: "false"
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"

256
.rake-exec: &rake-exec
257
  <<: *ruby-static-analysis
258
  <<: *dedicated-runner
259
  <<: *except-docs
260 261
  stage: test
  script:
262
    - bundle exec rake $CI_JOB_NAME
263

264
static-analysis:
265 266
  <<: *ruby-static-analysis
  <<: *dedicated-runner
267
  <<: *except-docs
268 269
  stage: test
  script:
270 271 272 273 274 275 276 277
    - bundle exec rake config_lint
    - bundle exec rake flay
    - bundle exec rake haml_lint
    - bundle exec rake scss_lint
    - bundle exec rake brakeman
    - bundle exec license_finder
    - scripts/lint-doc.sh
    - yarn run eslint
278 279
    - bundle exec "rubocop --require rubocop-rspec"

280 281
downtime_check:
  <<: *rake-exec
282 283 284 285
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
286
    - /^docs\/*/
287

288 289
ee_compat_check:
  <<: *rake-exec
290 291 292 293 294 295 296 297
  only:
    - branches@gitlab-org/gitlab-ce
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
  allow_failure: yes
  cache:
298
    key: "ee_compat_check_repo"
299
    paths:
300
      - ee_compat_check/ee-repo/
301
  artifacts:
302
    name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
303 304 305 306 307
    when: on_failure
    expire_in: 10d
    paths:
      - ee_compat_check/patches/*.patch

308
.db-migrate-reset: &db-migrate-reset
309
  stage: test
310
  <<: *dedicated-runner
311
  script:
312
    - bundle exec rake db:migrate:reset
313

314
db:migrate:reset pg:
315 316
  <<: *db-migrate-reset
  <<: *use-pg
317
  <<: *except-docs
318

319
db:migrate:reset mysql:
320 321
  <<: *db-migrate-reset
  <<: *use-mysql
322
  <<: *except-docs
323 324

.db-rollback: &db-rollback
325 326 327
  stage: test
  <<: *dedicated-runner
  script:
328 329
    - bundle exec rake db:rollback STEP=120
    - bundle exec rake db:migrate
330

331
db:rollback pg:
332 333
  <<: *db-rollback
  <<: *use-pg
334
  <<: *except-docs
335

336
db:rollback mysql:
337 338
  <<: *db-rollback
  <<: *use-mysql
339
  <<: *except-docs
340 341

.db-seed_fu: &db-seed_fu
342
  stage: test
343
  <<: *dedicated-runner
344 345 346 347 348 349 350 351 352 353 354 355 356 357
  variables:
    SIZE: "1"
    SETUP_DB: "false"
    RAILS_ENV: "development"
  script:
    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
       /home/git/repositories/gitlab-org/gitlab-test.git
    - bundle exec rake db:setup db:seed_fu
  artifacts:
    when: on_failure
    expire_in: 1d
    paths:
      - log/development.log

358
db:seed_fu pg:
359 360
  <<: *db-seed_fu
  <<: *use-pg
361
  <<: *except-docs
362

363
db:seed_fu mysql:
364 365
  <<: *db-seed_fu
  <<: *use-mysql
366
  <<: *except-docs
367

368
gitlab:assets:compile:
369 370
  stage: test
  <<: *dedicated-runner
371
  <<: *except-docs
372 373 374 375 376 377 378
  dependencies: []
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"
    SETUP_DB: "false"
    USE_DB: "false"
    SKIP_STORAGE_VALIDATION: "true"
379 380 381 382 383 384 385 386
    WEBPACK_REPORT: "true"
  script:
    - bundle exec rake yarn:install gitlab:assets:compile
  artifacts:
    name: webpack-report
    expire_in: 31d
    paths:
    - webpack-report/
387

388
karma:
389 390 391
  cache:
    paths:
      - vendor/ruby
392
  stage: test
393
  <<: *use-pg
394
  <<: *dedicated-runner
395
  <<: *except-docs
396 397
  variables:
    BABEL_ENV: "coverage"
398
  script:
399
    - bundle exec rake karma
400
  coverage: '/^Statements *: (\d+\.\d+%)/'
401 402 403 404
  artifacts:
    name: coverage-javascript
    expire_in: 31d
    paths:
405
    - coverage-javascript/
406

407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
docs:check:links:
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
  stage: test
  <<: *dedicated-runner
  cache: {}
  dependencies: []
  before_script: []
  script:
    - mv doc/ /nanoc/content/
    - cd /nanoc
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links

422 423 424
bundler:audit:
  stage: test
  <<: *ruby-static-analysis
425
  <<: *dedicated-runner
426 427 428 429 430 431
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
  script:
432
    - "bundle exec bundle-audit check --update --ignore CVE-2016-4658"
433

434
.migration-paths: &migration-paths
435
  stage: test
436
  <<: *dedicated-runner
437 438
  variables:
    SETUP_DB: "false"
439 440 441 442 443
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
444
  script:
445
    - git fetch origin v8.14.10
Kamil Trzcinski's avatar
Kamil Trzcinski committed
446
    - git checkout -f FETCH_HEAD
447
    - bundle install $BUNDLE_INSTALL_FLAGS
448
    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
449
    - git checkout $CI_COMMIT_SHA
450
    - bundle install $BUNDLE_INSTALL_FLAGS
451
    - . scripts/prepare_build.sh
452
    - bundle exec rake db:migrate
453

454
migration path pg:
455 456 457
  <<: *migration-paths
  <<: *use-pg

458
migration path mysql:
459 460 461
  <<: *migration-paths
  <<: *use-mysql

462 463 464
coverage:
  stage: post-test
  services: []
465
  <<: *dedicated-runner
466
  <<: *except-docs
467 468 469 470 471
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"
  script:
    - bundle exec scripts/merge-simplecov
472
  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
473 474 475 476 477 478 479
  artifacts:
    name: coverage
    expire_in: 31d
    paths:
    - coverage/index.html
    - coverage/assets/

480
lint:javascript:report:
481
  <<: *dedicated-runner
482
  <<: *except-docs
483
  stage: post-test
484
  before_script: []
485
  script:
486
    - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files
487
    - yarn run eslint-report || true # ignore exit code
488 489 490 491 492
  artifacts:
    name: eslint-report
    expire_in: 31d
    paths:
    - eslint-report.html
493 494 495 496 497 498

# Trigger docs build
# https://gitlab.com/gitlab-com/doc-gitlab-com/blob/master/README.md#deployment-process
trigger_docs:
  stage: post-test
  image: "alpine"
499
  <<: *dedicated-runner
500 501 502
  before_script:
    - apk update && apk add curl
  variables:
503
    GIT_STRATEGY: "none"
504 505 506
  cache: {}
  artifacts: {}
  script:
507 508
    - "HTTP_STATUS=$(curl -X POST -F token=${DOCS_TRIGGER_TOKEN} -F ref=master -F variables[PROJECT]=${CI_PROJECT_NAME} --silent --output curl.log --write-out '%{http_code}' https://gitlab.com/api/v3/projects/1794617/trigger/builds)"
    - if [ "${HTTP_STATUS}" -ne "201" ]; then echo "Error ${HTTP_STATUS}"; cat curl.log; echo; exit 1; fi
509 510
  only:
    - master@gitlab-org/gitlab-ce
511
    - master@gitlab-org/gitlab-ee
512 513 514 515

# Notify slack in the end
notify:slack:
  stage: post-test
516
  <<: *dedicated-runner
517 518 519 520
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
  script:
521
    - ./scripts/notify_slack.sh "#development" "Build on \`$CI_COMMIT_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_COMMIT_SHA"/pipelines>"
522 523 524 525 526 527 528 529 530 531
  when: on_failure
  only:
    - master@gitlab-org/gitlab-ce
    - tags@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - tags@gitlab-org/gitlab-ee

pages:
  before_script: []
  stage: pages
532
  <<: *dedicated-runner
533 534
  dependencies:
    - coverage
535 536
    - karma
    - gitlab:assets:compile
537
    - lint:javascript:report
538 539 540
  script:
    - mv public/ .public/
    - mkdir public/
541
    - mv coverage/ public/coverage-ruby/ || true
542
    - mv coverage-javascript/ public/coverage-javascript/ || true
543
    - mv eslint-report.html public/ || true
544
    - mv webpack-report/ public/webpack-report/ || true
545 546 547 548
  artifacts:
    paths:
      - public
  only:
549
    - master@gitlab-org/gitlab-ce
550
    - master@gitlab-org/gitlab-ee
551 552 553 554

# Insurance in case a gem needed by one of our releases gets yanked from
# rubygems.org in the future.
cache gems:
555
  <<: *dedicated-runner
556 557 558 559 560 561 562 563 564
  only:
    - tags
  variables:
    SETUP_DB: "false"
  script:
    - bundle package --all --all-platforms
  artifacts:
    paths:
      - vendor/cache
565 566
  only:
    - master@gitlab-org/gitlab-ce
567
    - master@gitlab-org/gitlab-ee