.gitlab-ci.yml 14.5 KB
Newer Older
1
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
2

3
cache:
Z.J. van de Weg's avatar
Z.J. van de Weg committed
4
  key: "ruby-233"
5
  paths:
6
  - vendor/ruby
7

8 9
variables:
  MYSQL_ALLOW_EMPTY_PASSWORD: "1"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
10
  RAILS_ENV: "test"
Mike Greiling's avatar
Mike Greiling committed
11
  NODE_ENV: "test"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
12
  SIMPLECOV: "true"
13
  GIT_DEPTH: "20"
14
  PHANTOMJS_VERSION: "2.1.1"
15
  GET_SOURCES_ATTEMPTS: "3"
16 17
  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
  KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json
18

Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
19
before_script:
Kamil Trzcinski's avatar
Test  
Kamil Trzcinski committed
20
  - bundle --version
21 22
  - . scripts/utils.sh
  - ./scripts/prepare_build.sh
23

Kamil Trzcinski's avatar
Kamil Trzcinski committed
24
stages:
Kamil Trzcinski's avatar
Kamil Trzcinski committed
25
- prepare
Kamil Trzcinski's avatar
Kamil Trzcinski committed
26
- test
27
- post-test
28
- pages
29

30 31 32 33 34
# Predefined scopes
.dedicated-runner: &dedicated-runner
  tags:
    - gitlab-org

35 36 37 38 39
.knapsack-state: &knapsack-state
  services: []
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
40
    KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
41 42 43
  cache:
    key: "knapsack"
    paths:
44
      - knapsack/
45 46 47
  artifacts:
    expire_in: 31d
    paths:
48
      - knapsack/
49

50 51 52 53 54 55
.use-pg: &use-pg
  services:
    - postgres:latest
    - redis:alpine

.use-mysql: &use-mysql
56 57 58
  services:
    - mysql:latest
    - redis:alpine
59

60 61 62 63 64 65 66 67 68 69
.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
  only:
    - /\-(?i)mysql$/
    - master@gitlab-org/gitlab-ce
    - master@gitlab/gitlabhq
    - tags@gitlab-org/gitlab-ce
    - tags@gitlab/gitlabhq
    - //@gitlab-org/gitlab-ee
    - //@gitlab/gitlab-ee

70 71
.rspec-knapsack: &rspec-knapsack
  stage: test
72
  <<: *dedicated-runner
73
  script:
74
    - JOB_NAME=( $CI_JOB_NAME )
75 76
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
77
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
78
    - export KNAPSACK_GENERATE_REPORT=true
79
    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
80 81 82
    - knapsack rspec "--color --format documentation"
  artifacts:
    expire_in: 31d
83
    when: always
84
    paths:
85 86 87
      - coverage/
      - knapsack/
      - tmp/capybara/
88

89 90 91 92 93 94 95
.rspec-knapsack-pg: &rspec-knapsack-pg
  <<: *rspec-knapsack
  <<: *use-pg

.rspec-knapsack-mysql: &rspec-knapsack-mysql
  <<: *rspec-knapsack
  <<: *use-mysql
96
  <<: *only-master-and-ee-or-mysql
97

98 99
.spinach-knapsack: &spinach-knapsack
  stage: test
100
  <<: *dedicated-runner
101
  script:
102
    - JOB_NAME=( $CI_JOB_NAME )
103 104
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
105
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
106
    - export KNAPSACK_GENERATE_REPORT=true
107
    - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
108 109 110
    - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
  artifacts:
    expire_in: 31d
111
    when: always
112
    paths:
113 114 115
      - coverage/
      - knapsack/
      - tmp/capybara/
116

117 118 119 120 121 122 123
.spinach-knapsack-pg: &spinach-knapsack-pg
  <<: *spinach-knapsack
  <<: *use-pg

.spinach-knapsack-mysql: &spinach-knapsack-mysql
  <<: *spinach-knapsack
  <<: *use-mysql
124
  <<: *only-master-and-ee-or-mysql
125

126 127 128 129 130 131
# Prepare and merge knapsack tests
knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: prepare
  script:
132
    - mkdir -p knapsack/${CI_PROJECT_NAME}/
133 134
    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
    - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH
135 136 137 138 139 140 141 142
    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
    - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'

update-knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: post-test
  script:
143 144
    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json
    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json
145 146 147 148 149 150 151
    - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
152 153

setup-test-env:
154
  <<: *use-pg
155 156 157
  <<: *dedicated-runner
  stage: prepare
  script:
158
    - node --version
159
    - yarn install --pure-lockfile
Mike Greiling's avatar
Mike Greiling committed
160
    - bundle exec rake gitlab:assets:compile
161 162 163 164
    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
  artifacts:
    expire_in: 7d
    paths:
165
      - node_modules
166 167 168
      - public/assets
      - tmp/tests

169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
rspec pg 0 20: *rspec-knapsack-pg
rspec pg 1 20: *rspec-knapsack-pg
rspec pg 2 20: *rspec-knapsack-pg
rspec pg 3 20: *rspec-knapsack-pg
rspec pg 4 20: *rspec-knapsack-pg
rspec pg 5 20: *rspec-knapsack-pg
rspec pg 6 20: *rspec-knapsack-pg
rspec pg 7 20: *rspec-knapsack-pg
rspec pg 8 20: *rspec-knapsack-pg
rspec pg 9 20: *rspec-knapsack-pg
rspec pg 10 20: *rspec-knapsack-pg
rspec pg 11 20: *rspec-knapsack-pg
rspec pg 12 20: *rspec-knapsack-pg
rspec pg 13 20: *rspec-knapsack-pg
rspec pg 14 20: *rspec-knapsack-pg
rspec pg 15 20: *rspec-knapsack-pg
rspec pg 16 20: *rspec-knapsack-pg
rspec pg 17 20: *rspec-knapsack-pg
rspec pg 18 20: *rspec-knapsack-pg
rspec pg 19 20: *rspec-knapsack-pg

rspec mysql 0 20: *rspec-knapsack-mysql
rspec mysql 1 20: *rspec-knapsack-mysql
rspec mysql 2 20: *rspec-knapsack-mysql
rspec mysql 3 20: *rspec-knapsack-mysql
rspec mysql 4 20: *rspec-knapsack-mysql
rspec mysql 5 20: *rspec-knapsack-mysql
rspec mysql 6 20: *rspec-knapsack-mysql
rspec mysql 7 20: *rspec-knapsack-mysql
rspec mysql 8 20: *rspec-knapsack-mysql
rspec mysql 9 20: *rspec-knapsack-mysql
rspec mysql 10 20: *rspec-knapsack-mysql
rspec mysql 11 20: *rspec-knapsack-mysql
rspec mysql 12 20: *rspec-knapsack-mysql
rspec mysql 13 20: *rspec-knapsack-mysql
rspec mysql 14 20: *rspec-knapsack-mysql
rspec mysql 15 20: *rspec-knapsack-mysql
rspec mysql 16 20: *rspec-knapsack-mysql
rspec mysql 17 20: *rspec-knapsack-mysql
rspec mysql 18 20: *rspec-knapsack-mysql
rspec mysql 19 20: *rspec-knapsack-mysql

spinach pg 0 10: *spinach-knapsack-pg
spinach pg 1 10: *spinach-knapsack-pg
spinach pg 2 10: *spinach-knapsack-pg
spinach pg 3 10: *spinach-knapsack-pg
spinach pg 4 10: *spinach-knapsack-pg
spinach pg 5 10: *spinach-knapsack-pg
spinach pg 6 10: *spinach-knapsack-pg
spinach pg 7 10: *spinach-knapsack-pg
spinach pg 8 10: *spinach-knapsack-pg
spinach pg 9 10: *spinach-knapsack-pg

spinach mysql 0 10: *spinach-knapsack-mysql
spinach mysql 1 10: *spinach-knapsack-mysql
spinach mysql 2 10: *spinach-knapsack-mysql
spinach mysql 3 10: *spinach-knapsack-mysql
spinach mysql 4 10: *spinach-knapsack-mysql
spinach mysql 5 10: *spinach-knapsack-mysql
spinach mysql 6 10: *spinach-knapsack-mysql
spinach mysql 7 10: *spinach-knapsack-mysql
spinach mysql 8 10: *spinach-knapsack-mysql
spinach mysql 9 10: *spinach-knapsack-mysql
232 233 234 235 236 237 238 239 240 241

# Other generic tests
.ruby-static-analysis: &ruby-static-analysis
  variables:
    SIMPLECOV: "false"
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"

.exec: &exec
  <<: *ruby-static-analysis
242
  <<: *dedicated-runner
243 244
  stage: test
  script:
245
    - bundle exec $CI_JOB_NAME
246

247
rubocop:
248 249 250 251 252 253
  <<: *ruby-static-analysis
  <<: *dedicated-runner
  stage: test
  script:
    - bundle exec "rubocop --require rubocop-rspec"

254 255
rake haml_lint: *exec
rake scss_lint: *exec
256
rake config_lint: *exec
257 258 259 260 261 262 263 264 265 266 267 268 269 270
rake brakeman: *exec
rake flay: *exec
license_finder: *exec
rake downtime_check: *exec
rake ee_compat_check:
  <<: *exec
  only:
    - branches@gitlab-org/gitlab-ce
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
  allow_failure: yes
  cache:
271
    key: "ee_compat_check_repo"
272
    paths:
273
      - ee_compat_check/ee-repo/
274
  artifacts:
275
    name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
276 277 278 279 280
    when: on_failure
    expire_in: 10d
    paths:
      - ee_compat_check/patches/*.patch

281
.db-migrate-reset: &db-migrate-reset
282
  stage: test
283
  <<: *dedicated-runner
284
  script:
285
    - bundle exec rake db:migrate:reset
286

287
rake pg db:migrate:reset:
288 289 290
  <<: *db-migrate-reset
  <<: *use-pg

291
rake mysql db:migrate:reset:
292 293 294 295
  <<: *db-migrate-reset
  <<: *use-mysql

.db-rollback: &db-rollback
296 297 298
  stage: test
  <<: *dedicated-runner
  script:
299 300
    - bundle exec rake db:rollback STEP=120
    - bundle exec rake db:migrate
301

302
rake pg db:rollback:
303 304 305
  <<: *db-rollback
  <<: *use-pg

306
rake mysql db:rollback:
307 308 309 310
  <<: *db-rollback
  <<: *use-mysql

.db-seed_fu: &db-seed_fu
311
  stage: test
312
  <<: *dedicated-runner
313 314 315 316
  variables:
    SIZE: "1"
    SETUP_DB: "false"
    RAILS_ENV: "development"
317 318 319 320
  script:
    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
       /home/git/repositories/gitlab-org/gitlab-test.git
    - bundle exec rake db:setup db:seed_fu
321 322 323 324 325 326
  artifacts:
    when: on_failure
    expire_in: 1d
    paths:
      - log/development.log

327
rake pg db:seed_fu:
328 329 330
  <<: *db-seed_fu
  <<: *use-pg

331
rake mysql db:seed_fu:
332 333 334
  <<: *db-seed_fu
  <<: *use-mysql

335 336 337 338 339 340 341 342 343 344
rake gitlab:assets:compile:
  stage: test
  <<: *dedicated-runner
  dependencies: []
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"
    SETUP_DB: "false"
    USE_DB: "false"
    SKIP_STORAGE_VALIDATION: "true"
345 346 347 348 349 350 351 352
    WEBPACK_REPORT: "true"
  script:
    - bundle exec rake yarn:install gitlab:assets:compile
  artifacts:
    name: webpack-report
    expire_in: 31d
    paths:
    - webpack-report/
353

Clement Ho's avatar
Clement Ho committed
354
rake karma:
355 356 357
  cache:
    paths:
      - vendor/ruby
358
      - node_modules
359
  stage: test
360
  <<: *use-pg
361
  <<: *dedicated-runner
362 363
  variables:
    BABEL_ENV: "coverage"
364
  script:
365
    - bundle exec rake karma
366
  coverage: '/^Statements *: (\d+\.\d+%)/'
367 368 369 370
  artifacts:
    name: coverage-javascript
    expire_in: 31d
    paths:
371
    - coverage-javascript/
372

373 374
docs:check:apilint:
  image: "phusion/baseimage"
375
  stage: test
376
  <<: *dedicated-runner
377 378
  cache: {}
  dependencies: []
379 380 381 382
  before_script: []
  script:
    - scripts/lint-doc.sh

383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
docs:check:links:
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
  stage: test
  <<: *dedicated-runner
  cache: {}
  dependencies: []
  before_script: []
  script:
    - mv doc/ /nanoc/content/
    - cd /nanoc
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links

398
bundler:check:
399 400 401 402
  stage: test
  <<: *dedicated-runner
  <<: *ruby-static-analysis
  script:
403 404 405 406 407
   - bundle check

bundler:audit:
  stage: test
  <<: *ruby-static-analysis
408
  <<: *dedicated-runner
409 410 411 412 413 414
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
  script:
415
    - "bundle exec bundle-audit check --update --ignore CVE-2016-4658"
416

417
.migration-paths: &migration-paths
Z.J. van de Weg's avatar
Z.J. van de Weg committed
418
  stage: test
419
  <<: *dedicated-runner
420 421
  variables:
    SETUP_DB: "false"
422 423 424 425 426
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
Z.J. van de Weg's avatar
Z.J. van de Weg committed
427
  script:
Z.J. van de Weg's avatar
Z.J. van de Weg committed
428
    - git fetch origin v8.14.10
Kamil Trzcinski's avatar
Kamil Trzcinski committed
429
    - git checkout -f FETCH_HEAD
430
    - bundle install --without production --jobs $(nproc) $FLAGS --retry=3
431
    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
432
    - git checkout $CI_COMMIT_SHA
433
    - bundle install --without production --jobs $(nproc) $FLAGS --retry=3
434
    - source scripts/prepare_build.sh
435
    - bundle exec rake db:migrate
436

437
migration pg paths:
438 439 440
  <<: *migration-paths
  <<: *use-pg

441
migration mysql paths:
442 443 444
  <<: *migration-paths
  <<: *use-mysql

445 446 447
coverage:
  stage: post-test
  services: []
448
  <<: *dedicated-runner
449 450 451 452 453
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"
  script:
    - bundle exec scripts/merge-simplecov
454
  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
455 456 457 458 459 460 461
  artifacts:
    name: coverage
    expire_in: 31d
    paths:
    - coverage/index.html
    - coverage/assets/

462
lint:javascript:
463
  <<: *dedicated-runner
464 465 466
  cache:
    paths:
      - node_modules/
467
  stage: test
468
  before_script: []
469
  script:
470
    - yarn run eslint
471 472

lint:javascript:report:
473
  <<: *dedicated-runner
474 475 476 477
  cache:
    paths:
      - node_modules/
  stage: post-test
478
  before_script: []
479
  script:
480
    - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files
481
    - yarn run eslint-report || true # ignore exit code
482 483 484 485 486
  artifacts:
    name: eslint-report
    expire_in: 31d
    paths:
    - eslint-report.html
487 488 489 490 491 492

# Trigger docs build
# https://gitlab.com/gitlab-com/doc-gitlab-com/blob/master/README.md#deployment-process
trigger_docs:
  stage: post-test
  image: "alpine"
493
  <<: *dedicated-runner
494 495 496
  before_script:
    - apk update && apk add curl
  variables:
497
    GIT_STRATEGY: "none"
498 499 500
  cache: {}
  artifacts: {}
  script:
501 502
    - "HTTP_STATUS=$(curl -X POST -F token=${DOCS_TRIGGER_TOKEN} -F ref=master -F variables[PROJECT]=${CI_PROJECT_NAME} --silent --output curl.log --write-out '%{http_code}' https://gitlab.com/api/v3/projects/1794617/trigger/builds)"
    - if [ "${HTTP_STATUS}" -ne "201" ]; then echo "Error ${HTTP_STATUS}"; cat curl.log; echo; exit 1; fi
503 504
  only:
    - master@gitlab-org/gitlab-ce
505
    - master@gitlab-org/gitlab-ee
506 507 508 509

# Notify slack in the end
notify:slack:
  stage: post-test
510
  <<: *dedicated-runner
511 512 513 514
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
  script:
515
    - ./scripts/notify_slack.sh "#development" "Build on \`$CI_COMMIT_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_COMMIT_SHA"/pipelines>"
516 517 518 519 520 521 522 523 524 525
  when: on_failure
  only:
    - master@gitlab-org/gitlab-ce
    - tags@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - tags@gitlab-org/gitlab-ee

pages:
  before_script: []
  stage: pages
526
  <<: *dedicated-runner
527 528
  dependencies:
    - coverage
Clement Ho's avatar
Clement Ho committed
529
    - rake karma
530
    - rake gitlab:assets:compile
531
    - lint:javascript:report
532 533 534
  script:
    - mv public/ .public/
    - mkdir public/
535
    - mv coverage/ public/coverage-ruby/ || true
536
    - mv coverage-javascript/ public/coverage-javascript/ || true
537
    - mv eslint-report.html public/ || true
538
    - mv webpack-report/ public/webpack-report/ || true
539 540 541 542
  artifacts:
    paths:
      - public
  only:
543
    - master@gitlab-org/gitlab-ce
544
    - master@gitlab-org/gitlab-ee
545 546 547 548

# Insurance in case a gem needed by one of our releases gets yanked from
# rubygems.org in the future.
cache gems:
549
  <<: *dedicated-runner
550 551 552 553 554 555 556 557 558
  only:
    - tags
  variables:
    SETUP_DB: "false"
  script:
    - bundle package --all --all-platforms
  artifacts:
    paths:
      - vendor/cache
559 560
  only:
    - master@gitlab-org/gitlab-ce
561
    - master@gitlab-org/gitlab-ee