.gitlab-ci.yml 14.5 KB
Newer Older
1
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
2

3
cache:
Z.J. van de Weg's avatar
Z.J. van de Weg committed
4
  key: "ruby-233"
5
  paths:
6
  - vendor/ruby
7

8 9
variables:
  MYSQL_ALLOW_EMPTY_PASSWORD: "1"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
10
  RAILS_ENV: "test"
Mike Greiling's avatar
Mike Greiling committed
11
  NODE_ENV: "test"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
12
  SIMPLECOV: "true"
13
  GIT_DEPTH: "20"
14
  PHANTOMJS_VERSION: "2.1.1"
15
  GET_SOURCES_ATTEMPTS: "3"
16 17
  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
  KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json
18

Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
19
before_script:
Kamil Trzcinski's avatar
Test  
Kamil Trzcinski committed
20
  - bundle --version
21 22
  - . scripts/utils.sh
  - ./scripts/prepare_build.sh
23

Kamil Trzcinski's avatar
Kamil Trzcinski committed
24
stages:
Kamil Trzcinski's avatar
Kamil Trzcinski committed
25
- prepare
Kamil Trzcinski's avatar
Kamil Trzcinski committed
26
- test
27
- post-test
28
- pages
29

30 31 32 33 34
# Predefined scopes
.dedicated-runner: &dedicated-runner
  tags:
    - gitlab-org

35 36 37 38 39
.knapsack-state: &knapsack-state
  services: []
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
40
    KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
41 42 43
  cache:
    key: "knapsack"
    paths:
44
      - knapsack/
45 46 47
  artifacts:
    expire_in: 31d
    paths:
48
      - knapsack/
49

50 51 52 53 54 55
.use-pg: &use-pg
  services:
    - postgres:latest
    - redis:alpine

.use-mysql: &use-mysql
56 57 58
  services:
    - mysql:latest
    - redis:alpine
59

60 61
.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
  only:
62
    - /mysql/
63 64 65 66 67 68 69
    - master@gitlab-org/gitlab-ce
    - master@gitlab/gitlabhq
    - tags@gitlab-org/gitlab-ce
    - tags@gitlab/gitlabhq
    - //@gitlab-org/gitlab-ee
    - //@gitlab/gitlab-ee

70 71
.rspec-knapsack: &rspec-knapsack
  stage: test
72
  <<: *dedicated-runner
73
  script:
74
    - JOB_NAME=( $CI_JOB_NAME )
75 76
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
77
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
78
    - export KNAPSACK_GENERATE_REPORT=true
79
    - export CACHE_CLASSES=true
80
    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
81 82 83
    - knapsack rspec "--color --format documentation"
  artifacts:
    expire_in: 31d
84
    when: always
85
    paths:
86 87 88
      - coverage/
      - knapsack/
      - tmp/capybara/
89

90 91 92 93 94 95 96
.rspec-knapsack-pg: &rspec-knapsack-pg
  <<: *rspec-knapsack
  <<: *use-pg

.rspec-knapsack-mysql: &rspec-knapsack-mysql
  <<: *rspec-knapsack
  <<: *use-mysql
97
  <<: *only-master-and-ee-or-mysql
98

99 100
.spinach-knapsack: &spinach-knapsack
  stage: test
101
  <<: *dedicated-runner
102
  script:
103
    - JOB_NAME=( $CI_JOB_NAME )
104 105
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
106
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
107
    - export KNAPSACK_GENERATE_REPORT=true
108
    - export CACHE_CLASSES=true
109
    - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
110 111 112
    - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
  artifacts:
    expire_in: 31d
113
    when: always
114
    paths:
115 116 117
      - coverage/
      - knapsack/
      - tmp/capybara/
118

119 120 121 122 123 124 125
.spinach-knapsack-pg: &spinach-knapsack-pg
  <<: *spinach-knapsack
  <<: *use-pg

.spinach-knapsack-mysql: &spinach-knapsack-mysql
  <<: *spinach-knapsack
  <<: *use-mysql
126
  <<: *only-master-and-ee-or-mysql
127

128 129 130 131 132 133
# Prepare and merge knapsack tests
knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: prepare
  script:
134
    - mkdir -p knapsack/${CI_PROJECT_NAME}/
135 136
    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
    - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH
137 138 139 140 141 142 143 144
    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
    - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'

update-knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: post-test
  script:
145 146
    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json
    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json
147 148 149 150 151 152 153
    - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
154 155

setup-test-env:
156
  <<: *use-pg
157 158 159
  <<: *dedicated-runner
  stage: prepare
  script:
160
    - node --version
161
    - yarn install --pure-lockfile
Mike Greiling's avatar
Mike Greiling committed
162
    - bundle exec rake gitlab:assets:compile
163 164 165 166
    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
  artifacts:
    expire_in: 7d
    paths:
167
      - node_modules
168 169 170
      - public/assets
      - tmp/tests

171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
rspec pg 0 20: *rspec-knapsack-pg
rspec pg 1 20: *rspec-knapsack-pg
rspec pg 2 20: *rspec-knapsack-pg
rspec pg 3 20: *rspec-knapsack-pg
rspec pg 4 20: *rspec-knapsack-pg
rspec pg 5 20: *rspec-knapsack-pg
rspec pg 6 20: *rspec-knapsack-pg
rspec pg 7 20: *rspec-knapsack-pg
rspec pg 8 20: *rspec-knapsack-pg
rspec pg 9 20: *rspec-knapsack-pg
rspec pg 10 20: *rspec-knapsack-pg
rspec pg 11 20: *rspec-knapsack-pg
rspec pg 12 20: *rspec-knapsack-pg
rspec pg 13 20: *rspec-knapsack-pg
rspec pg 14 20: *rspec-knapsack-pg
rspec pg 15 20: *rspec-knapsack-pg
rspec pg 16 20: *rspec-knapsack-pg
rspec pg 17 20: *rspec-knapsack-pg
rspec pg 18 20: *rspec-knapsack-pg
rspec pg 19 20: *rspec-knapsack-pg

rspec mysql 0 20: *rspec-knapsack-mysql
rspec mysql 1 20: *rspec-knapsack-mysql
rspec mysql 2 20: *rspec-knapsack-mysql
rspec mysql 3 20: *rspec-knapsack-mysql
rspec mysql 4 20: *rspec-knapsack-mysql
rspec mysql 5 20: *rspec-knapsack-mysql
rspec mysql 6 20: *rspec-knapsack-mysql
rspec mysql 7 20: *rspec-knapsack-mysql
rspec mysql 8 20: *rspec-knapsack-mysql
rspec mysql 9 20: *rspec-knapsack-mysql
rspec mysql 10 20: *rspec-knapsack-mysql
rspec mysql 11 20: *rspec-knapsack-mysql
rspec mysql 12 20: *rspec-knapsack-mysql
rspec mysql 13 20: *rspec-knapsack-mysql
rspec mysql 14 20: *rspec-knapsack-mysql
rspec mysql 15 20: *rspec-knapsack-mysql
rspec mysql 16 20: *rspec-knapsack-mysql
rspec mysql 17 20: *rspec-knapsack-mysql
rspec mysql 18 20: *rspec-knapsack-mysql
rspec mysql 19 20: *rspec-knapsack-mysql

spinach pg 0 10: *spinach-knapsack-pg
spinach pg 1 10: *spinach-knapsack-pg
spinach pg 2 10: *spinach-knapsack-pg
spinach pg 3 10: *spinach-knapsack-pg
spinach pg 4 10: *spinach-knapsack-pg
spinach pg 5 10: *spinach-knapsack-pg
spinach pg 6 10: *spinach-knapsack-pg
spinach pg 7 10: *spinach-knapsack-pg
spinach pg 8 10: *spinach-knapsack-pg
spinach pg 9 10: *spinach-knapsack-pg

spinach mysql 0 10: *spinach-knapsack-mysql
spinach mysql 1 10: *spinach-knapsack-mysql
spinach mysql 2 10: *spinach-knapsack-mysql
spinach mysql 3 10: *spinach-knapsack-mysql
spinach mysql 4 10: *spinach-knapsack-mysql
spinach mysql 5 10: *spinach-knapsack-mysql
spinach mysql 6 10: *spinach-knapsack-mysql
spinach mysql 7 10: *spinach-knapsack-mysql
spinach mysql 8 10: *spinach-knapsack-mysql
spinach mysql 9 10: *spinach-knapsack-mysql
234 235 236 237 238 239 240 241 242 243

# Other generic tests
.ruby-static-analysis: &ruby-static-analysis
  variables:
    SIMPLECOV: "false"
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"

.exec: &exec
  <<: *ruby-static-analysis
244
  <<: *dedicated-runner
245 246
  stage: test
  script:
247
    - bundle exec $CI_JOB_NAME
248

249
rubocop:
250 251 252 253 254 255
  <<: *ruby-static-analysis
  <<: *dedicated-runner
  stage: test
  script:
    - bundle exec "rubocop --require rubocop-rspec"

256 257
rake haml_lint: *exec
rake scss_lint: *exec
258
rake config_lint: *exec
259 260 261
rake brakeman: *exec
rake flay: *exec
license_finder: *exec
262 263 264 265 266 267 268
rake downtime_check:
  <<: *exec
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/

269 270 271 272 273 274 275 276 277 278
rake ee_compat_check:
  <<: *exec
  only:
    - branches@gitlab-org/gitlab-ce
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
  allow_failure: yes
  cache:
279
    key: "ee_compat_check_repo"
280
    paths:
281
      - ee_compat_check/ee-repo/
282
  artifacts:
283
    name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
284 285 286 287 288
    when: on_failure
    expire_in: 10d
    paths:
      - ee_compat_check/patches/*.patch

289
.db-migrate-reset: &db-migrate-reset
290
  stage: test
291
  <<: *dedicated-runner
292
  script:
293
    - bundle exec rake db:migrate:reset
294

295
rake pg db:migrate:reset:
296 297 298
  <<: *db-migrate-reset
  <<: *use-pg

299
rake mysql db:migrate:reset:
300 301 302 303
  <<: *db-migrate-reset
  <<: *use-mysql

.db-rollback: &db-rollback
304 305 306
  stage: test
  <<: *dedicated-runner
  script:
307 308
    - bundle exec rake db:rollback STEP=120
    - bundle exec rake db:migrate
309

310
rake pg db:rollback:
311 312 313
  <<: *db-rollback
  <<: *use-pg

314
rake mysql db:rollback:
315 316 317 318
  <<: *db-rollback
  <<: *use-mysql

.db-seed_fu: &db-seed_fu
319
  stage: test
320
  <<: *dedicated-runner
321 322 323 324 325 326 327 328 329 330 331 332 333 334
  variables:
    SIZE: "1"
    SETUP_DB: "false"
    RAILS_ENV: "development"
  script:
    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
       /home/git/repositories/gitlab-org/gitlab-test.git
    - bundle exec rake db:setup db:seed_fu
  artifacts:
    when: on_failure
    expire_in: 1d
    paths:
      - log/development.log

335
rake pg db:seed_fu:
336 337 338
  <<: *db-seed_fu
  <<: *use-pg

339
rake mysql db:seed_fu:
340 341 342
  <<: *db-seed_fu
  <<: *use-mysql

343 344 345 346 347 348 349 350 351 352
rake gitlab:assets:compile:
  stage: test
  <<: *dedicated-runner
  dependencies: []
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"
    SETUP_DB: "false"
    USE_DB: "false"
    SKIP_STORAGE_VALIDATION: "true"
353 354 355 356 357 358 359 360
    WEBPACK_REPORT: "true"
  script:
    - bundle exec rake yarn:install gitlab:assets:compile
  artifacts:
    name: webpack-report
    expire_in: 31d
    paths:
    - webpack-report/
361

Clement Ho's avatar
Clement Ho committed
362
rake karma:
363 364 365
  cache:
    paths:
      - vendor/ruby
366
  stage: test
367
  <<: *use-pg
368
  <<: *dedicated-runner
369 370
  variables:
    BABEL_ENV: "coverage"
371
  script:
372
    - bundle exec rake karma
373
  coverage: '/^Statements *: (\d+\.\d+%)/'
374 375 376 377
  artifacts:
    name: coverage-javascript
    expire_in: 31d
    paths:
378
    - coverage-javascript/
379

380 381
docs:check:apilint:
  image: "phusion/baseimage"
382
  stage: test
383
  <<: *dedicated-runner
384 385
  cache: {}
  dependencies: []
386 387 388 389
  before_script: []
  script:
    - scripts/lint-doc.sh

390 391 392 393 394 395 396 397 398 399 400 401 402 403 404
docs:check:links:
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
  stage: test
  <<: *dedicated-runner
  cache: {}
  dependencies: []
  before_script: []
  script:
    - mv doc/ /nanoc/content/
    - cd /nanoc
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links

405
bundler:check:
406 407 408 409
  stage: test
  <<: *dedicated-runner
  <<: *ruby-static-analysis
  script:
410 411 412 413 414
   - bundle check

bundler:audit:
  stage: test
  <<: *ruby-static-analysis
415
  <<: *dedicated-runner
416 417 418 419 420 421
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
  script:
422
    - "bundle exec bundle-audit check --update --ignore CVE-2016-4658"
423

424
.migration-paths: &migration-paths
425
  stage: test
426
  <<: *dedicated-runner
427 428
  variables:
    SETUP_DB: "false"
429 430 431 432 433
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
434
  script:
435
    - git fetch origin v8.14.10
Kamil Trzcinski's avatar
Kamil Trzcinski committed
436
    - git checkout -f FETCH_HEAD
437
    - bundle install $BUNDLE_INSTALL_FLAGS
438
    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
439
    - git checkout $CI_COMMIT_SHA
440
    - bundle install $BUNDLE_INSTALL_FLAGS
441
    - . scripts/prepare_build.sh
442
    - bundle exec rake db:migrate
443

444
migration pg paths:
445 446 447
  <<: *migration-paths
  <<: *use-pg

448
migration mysql paths:
449 450 451
  <<: *migration-paths
  <<: *use-mysql

452 453 454
coverage:
  stage: post-test
  services: []
455
  <<: *dedicated-runner
456 457 458 459 460
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"
  script:
    - bundle exec scripts/merge-simplecov
461
  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
462 463 464 465 466 467 468
  artifacts:
    name: coverage
    expire_in: 31d
    paths:
    - coverage/index.html
    - coverage/assets/

469
lint:javascript:
470
  <<: *dedicated-runner
471
  stage: test
472
  before_script: []
473
  script:
474
    - yarn run eslint
475 476

lint:javascript:report:
477
  <<: *dedicated-runner
478
  stage: post-test
479
  before_script: []
480
  script:
481
    - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files
482
    - yarn run eslint-report || true # ignore exit code
483 484 485 486 487
  artifacts:
    name: eslint-report
    expire_in: 31d
    paths:
    - eslint-report.html
488 489 490 491 492 493

# Trigger docs build
# https://gitlab.com/gitlab-com/doc-gitlab-com/blob/master/README.md#deployment-process
trigger_docs:
  stage: post-test
  image: "alpine"
494
  <<: *dedicated-runner
495 496 497
  before_script:
    - apk update && apk add curl
  variables:
498
    GIT_STRATEGY: "none"
499 500 501
  cache: {}
  artifacts: {}
  script:
502 503
    - "HTTP_STATUS=$(curl -X POST -F token=${DOCS_TRIGGER_TOKEN} -F ref=master -F variables[PROJECT]=${CI_PROJECT_NAME} --silent --output curl.log --write-out '%{http_code}' https://gitlab.com/api/v3/projects/1794617/trigger/builds)"
    - if [ "${HTTP_STATUS}" -ne "201" ]; then echo "Error ${HTTP_STATUS}"; cat curl.log; echo; exit 1; fi
504 505
  only:
    - master@gitlab-org/gitlab-ce
506
    - master@gitlab-org/gitlab-ee
507 508 509 510

# Notify slack in the end
notify:slack:
  stage: post-test
511
  <<: *dedicated-runner
512 513 514 515
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
  script:
516
    - ./scripts/notify_slack.sh "#development" "Build on \`$CI_COMMIT_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_COMMIT_SHA"/pipelines>"
517 518 519 520 521 522 523 524 525 526
  when: on_failure
  only:
    - master@gitlab-org/gitlab-ce
    - tags@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - tags@gitlab-org/gitlab-ee

pages:
  before_script: []
  stage: pages
527
  <<: *dedicated-runner
528 529
  dependencies:
    - coverage
Clement Ho's avatar
Clement Ho committed
530
    - rake karma
531
    - rake gitlab:assets:compile
532
    - lint:javascript:report
533 534 535
  script:
    - mv public/ .public/
    - mkdir public/
536
    - mv coverage/ public/coverage-ruby/ || true
537
    - mv coverage-javascript/ public/coverage-javascript/ || true
538
    - mv eslint-report.html public/ || true
539
    - mv webpack-report/ public/webpack-report/ || true
540 541 542 543
  artifacts:
    paths:
      - public
  only:
544
    - master@gitlab-org/gitlab-ce
545
    - master@gitlab-org/gitlab-ee
546 547 548 549

# Insurance in case a gem needed by one of our releases gets yanked from
# rubygems.org in the future.
cache gems:
550
  <<: *dedicated-runner
551 552 553 554 555 556 557 558 559
  only:
    - tags
  variables:
    SETUP_DB: "false"
  script:
    - bundle package --all --all-platforms
  artifacts:
    paths:
      - vendor/cache
560 561
  only:
    - master@gitlab-org/gitlab-ce
562
    - master@gitlab-org/gitlab-ee