.gitlab-ci.yml 14.4 KB
Newer Older
1
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
2

3
cache:
Z.J. van de Weg's avatar
Z.J. van de Weg committed
4
  key: "ruby-233"
5
  paths:
6
  - vendor/ruby
7

8 9
variables:
  MYSQL_ALLOW_EMPTY_PASSWORD: "1"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
10
  RAILS_ENV: "test"
Mike Greiling's avatar
Mike Greiling committed
11
  NODE_ENV: "test"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
12
  SIMPLECOV: "true"
13
  GIT_DEPTH: "20"
14
  GIT_SUBMODULE_STRATEGY: "none"
15
  PHANTOMJS_VERSION: "2.1.1"
16
  GET_SOURCES_ATTEMPTS: "3"
17 18
  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
  KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json
19

Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
20
before_script:
Kamil Trzcinski's avatar
Test  
Kamil Trzcinski committed
21
  - bundle --version
22 23
  - . scripts/utils.sh
  - ./scripts/prepare_build.sh
24

Kamil Trzcinski's avatar
Kamil Trzcinski committed
25
stages:
Kamil Trzcinski's avatar
Kamil Trzcinski committed
26
- prepare
Kamil Trzcinski's avatar
Kamil Trzcinski committed
27
- test
28
- post-test
29
- pages
30

31 32 33 34 35
# Predefined scopes
.dedicated-runner: &dedicated-runner
  tags:
    - gitlab-org

36 37 38 39 40
.knapsack-state: &knapsack-state
  services: []
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
41
    KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
42 43 44
  cache:
    key: "knapsack"
    paths:
45
      - knapsack/
46 47 48
  artifacts:
    expire_in: 31d
    paths:
49
      - knapsack/
50

51 52 53 54 55 56
.use-pg: &use-pg
  services:
    - postgres:latest
    - redis:alpine

.use-mysql: &use-mysql
57 58 59
  services:
    - mysql:latest
    - redis:alpine
60

61 62
.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
  only:
63
    - /mysql/
64 65 66 67 68 69 70
    - master@gitlab-org/gitlab-ce
    - master@gitlab/gitlabhq
    - tags@gitlab-org/gitlab-ce
    - tags@gitlab/gitlabhq
    - //@gitlab-org/gitlab-ee
    - //@gitlab/gitlab-ee

71 72
.rspec-knapsack: &rspec-knapsack
  stage: test
73
  <<: *dedicated-runner
74
  script:
75
    - JOB_NAME=( $CI_JOB_NAME )
76 77
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
78
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
79
    - export KNAPSACK_GENERATE_REPORT=true
80
    - export CACHE_CLASSES=true
81
    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
82 83 84
    - knapsack rspec "--color --format documentation"
  artifacts:
    expire_in: 31d
85
    when: always
86
    paths:
87 88 89
      - coverage/
      - knapsack/
      - tmp/capybara/
90

91 92 93 94 95 96 97
.rspec-knapsack-pg: &rspec-knapsack-pg
  <<: *rspec-knapsack
  <<: *use-pg

.rspec-knapsack-mysql: &rspec-knapsack-mysql
  <<: *rspec-knapsack
  <<: *use-mysql
98
  <<: *only-master-and-ee-or-mysql
99

100 101
.spinach-knapsack: &spinach-knapsack
  stage: test
102
  <<: *dedicated-runner
103
  script:
104
    - JOB_NAME=( $CI_JOB_NAME )
105 106
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
107
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
108
    - export KNAPSACK_GENERATE_REPORT=true
109
    - export CACHE_CLASSES=true
110
    - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
111 112 113
    - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
  artifacts:
    expire_in: 31d
114
    when: always
115
    paths:
116 117 118
      - coverage/
      - knapsack/
      - tmp/capybara/
119

120 121 122 123 124 125 126
.spinach-knapsack-pg: &spinach-knapsack-pg
  <<: *spinach-knapsack
  <<: *use-pg

.spinach-knapsack-mysql: &spinach-knapsack-mysql
  <<: *spinach-knapsack
  <<: *use-mysql
127
  <<: *only-master-and-ee-or-mysql
128

129 130 131 132 133 134
# Prepare and merge knapsack tests
knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: prepare
  script:
135
    - mkdir -p knapsack/${CI_PROJECT_NAME}/
136 137
    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
    - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH
138 139 140 141 142 143 144 145
    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
    - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'

update-knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: post-test
  script:
146 147
    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json
    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json
148 149 150 151 152 153 154
    - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
155 156

setup-test-env:
157
  <<: *use-pg
158 159 160
  <<: *dedicated-runner
  stage: prepare
  script:
161
    - node --version
162
    - yarn install --pure-lockfile
Mike Greiling's avatar
Mike Greiling committed
163
    - bundle exec rake gitlab:assets:compile
164 165 166 167
    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
  artifacts:
    expire_in: 7d
    paths:
168
      - node_modules
169 170 171
      - public/assets
      - tmp/tests

172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
rspec pg 0 20: *rspec-knapsack-pg
rspec pg 1 20: *rspec-knapsack-pg
rspec pg 2 20: *rspec-knapsack-pg
rspec pg 3 20: *rspec-knapsack-pg
rspec pg 4 20: *rspec-knapsack-pg
rspec pg 5 20: *rspec-knapsack-pg
rspec pg 6 20: *rspec-knapsack-pg
rspec pg 7 20: *rspec-knapsack-pg
rspec pg 8 20: *rspec-knapsack-pg
rspec pg 9 20: *rspec-knapsack-pg
rspec pg 10 20: *rspec-knapsack-pg
rspec pg 11 20: *rspec-knapsack-pg
rspec pg 12 20: *rspec-knapsack-pg
rspec pg 13 20: *rspec-knapsack-pg
rspec pg 14 20: *rspec-knapsack-pg
rspec pg 15 20: *rspec-knapsack-pg
rspec pg 16 20: *rspec-knapsack-pg
rspec pg 17 20: *rspec-knapsack-pg
rspec pg 18 20: *rspec-knapsack-pg
rspec pg 19 20: *rspec-knapsack-pg

rspec mysql 0 20: *rspec-knapsack-mysql
rspec mysql 1 20: *rspec-knapsack-mysql
rspec mysql 2 20: *rspec-knapsack-mysql
rspec mysql 3 20: *rspec-knapsack-mysql
rspec mysql 4 20: *rspec-knapsack-mysql
rspec mysql 5 20: *rspec-knapsack-mysql
rspec mysql 6 20: *rspec-knapsack-mysql
rspec mysql 7 20: *rspec-knapsack-mysql
rspec mysql 8 20: *rspec-knapsack-mysql
rspec mysql 9 20: *rspec-knapsack-mysql
rspec mysql 10 20: *rspec-knapsack-mysql
rspec mysql 11 20: *rspec-knapsack-mysql
rspec mysql 12 20: *rspec-knapsack-mysql
rspec mysql 13 20: *rspec-knapsack-mysql
rspec mysql 14 20: *rspec-knapsack-mysql
rspec mysql 15 20: *rspec-knapsack-mysql
rspec mysql 16 20: *rspec-knapsack-mysql
rspec mysql 17 20: *rspec-knapsack-mysql
rspec mysql 18 20: *rspec-knapsack-mysql
rspec mysql 19 20: *rspec-knapsack-mysql

spinach pg 0 10: *spinach-knapsack-pg
spinach pg 1 10: *spinach-knapsack-pg
spinach pg 2 10: *spinach-knapsack-pg
spinach pg 3 10: *spinach-knapsack-pg
spinach pg 4 10: *spinach-knapsack-pg
spinach pg 5 10: *spinach-knapsack-pg
spinach pg 6 10: *spinach-knapsack-pg
spinach pg 7 10: *spinach-knapsack-pg
spinach pg 8 10: *spinach-knapsack-pg
spinach pg 9 10: *spinach-knapsack-pg

spinach mysql 0 10: *spinach-knapsack-mysql
spinach mysql 1 10: *spinach-knapsack-mysql
spinach mysql 2 10: *spinach-knapsack-mysql
spinach mysql 3 10: *spinach-knapsack-mysql
spinach mysql 4 10: *spinach-knapsack-mysql
spinach mysql 5 10: *spinach-knapsack-mysql
spinach mysql 6 10: *spinach-knapsack-mysql
spinach mysql 7 10: *spinach-knapsack-mysql
spinach mysql 8 10: *spinach-knapsack-mysql
spinach mysql 9 10: *spinach-knapsack-mysql
235 236 237 238 239 240 241 242 243 244

# Other generic tests
.ruby-static-analysis: &ruby-static-analysis
  variables:
    SIMPLECOV: "false"
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"

.exec: &exec
  <<: *ruby-static-analysis
245
  <<: *dedicated-runner
246 247
  stage: test
  script:
248
    - bundle exec $CI_JOB_NAME
249

250
rubocop:
251 252 253 254 255 256
  <<: *ruby-static-analysis
  <<: *dedicated-runner
  stage: test
  script:
    - bundle exec "rubocop --require rubocop-rspec"

257 258
rake haml_lint: *exec
rake scss_lint: *exec
259
rake config_lint: *exec
260 261 262
rake brakeman: *exec
rake flay: *exec
license_finder: *exec
263 264 265 266 267 268 269
rake downtime_check:
  <<: *exec
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/

270 271 272 273 274 275 276 277 278 279
rake ee_compat_check:
  <<: *exec
  only:
    - branches@gitlab-org/gitlab-ce
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
  allow_failure: yes
  cache:
280
    key: "ee_compat_check_repo"
281
    paths:
282
      - ee_compat_check/ee-repo/
283
  artifacts:
284
    name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
285 286 287 288 289
    when: on_failure
    expire_in: 10d
    paths:
      - ee_compat_check/patches/*.patch

290
.db-migrate-reset: &db-migrate-reset
291
  stage: test
292
  <<: *dedicated-runner
293
  script:
294
    - bundle exec rake db:migrate:reset
295

296
rake pg db:migrate:reset:
297 298 299
  <<: *db-migrate-reset
  <<: *use-pg

300
rake mysql db:migrate:reset:
301 302 303 304
  <<: *db-migrate-reset
  <<: *use-mysql

.db-rollback: &db-rollback
305 306 307
  stage: test
  <<: *dedicated-runner
  script:
308 309
    - bundle exec rake db:rollback STEP=120
    - bundle exec rake db:migrate
310

311
rake pg db:rollback:
312 313 314
  <<: *db-rollback
  <<: *use-pg

315
rake mysql db:rollback:
316 317 318 319
  <<: *db-rollback
  <<: *use-mysql

.db-seed_fu: &db-seed_fu
320
  stage: test
321
  <<: *dedicated-runner
322 323 324 325 326 327 328 329 330 331 332 333 334 335
  variables:
    SIZE: "1"
    SETUP_DB: "false"
    RAILS_ENV: "development"
  script:
    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
       /home/git/repositories/gitlab-org/gitlab-test.git
    - bundle exec rake db:setup db:seed_fu
  artifacts:
    when: on_failure
    expire_in: 1d
    paths:
      - log/development.log

336
rake pg db:seed_fu:
337 338 339
  <<: *db-seed_fu
  <<: *use-pg

340
rake mysql db:seed_fu:
341 342 343
  <<: *db-seed_fu
  <<: *use-mysql

344 345 346 347 348 349 350 351 352 353
rake gitlab:assets:compile:
  stage: test
  <<: *dedicated-runner
  dependencies: []
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"
    SETUP_DB: "false"
    USE_DB: "false"
    SKIP_STORAGE_VALIDATION: "true"
354 355 356 357 358 359 360 361
    WEBPACK_REPORT: "true"
  script:
    - bundle exec rake yarn:install gitlab:assets:compile
  artifacts:
    name: webpack-report
    expire_in: 31d
    paths:
    - webpack-report/
362

Clement Ho's avatar
Clement Ho committed
363
rake karma:
364 365 366
  cache:
    paths:
      - vendor/ruby
367
  stage: test
368
  <<: *use-pg
369
  <<: *dedicated-runner
370 371
  variables:
    BABEL_ENV: "coverage"
372
  script:
373
    - bundle exec rake karma
374
  coverage: '/^Statements *: (\d+\.\d+%)/'
375 376 377 378
  artifacts:
    name: coverage-javascript
    expire_in: 31d
    paths:
379
    - coverage-javascript/
380

381 382
docs:check:apilint:
  image: "phusion/baseimage"
383
  stage: test
384
  <<: *dedicated-runner
385 386
  cache: {}
  dependencies: []
387 388 389 390
  before_script: []
  script:
    - scripts/lint-doc.sh

391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
docs:check:links:
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
  stage: test
  <<: *dedicated-runner
  cache: {}
  dependencies: []
  before_script: []
  script:
    - mv doc/ /nanoc/content/
    - cd /nanoc
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links

406 407 408
bundler:audit:
  stage: test
  <<: *ruby-static-analysis
409
  <<: *dedicated-runner
410 411 412 413 414 415
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
  script:
416
    - "bundle exec bundle-audit check --update --ignore CVE-2016-4658"
417

418
.migration-paths: &migration-paths
419
  stage: test
420
  <<: *dedicated-runner
421 422
  variables:
    SETUP_DB: "false"
423 424 425 426 427
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
428
  script:
429
    - git fetch origin v8.14.10
Kamil Trzcinski's avatar
Kamil Trzcinski committed
430
    - git checkout -f FETCH_HEAD
431
    - bundle install $BUNDLE_INSTALL_FLAGS
432
    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
433
    - git checkout $CI_COMMIT_SHA
434
    - bundle install $BUNDLE_INSTALL_FLAGS
435
    - . scripts/prepare_build.sh
436
    - bundle exec rake db:migrate
437

438
migration pg paths:
439 440 441
  <<: *migration-paths
  <<: *use-pg

442
migration mysql paths:
443 444 445
  <<: *migration-paths
  <<: *use-mysql

446 447 448
coverage:
  stage: post-test
  services: []
449
  <<: *dedicated-runner
450 451 452 453 454
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"
  script:
    - bundle exec scripts/merge-simplecov
455
  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
456 457 458 459 460 461 462
  artifacts:
    name: coverage
    expire_in: 31d
    paths:
    - coverage/index.html
    - coverage/assets/

463
lint:javascript:
464
  <<: *dedicated-runner
465
  stage: test
466
  before_script: []
467
  script:
468
    - yarn run eslint
469 470

lint:javascript:report:
471
  <<: *dedicated-runner
472
  stage: post-test
473
  before_script: []
474
  script:
475
    - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files
476
    - yarn run eslint-report || true # ignore exit code
477 478 479 480 481
  artifacts:
    name: eslint-report
    expire_in: 31d
    paths:
    - eslint-report.html
482 483 484 485 486 487

# Trigger docs build
# https://gitlab.com/gitlab-com/doc-gitlab-com/blob/master/README.md#deployment-process
trigger_docs:
  stage: post-test
  image: "alpine"
488
  <<: *dedicated-runner
489 490 491
  before_script:
    - apk update && apk add curl
  variables:
492
    GIT_STRATEGY: "none"
493 494 495
  cache: {}
  artifacts: {}
  script:
496 497
    - "HTTP_STATUS=$(curl -X POST -F token=${DOCS_TRIGGER_TOKEN} -F ref=master -F variables[PROJECT]=${CI_PROJECT_NAME} --silent --output curl.log --write-out '%{http_code}' https://gitlab.com/api/v3/projects/1794617/trigger/builds)"
    - if [ "${HTTP_STATUS}" -ne "201" ]; then echo "Error ${HTTP_STATUS}"; cat curl.log; echo; exit 1; fi
498 499
  only:
    - master@gitlab-org/gitlab-ce
500
    - master@gitlab-org/gitlab-ee
501 502 503 504

# Notify slack in the end
notify:slack:
  stage: post-test
505
  <<: *dedicated-runner
506 507 508 509
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
  script:
510
    - ./scripts/notify_slack.sh "#development" "Build on \`$CI_COMMIT_REF_NAME\` failed! Commit \`$(git log -1 --oneline)\` See <https://gitlab.com/gitlab-org/$(basename "$PWD")/commit/"$CI_COMMIT_SHA"/pipelines>"
511 512 513 514 515 516 517 518 519 520
  when: on_failure
  only:
    - master@gitlab-org/gitlab-ce
    - tags@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - tags@gitlab-org/gitlab-ee

pages:
  before_script: []
  stage: pages
521
  <<: *dedicated-runner
522 523
  dependencies:
    - coverage
Clement Ho's avatar
Clement Ho committed
524
    - rake karma
525
    - rake gitlab:assets:compile
526
    - lint:javascript:report
527 528 529
  script:
    - mv public/ .public/
    - mkdir public/
530
    - mv coverage/ public/coverage-ruby/ || true
531
    - mv coverage-javascript/ public/coverage-javascript/ || true
532
    - mv eslint-report.html public/ || true
533
    - mv webpack-report/ public/webpack-report/ || true
534 535 536 537
  artifacts:
    paths:
      - public
  only:
538
    - master@gitlab-org/gitlab-ce
539
    - master@gitlab-org/gitlab-ee
540 541 542 543

# Insurance in case a gem needed by one of our releases gets yanked from
# rubygems.org in the future.
cache gems:
544
  <<: *dedicated-runner
545 546 547 548 549 550 551 552 553
  only:
    - tags
  variables:
    SETUP_DB: "false"
  script:
    - bundle package --all --all-platforms
  artifacts:
    paths:
      - vendor/cache
554 555
  only:
    - master@gitlab-org/gitlab-ce
556
    - master@gitlab-org/gitlab-ee