.gitlab-ci.yml 15.1 KB
Newer Older
1
image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
2

3
cache:
Z.J. van de Weg's avatar
Z.J. van de Weg committed
4
  key: "ruby-233"
5
  paths:
6
  - vendor/ruby
7

8 9
variables:
  MYSQL_ALLOW_EMPTY_PASSWORD: "1"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
10
  RAILS_ENV: "test"
Mike Greiling's avatar
Mike Greiling committed
11
  NODE_ENV: "test"
Kamil Trzcinski's avatar
Kamil Trzcinski committed
12
  SIMPLECOV: "true"
13
  GIT_DEPTH: "20"
14
  GIT_SUBMODULE_STRATEGY: "none"
15
  PHANTOMJS_VERSION: "2.1.1"
16
  GET_SOURCES_ATTEMPTS: "3"
17 18
  KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json
  KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json
19

Dmitriy Zaporozhets's avatar
Dmitriy Zaporozhets committed
20
before_script:
Kamil Trzcinski's avatar
Test  
Kamil Trzcinski committed
21
  - bundle --version
22 23
  - source scripts/utils.sh
  - source scripts/prepare_build.sh
24

Kamil Trzcinski's avatar
Kamil Trzcinski committed
25
stages:
26
- build
Kamil Trzcinski's avatar
Kamil Trzcinski committed
27
- prepare
Kamil Trzcinski's avatar
Kamil Trzcinski committed
28
- test
29
- post-test
30
- pages
31

32 33 34 35 36
# Predefined scopes
.dedicated-runner: &dedicated-runner
  tags:
    - gitlab-org

37 38 39 40 41
.knapsack-state: &knapsack-state
  services: []
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
42
    KNAPSACK_S3_BUCKET: "gitlab-ce-cache"
43 44 45
  cache:
    key: "knapsack"
    paths:
46
      - knapsack/
47 48 49
  artifacts:
    expire_in: 31d
    paths:
50
      - knapsack/
51

52 53 54 55 56 57
.use-pg: &use-pg
  services:
    - postgres:latest
    - redis:alpine

.use-mysql: &use-mysql
58 59 60
  services:
    - mysql:latest
    - redis:alpine
61

62 63
.only-master-and-ee-or-mysql: &only-master-and-ee-or-mysql
  only:
64
    - /mysql/
65 66 67 68 69 70 71
    - master@gitlab-org/gitlab-ce
    - master@gitlab/gitlabhq
    - tags@gitlab-org/gitlab-ce
    - tags@gitlab/gitlabhq
    - //@gitlab-org/gitlab-ee
    - //@gitlab/gitlab-ee

72 73 74
# Skip all jobs except the ones that begin with 'docs/'.
# Used for commits including ONLY documentation changes.
# https://docs.gitlab.com/ce/development/writing_documentation.html#testing
75 76
.except-docs: &except-docs
  except:
77
    - /^docs\/.*/
78

79 80
.rspec-knapsack: &rspec-knapsack
  stage: test
81
  <<: *dedicated-runner
82
  script:
83
    - JOB_NAME=( $CI_JOB_NAME )
84 85
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
86
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
87
    - export KNAPSACK_GENERATE_REPORT=true
88
    - export CACHE_CLASSES=true
89
    - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
90 91 92
    - knapsack rspec "--color --format documentation"
  artifacts:
    expire_in: 31d
93
    when: always
94
    paths:
95 96 97
      - coverage/
      - knapsack/
      - tmp/capybara/
98

99 100 101
.rspec-knapsack-pg: &rspec-knapsack-pg
  <<: *rspec-knapsack
  <<: *use-pg
102
  <<: *except-docs
103 104 105 106

.rspec-knapsack-mysql: &rspec-knapsack-mysql
  <<: *rspec-knapsack
  <<: *use-mysql
107
  <<: *only-master-and-ee-or-mysql
108
  <<: *except-docs
109

110 111
.spinach-knapsack: &spinach-knapsack
  stage: test
112
  <<: *dedicated-runner
113
  script:
114
    - JOB_NAME=( $CI_JOB_NAME )
115 116
    - export CI_NODE_INDEX=${JOB_NAME[-2]}
    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
117
    - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_${JOB_NAME[1]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
118
    - export KNAPSACK_GENERATE_REPORT=true
119
    - export CACHE_CLASSES=true
120
    - cp ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH}
121 122 123
    - knapsack spinach "-r rerun" || retry '[[ -e tmp/spinach-rerun.txt ]] && bundle exec spinach -r rerun $(cat tmp/spinach-rerun.txt)'
  artifacts:
    expire_in: 31d
124
    when: always
125
    paths:
126 127 128
      - coverage/
      - knapsack/
      - tmp/capybara/
129

130 131 132
.spinach-knapsack-pg: &spinach-knapsack-pg
  <<: *spinach-knapsack
  <<: *use-pg
133
  <<: *except-docs
134 135 136 137

.spinach-knapsack-mysql: &spinach-knapsack-mysql
  <<: *spinach-knapsack
  <<: *use-mysql
138
  <<: *only-master-and-ee-or-mysql
139
  <<: *except-docs
140

141 142 143
# Trigger a package build on omnibus-gitlab repository

build-package:
144
  before_script: []
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
  services: []
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "false"
  stage: build
  when: manual
  script:
    # If no branch in omnibus is specified, trigger pipeline against master
    - if [ -z "$OMNIBUS_BRANCH" ] ; then export OMNIBUS_BRANCH=master ;fi
    - echo "token=${BUILD_TRIGGER_TOKEN}" > version_details
    - echo "ref=${OMNIBUS_BRANCH}" >> version_details
    - echo "variables[ALTERNATIVE_SOURCES]=true" >> version_details
    - echo "variables[GITLAB_VERSION]=${CI_COMMIT_SHA}" >> version_details
    # Collect version details of all components
    - for f in *_VERSION; do echo "variables[$f]=$(cat $f)" >> version_details; done
    # Trigger the API and pass values collected above as parameters to it
161 162
    - pipeline_id=$(cat version_details | tr '\n' '&' | curl -s -X POST https://gitlab.com/api/v4/projects/20699/trigger/pipeline --data-binary @- | ruby -e "require 'json'; puts JSON.parse(STDIN.read)['id']") > /dev/null
    - echo "Triggered pipeline can be found at https://gitlab.com/gitlab-org/omnibus-gitlab/pipelines/${pipeline_id}"
163 164
    - rm version_details

165 166 167 168
# Prepare and merge knapsack tests
knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
169
  <<: *except-docs
170 171
  stage: prepare
  script:
172
    - mkdir -p knapsack/${CI_PROJECT_NAME}/
173 174
    - wget -O $KNAPSACK_RSPEC_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_RSPEC_SUITE_REPORT_PATH || rm $KNAPSACK_RSPEC_SUITE_REPORT_PATH
    - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${KNAPSACK_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH
175 176 177 178 179 180 181 182
    - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}'
    - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}'

update-knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: post-test
  script:
183 184
    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec_pg_node_*.json
    - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach_pg_node_*.json
185 186 187 188 189 190 191
    - '[[ -z ${KNAPSACK_S3_BUCKET} ]] || scripts/sync-reports put $KNAPSACK_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH'
    - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
192 193

setup-test-env:
194
  <<: *use-pg
195
  <<: *dedicated-runner
196
  <<: *except-docs
197 198
  stage: prepare
  script:
199
    - node --version
200
    - yarn install --pure-lockfile
Mike Greiling's avatar
Mike Greiling committed
201
    - bundle exec rake gitlab:assets:compile
202 203 204 205
    - bundle exec ruby -Ispec -e 'require "spec_helper" ; TestEnv.init'
  artifacts:
    expire_in: 7d
    paths:
206
      - node_modules
207 208 209
      - public/assets
      - tmp/tests

210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
rspec pg 0 20: *rspec-knapsack-pg
rspec pg 1 20: *rspec-knapsack-pg
rspec pg 2 20: *rspec-knapsack-pg
rspec pg 3 20: *rspec-knapsack-pg
rspec pg 4 20: *rspec-knapsack-pg
rspec pg 5 20: *rspec-knapsack-pg
rspec pg 6 20: *rspec-knapsack-pg
rspec pg 7 20: *rspec-knapsack-pg
rspec pg 8 20: *rspec-knapsack-pg
rspec pg 9 20: *rspec-knapsack-pg
rspec pg 10 20: *rspec-knapsack-pg
rspec pg 11 20: *rspec-knapsack-pg
rspec pg 12 20: *rspec-knapsack-pg
rspec pg 13 20: *rspec-knapsack-pg
rspec pg 14 20: *rspec-knapsack-pg
rspec pg 15 20: *rspec-knapsack-pg
rspec pg 16 20: *rspec-knapsack-pg
rspec pg 17 20: *rspec-knapsack-pg
rspec pg 18 20: *rspec-knapsack-pg
rspec pg 19 20: *rspec-knapsack-pg

rspec mysql 0 20: *rspec-knapsack-mysql
rspec mysql 1 20: *rspec-knapsack-mysql
rspec mysql 2 20: *rspec-knapsack-mysql
rspec mysql 3 20: *rspec-knapsack-mysql
rspec mysql 4 20: *rspec-knapsack-mysql
rspec mysql 5 20: *rspec-knapsack-mysql
rspec mysql 6 20: *rspec-knapsack-mysql
rspec mysql 7 20: *rspec-knapsack-mysql
rspec mysql 8 20: *rspec-knapsack-mysql
rspec mysql 9 20: *rspec-knapsack-mysql
rspec mysql 10 20: *rspec-knapsack-mysql
rspec mysql 11 20: *rspec-knapsack-mysql
rspec mysql 12 20: *rspec-knapsack-mysql
rspec mysql 13 20: *rspec-knapsack-mysql
rspec mysql 14 20: *rspec-knapsack-mysql
rspec mysql 15 20: *rspec-knapsack-mysql
rspec mysql 16 20: *rspec-knapsack-mysql
rspec mysql 17 20: *rspec-knapsack-mysql
rspec mysql 18 20: *rspec-knapsack-mysql
rspec mysql 19 20: *rspec-knapsack-mysql

spinach pg 0 10: *spinach-knapsack-pg
spinach pg 1 10: *spinach-knapsack-pg
spinach pg 2 10: *spinach-knapsack-pg
spinach pg 3 10: *spinach-knapsack-pg
spinach pg 4 10: *spinach-knapsack-pg
spinach pg 5 10: *spinach-knapsack-pg
spinach pg 6 10: *spinach-knapsack-pg
spinach pg 7 10: *spinach-knapsack-pg
spinach pg 8 10: *spinach-knapsack-pg
spinach pg 9 10: *spinach-knapsack-pg

spinach mysql 0 10: *spinach-knapsack-mysql
spinach mysql 1 10: *spinach-knapsack-mysql
spinach mysql 2 10: *spinach-knapsack-mysql
spinach mysql 3 10: *spinach-knapsack-mysql
spinach mysql 4 10: *spinach-knapsack-mysql
spinach mysql 5 10: *spinach-knapsack-mysql
spinach mysql 6 10: *spinach-knapsack-mysql
spinach mysql 7 10: *spinach-knapsack-mysql
spinach mysql 8 10: *spinach-knapsack-mysql
spinach mysql 9 10: *spinach-knapsack-mysql
273 274 275 276 277 278 279 280

# Other generic tests
.ruby-static-analysis: &ruby-static-analysis
  variables:
    SIMPLECOV: "false"
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"

281
.rake-exec: &rake-exec
282
  <<: *ruby-static-analysis
283
  <<: *dedicated-runner
284
  <<: *except-docs
285 286
  stage: test
  script:
287
    - bundle exec rake $CI_JOB_NAME
288

289
static-analysis:
290 291
  <<: *ruby-static-analysis
  <<: *dedicated-runner
292
  <<: *except-docs
293 294
  stage: test
  script:
295
    - scripts/static-analysis
296

297 298 299 300
# Documentation checks:
# - Check validity of relative links
# - Make sure cURL examples in API docs use the full switches
docs lint:
301 302 303 304 305 306 307
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
  stage: test
  <<: *dedicated-runner
  cache: {}
  dependencies: []
  before_script: []
  script:
308
    - scripts/lint-doc.sh
309 310 311 312 313 314 315
    - mv doc/ /nanoc/content/
    - cd /nanoc
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links

316 317
downtime_check:
  <<: *rake-exec
318 319 320 321
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
322
    - /^docs\/*/
323

324 325
ee_compat_check:
  <<: *rake-exec
326 327 328 329 330 331 332 333
  only:
    - branches@gitlab-org/gitlab-ce
  except:
    - master
    - tags
    - /^[\d-]+-stable(-ee)?$/
  allow_failure: yes
  cache:
334
    key: "ee_compat_check_repo"
335
    paths:
336
      - ee_compat_check/ee-repo/
337
  artifacts:
338
    name: "${CI_JOB_NAME}_${CI_COMIT_REF_NAME}_${CI_COMMIT_SHA}"
339 340 341 342 343
    when: on_failure
    expire_in: 10d
    paths:
      - ee_compat_check/patches/*.patch

344
.db-migrate-reset: &db-migrate-reset
345
  stage: test
346
  <<: *dedicated-runner
347
  <<: *except-docs
348
  script:
349
    - bundle exec rake db:migrate:reset
350

351
rake pg db:migrate:reset:
352 353 354
  <<: *db-migrate-reset
  <<: *use-pg

355
rake mysql db:migrate:reset:
356 357 358 359
  <<: *db-migrate-reset
  <<: *use-mysql

.db-rollback: &db-rollback
360 361
  stage: test
  <<: *dedicated-runner
362
  <<: *except-docs
363
  script:
364 365
    - bundle exec rake db:rollback STEP=120
    - bundle exec rake db:migrate
366

367
rake pg db:rollback:
368 369 370
  <<: *db-rollback
  <<: *use-pg

371
rake mysql db:rollback:
372 373 374 375
  <<: *db-rollback
  <<: *use-mysql

.db-seed_fu: &db-seed_fu
376
  stage: test
377
  <<: *dedicated-runner
378
  <<: *except-docs
379 380 381 382 383 384 385 386 387 388 389 390 391 392
  variables:
    SIZE: "1"
    SETUP_DB: "false"
    RAILS_ENV: "development"
  script:
    - git clone https://gitlab.com/gitlab-org/gitlab-test.git
       /home/git/repositories/gitlab-org/gitlab-test.git
    - bundle exec rake db:setup db:seed_fu
  artifacts:
    when: on_failure
    expire_in: 1d
    paths:
      - log/development.log

393
rake pg db:seed_fu:
394 395 396
  <<: *db-seed_fu
  <<: *use-pg

397
rake mysql db:seed_fu:
398 399 400
  <<: *db-seed_fu
  <<: *use-mysql

401
rake gitlab:assets:compile:
402 403
  stage: test
  <<: *dedicated-runner
404
  <<: *except-docs
405 406 407 408 409 410 411
  dependencies: []
  variables:
    NODE_ENV: "production"
    RAILS_ENV: "production"
    SETUP_DB: "false"
    USE_DB: "false"
    SKIP_STORAGE_VALIDATION: "true"
412 413 414 415 416 417 418 419
    WEBPACK_REPORT: "true"
  script:
    - bundle exec rake yarn:install gitlab:assets:compile
  artifacts:
    name: webpack-report
    expire_in: 31d
    paths:
    - webpack-report/
420

421
rake karma:
422 423 424
  cache:
    paths:
      - vendor/ruby
425
  stage: test
426
  <<: *use-pg
427
  <<: *dedicated-runner
428
  <<: *except-docs
429 430
  variables:
    BABEL_ENV: "coverage"
431
  script:
432
    - bundle exec rake karma
433
  coverage: '/^Statements *: (\d+\.\d+%)/'
434 435 436 437
  artifacts:
    name: coverage-javascript
    expire_in: 31d
    paths:
438
    - coverage-javascript/
439

440
.migration-paths: &migration-paths
Z.J. van de Weg's avatar
Z.J. van de Weg committed
441
  stage: test
442
  <<: *dedicated-runner
443 444
  variables:
    SETUP_DB: "false"
445 446 447 448 449
  only:
    - master@gitlab-org/gitlab-ce
    - master@gitlab-org/gitlab-ee
    - master@gitlab/gitlabhq
    - master@gitlab/gitlab-ee
Z.J. van de Weg's avatar
Z.J. van de Weg committed
450
  script:
Z.J. van de Weg's avatar
Z.J. van de Weg committed
451
    - git fetch origin v8.14.10
Kamil Trzcinski's avatar
Kamil Trzcinski committed
452
    - git checkout -f FETCH_HEAD
453
    - bundle install $BUNDLE_INSTALL_FLAGS
454
    - bundle exec rake db:drop db:create db:schema:load db:seed_fu
455
    - git checkout $CI_COMMIT_SHA
456
    - bundle install $BUNDLE_INSTALL_FLAGS
457
    - . scripts/prepare_build.sh
458
    - bundle exec rake db:migrate
459

460
migration pg paths:
461 462 463
  <<: *migration-paths
  <<: *use-pg

464
migration mysql paths:
465 466 467
  <<: *migration-paths
  <<: *use-mysql

468 469 470
coverage:
  stage: post-test
  services: []
471
  <<: *dedicated-runner
472
  <<: *except-docs
473 474 475 476 477
  variables:
    SETUP_DB: "false"
    USE_BUNDLE_INSTALL: "true"
  script:
    - bundle exec scripts/merge-simplecov
478
  coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
479 480 481 482 483 484 485
  artifacts:
    name: coverage
    expire_in: 31d
    paths:
    - coverage/index.html
    - coverage/assets/

486
lint:javascript:report:
487
  <<: *dedicated-runner
488
  <<: *except-docs
489
  stage: post-test
490
  before_script: []
491
  script:
492
    - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files
493
    - yarn run eslint-report || true # ignore exit code
494 495 496 497 498
  artifacts:
    name: eslint-report
    expire_in: 31d
    paths:
    - eslint-report.html
499 500 501 502 503 504

# Trigger docs build
# https://gitlab.com/gitlab-com/doc-gitlab-com/blob/master/README.md#deployment-process
trigger_docs:
  stage: post-test
  image: "alpine"
505
  <<: *dedicated-runner
506 507 508
  before_script:
    - apk update && apk add curl
  variables:
509
    GIT_STRATEGY: "none"
510 511 512
  cache: {}
  artifacts: {}
  script:
513 514
    - "HTTP_STATUS=$(curl -X POST -F token=${DOCS_TRIGGER_TOKEN} -F ref=master -F variables[PROJECT]=${CI_PROJECT_NAME} --silent --output curl.log --write-out '%{http_code}' https://gitlab.com/api/v3/projects/1794617/trigger/builds)"
    - if [ "${HTTP_STATUS}" -ne "201" ]; then echo "Error ${HTTP_STATUS}"; cat curl.log; echo; exit 1; fi
515 516
  only:
    - master@gitlab-org/gitlab-ce
517
    - master@gitlab-org/gitlab-ee
518 519 520 521

pages:
  before_script: []
  stage: pages
522
  <<: *dedicated-runner
523 524
  dependencies:
    - coverage
525 526
    - rake karma
    - rake gitlab:assets:compile
527
    - lint:javascript:report
528 529 530
  script:
    - mv public/ .public/
    - mkdir public/
531
    - mv coverage/ public/coverage-ruby/ || true
532
    - mv coverage-javascript/ public/coverage-javascript/ || true
533
    - mv eslint-report.html public/ || true
534
    - mv webpack-report/ public/webpack-report/ || true
535 536 537 538
  artifacts:
    paths:
      - public
  only:
539
    - master@gitlab-org/gitlab-ce
540
    - master@gitlab-org/gitlab-ee
541 542 543 544

# Insurance in case a gem needed by one of our releases gets yanked from
# rubygems.org in the future.
cache gems:
545
  <<: *dedicated-runner
546 547 548 549 550 551 552 553 554
  only:
    - tags
  variables:
    SETUP_DB: "false"
  script:
    - bundle package --all --all-platforms
  artifacts:
    paths:
      - vendor/cache
555 556
  only:
    - master@gitlab-org/gitlab-ce
557
    - master@gitlab-org/gitlab-ee