diff --git a/.github/workflows/test-core.yml b/.github/workflows/test-core.yml index 3242a2d566a..831989b4b78 100644 --- a/.github/workflows/test-core.yml +++ b/.github/workflows/test-core.yml @@ -28,47 +28,55 @@ jobs: env: DOCKER_BUILDKIT: 1 LOCAL_DEV_CHECK: 1 - CI_CACHE_PATH: "/tmp/cache" CI_RETRY_COUNT: 3 + CAPYBARA_DOWNLOADED_FILE_DIR: /tmp/ci/downloads CAPYBARA_AWS_ACCESS_KEY_ID: "${{ secrets.CAPYBARA_AWS_ACCESS_KEY_ID }}" CAPYBARA_AWS_SECRET_ACCESS_KEY: "${{ secrets.CAPYBARA_AWS_SECRET_ACCESS_KEY }}" steps: - - uses: actions/checkout@v3 - - name: cache + - uses: actions/checkout@v4 + - name: Cache GEM uses: actions/cache@v3 with: - path: /tmp/cache - key: ${{ runner.os }}-ci-cache-tests-all-${{ github.ref }}-${{ github.sha }} + path: cache/bundle + key: gem-${{ hashFiles('Gemfile.lock') }} restore-keys: | - ${{ runner.os }}-ci-cache-tests-all-${{ github.ref }}- - ${{ runner.os }}-ci-cache-tests-all- - - name: build - run: docker-compose -f docker-compose.ci.yml build --pull ci - - name: tests - run: | - sudo mkdir -p /tmp/cache && sudo chown -R 1000:1000 $CI_CACHE_PATH - ls -lrt $CI_CACHE_PATH/op-cache/ || true - docker-compose -f docker-compose.ci.yml run ci setup-tests run-units run-features - - name: cleanup + gem- + - name: Cache NPM + uses: actions/cache@v3 + with: + path: cache/npm + key: npm-${{ hashFiles('frontend/package-lock.json') }} + restore-keys: | + npm- + - name: Cache ANGULAR + uses: actions/cache@v3 + with: + path: cache/angular + key: angular-${{ hashFiles('frontend/package-lock.json') }} + restore-keys: | + angular- + - name: Cache TEST RUNTIME + uses: actions/cache@v3 + with: + path: cache/runtime-logs + key: runtime-logs-${{ github.head_ref || github.ref }}-${{ github.ref }} + restore-keys: | + runtime-logs-${{ github.head_ref || github.ref }}- + runtime-logs- + - name: Build + run: bin/ci setup-tests + - name: APIv3 specification (OpenAPI 3.0) + run: bin/ci ./script/api/validate_spec + - name: Unit tests + run: bin/ci run-units + - name: Feature tests + run: bin/ci run-features + - name: Cleanup if: ${{ always() }} run: | - docker-compose -f docker-compose.ci.yml down --remove-orphans -t 10 - sudo chown -R $(id -u):$(id -g) $CI_CACHE_PATH - ls -lrt spec/support/turbo* - ls -lrt $CI_CACHE_PATH/op-cache/ || true - api-spec: - name: APIv3 specification (OpenAPI 3.0) - if: github.repository == 'opf/openproject' - runs-on: [ubuntu-latest] - steps: - - uses: actions/checkout@v3 - - uses: ruby/setup-ruby@v1 - with: - bundler-cache: true - - uses: actions/setup-node@v3 - with: - node-version: '14' - - run: ./script/api/validate_spec + ls -al cache/runtime-logs || true + ls -al cache/ || true + du -sh cache/* || true # github.head_ref is only availabe in PR context and if it is absent then github.run_id # is used . And github.run_id is unique for each workflow run. So, this option makes diff --git a/Gemfile.lock b/Gemfile.lock index dd6d0e64205..6446702f784 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1206,4 +1206,4 @@ RUBY VERSION ruby 3.2.1p31 BUNDLED WITH - 2.4.7 + 2.4.6 diff --git a/bin/ci b/bin/ci new file mode 100755 index 00000000000..4b11f4a3b84 --- /dev/null +++ b/bin/ci @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +export DOCKER_BUILDKIT=1 +export APP_USER_UID=$(id -u) +export APP_USER_GID=$(id -g) +export COMPOSE_FILE=docker-compose.ci.yml +export RUBY_VERSION="$(cat .ruby-version)" +export LOCAL_CACHE_PATH="./cache" + +mkdir -p $LOCAL_CACHE_PATH/{bundle,npm,angular,runtime-logs} + +if [ "$1" = "down" ]; then + exec docker compose down --remove-orphans --volumes +else + exec docker compose run --rm --remove-orphans --build ci "$@" +fi \ No newline at end of file diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 961780e190e..0acc6eaf89a 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -3,20 +3,38 @@ version: "3.7" networks: testing: +name: "openproject-ci" + services: + db: + image: postgres:13 + tmpfs: + - /var/lib/postgresql/data + volumes: + - ./docker/ci/postgresql.conf:/etc/postgresql/postgresql.conf + environment: + POSTGRES_PASSWORD: p4ssw0rd ci: build: context: . dockerfile: ./docker/ci/Dockerfile + args: + - APP_USER_UID + - APP_USER_GID + - RUBY_VERSION environment: CI_JOBS: "${CI_JOBS}" RSPEC_RETRY_RETRY_COUNT: "${CI_RETRY_COUNT:-3}" CAPYBARA_AWS_ACCESS_KEY_ID: "${CAPYBARA_AWS_ACCESS_KEY_ID}" CAPYBARA_AWS_SECRET_ACCESS_KEY: "${CAPYBARA_AWS_SECRET_ACCESS_KEY}" + PGHOST: db tmpfs: - "/tmp" + depends_on: + - db volumes: - # the spec folder is docker-ignored, so we need to mount it explicitly - - "./spec:/app/spec" - # used to store stuff that must persist between CI runs, e.g. turbo_tests runtime logs - - "${CI_CACHE_PATH:-/tmp}/op-cache:/cache" + - .:/app + - ${LOCAL_CACHE_PATH}/bundle:/usr/local/bundle + - ${LOCAL_CACHE_PATH}/npm:/app/.npm + - ${LOCAL_CACHE_PATH}/angular:/app/frontend/.angular/cache + - ${LOCAL_CACHE_PATH}/runtime-logs:/app/spec/support/runtime-logs diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile index 94565dc3a15..8ccf12e42c7 100644 --- a/docker/ci/Dockerfile +++ b/docker/ci/Dockerfile @@ -1,45 +1,29 @@ # syntax=docker/dockerfile:1.4 -FROM ruby:3.2.1-buster as gems +ARG RUBY_VERSION +FROM ruby:${RUBY_VERSION}-bullseye -ENV BUNDLER_VERSION="2.4.7" +ENV NODE_VERSION="16.17.0" +ENV BUNDLER_VERSION="2.4.21" +ENV DEBIAN_FRONTEND=noninteractive ENV BUNDLE_WITHOUT="development:production:docker" -ENV BUNDLE_JOBS=8 -WORKDIR /app - -RUN gem install bundler --version "$BUNDLER_VERSION" --no-document - -COPY Gemfile Gemfile.modules Gemfile.lock ./ -COPY modules ./modules -RUN \ - --mount=type=cache,target=/usr/local/bundle/cache \ - bundle install - -FROM node:16.17 as nodejs - -WORKDIR /app - -COPY package.json ./ -COPY frontend/package.json frontend/package-lock.json frontend/.npmrc ./frontend/ -RUN \ - --mount=type=cache,target=/app/.npm \ - JOBS=$(nproc) npm install - -FROM ruby:3.2.1-buster as final - -RUN wget --quiet -O- https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list - -RUN --mount=type=cache,target=/var/cache/apt \ - apt-get update -qq && apt-get install -y \ - postgresql-13 postgresql-client-13 time imagemagick default-jre-headless firefox-esr +RUN apt-get update -qq && apt-get install -y postgresql-client time imagemagick default-jre-headless firefox-esr ENV CHROME_SOURCE_URL="https://dl.google.com/dl/linux/direct/google-chrome-stable_current_amd64.deb" RUN f="/tmp/chrome.deb"; wget --no-verbose -O $f $CHROME_SOURCE_URL && apt install -y $f && rm -f $f +RUN curl -L https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz -o - | tar xJf - -C /usr/local --strip=1 && node --version + +RUN gem install bundler --version "$BUNDLER_VERSION" --no-document + ENV APP_USER=dev ENV APP_PATH=/app -RUN useradd -d $APP_PATH -m $APP_USER -s /bin/bash +ARG APP_USER_UID +ARG APP_USER_GID + +RUN groupadd --force --gid $APP_USER_GID $APP_USER +RUN useradd -d $APP_PATH -m $APP_USER -s /bin/bash --uid $APP_USER_UID --gid $APP_USER_GID +RUN chown -R $APP_USER:$APP_USER /usr/local/bundle ENV CI=true ENV RAILS_ENV=test @@ -48,31 +32,9 @@ ENV CAPYBARA_DYNAMIC_BIND_IP=1 ENV CAPYBARA_DOWNLOADED_FILE_DIR=/tmp # disable deprecations and other warnings in output ENV RUBYOPT="-W0" -ENV PGVERSION=13 WORKDIR $APP_PATH - -RUN mkdir -p /cache && chown $APP_USER:$APP_USER /cache - -COPY --from=gems /usr/local/bundle /usr/local/bundle -COPY --from=nodejs /usr/local/bin/node /usr/local/bin/node -COPY --from=nodejs /usr/local/lib/node_modules /usr/local/lib/node_modules -COPY --from=nodejs /usr/local/include/node /usr/local/include/node -RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm - -COPY --from=nodejs /app/node_modules ./node_modules -COPY --from=nodejs /app/frontend/node_modules ./frontend/node_modules -COPY --chown=$APP_USER:$APP_USER . . - USER $APP_USER -ENV DATABASE_URL="postgres://appuser:p4ssw0rd@127.0.0.1/appdb" -ENV BUNDLE_WITHOUT="development:production:docker" - -RUN --mount=type=cache,target=/app/frontend/.angular/cache,uid=1000,gid=1000 DATABASE_URL=nulldb://db bin/rails openproject:plugins:register_frontend assets:precompile && \ - # only keep most current angular cache since webpack is unable to cleanup after itself - find frontend/.angular/cache -type d -exec sh -c 'ls -dt "$1"/*/ | tail -n +2 | xargs rm -r' sh {} \; -RUN cp -rp config/frontend_assets.manifest.json public/assets/frontend_assets.manifest.json -RUN cp docker/ci/database.yml config/ COPY ./docker/ci/entrypoint.sh /usr/sbin/entrypoint.sh diff --git a/docker/ci/database.yml b/docker/ci/database.yml index 6a8272ff569..98db2dd2bd8 100644 --- a/docker/ci/database.yml +++ b/docker/ci/database.yml @@ -1,2 +1,2 @@ test: - url: <%= ENV['DATABASE_URL'].sub(/\/appdb$/, "/appdb#{ENV['TEST_ENV_NUMBER']}") %> + url: <%= ENV['DATABASE_URL'].sub(/\/postgres$/, "/appdb#{ENV['TEST_ENV_NUMBER']}") %> diff --git a/docker/ci/entrypoint.sh b/docker/ci/entrypoint.sh index 095ac58dc2c..9937151f23f 100755 --- a/docker/ci/entrypoint.sh +++ b/docker/ci/entrypoint.sh @@ -1,24 +1,27 @@ #!/bin/bash set -e -export PGBIN="/usr/lib/postgresql/$PGVERSION/bin" +export PATH="/usr/lib/postgresql/$PGVERSION/bin:$PATH" export JOBS="${CI_JOBS:=$(nproc)}" # for parallel rspec export PARALLEL_TEST_PROCESSORS=$JOBS export PARALLEL_TEST_FIRST_IS_1=true export DISABLE_DATABASE_ENVIRONMENT_CHECK=1 -export NODE_OPTIONS="--max-old-space-size=8192" +# export NODE_OPTIONS="--max-old-space-size=8192" export LOG_FILE=/tmp/op-output.log +export PGUSER=${PGUSER:=postgres} +export PGHOST=${PGHOST:=localhost} +export PGPASSWORD=${PGPASSWORD:=p4ssw0rd} +export DATABASE_URL="postgres://$PGUSER:$PGPASSWORD@$PGHOST/postgres" run_psql() { - $PGBIN/psql -v ON_ERROR_STOP=1 -U dev -h 127.0.0.1 "$@" + psql -v ON_ERROR_STOP=1 "$@" } cleanup() { exit_code=$? echo "CLEANUP" rm -rf tmp/cache/parallel* - if [ -d tmp/features ]; then mv tmp/features spec/ ; fi if [ ! $exit_code -eq "0" ]; then echo "ERROR: exit code $exit_code" @@ -49,10 +52,18 @@ reset_dbs() { execute_quiet "cat db/structure.sql | run_psql -d appdb" # create and load schema for test databases "appdb1" to "appdb$JOBS", far faster than using parallel_rspec tasks for that for i in $(seq 1 $JOBS); do - execute_quiet "echo 'drop database if exists appdb$i ; create database appdb$i with template appdb owner appuser;' | run_psql -d postgres" + execute_quiet "echo 'drop database if exists appdb$i ; create database appdb$i with template appdb owner $PGUSER;' | run_psql -d postgres" done } +precompile_assets() { + execute "JOBS=8 npm install" + execute_quiet "DATABASE_URL=nulldb://db bin/rails openproject:plugins:register_frontend assets:precompile" + execute_quiet "cp -rp config/frontend_assets.manifest.json public/assets/frontend_assets.manifest.json" + # ls -al frontend/.angular/cache/ + # find frontend/.angular/cache -type d -exec sh -c 'ls -dt "$1"/*/ | tail -n +2 | xargs rm -r' sh {} \; +} + setup_tests() { echo "Preparing environment for running tests..." for i in $(seq 1 $JOBS); do @@ -60,49 +71,44 @@ setup_tests() { execute_quiet "rm -rf '$folder' ; mkdir -p '$folder' ; chmod 1777 '$folder'" done - if [ ! -d "/tmp/nulldb" ]; then - execute_quiet "$PGBIN/initdb -E UTF8 -D /tmp/nulldb" - execute_quiet "cp docker/ci/postgresql.conf /tmp/nulldb/" - execute_quiet "$PGBIN/pg_ctl -D /tmp/nulldb -l /dev/null -w start" - echo "create database appdb; create user appuser with superuser encrypted password 'p4ssw0rd'; grant all privileges on database appdb to appuser;" | run_psql -d postgres - fi + execute_quiet "cp docker/ci/database.yml config/" + execute_quiet "mkdir -p spec/support/runtime-logs/" + + execute "BUNDLE_JOBS=4 bundle install" + execute_quiet "bundle clean --force" # create test database "app" and dump schema because db/structure.sql is not checked in - execute_quiet "time bundle exec rails db:migrate db:schema:dump zeitwerk:check" + execute_quiet "time bundle exec rails db:create db:migrate db:schema:dump zeitwerk:check" # pre-cache browsers and their drivers binaries execute "$(bundle show selenium)/bin/linux/selenium-manager --browser chrome --debug" execute "$(bundle show selenium)/bin/linux/selenium-manager --browser firefox --debug" + + precompile_assets } run_units() { shopt -s extglob reset_dbs - execute_quiet "cp -f /cache/turbo_runtime_units.log spec/support/ || true" - execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/turbo_runtime_units.log spec/!(features) modules/**/spec/!(features)" - execute_quiet "cp -f spec/support/turbo_runtime_units.log /cache/ || true" + execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/runtime-logs/turbo_runtime_units.log spec/!(features) modules/**/spec/!(features)" cleanup } run_features() { shopt -s extglob reset_dbs - execute_quiet "cp -f /cache/turbo_runtime_features.log spec/support/ || true" - execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/turbo_runtime_features.log spec/features modules/**/spec/features" - execute_quiet "cp -f spec/support/turbo_runtime_features.log /cache/ || true" + execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/runtime-logs/turbo_runtime_features.log spec/features modules/**/spec/features" cleanup } run_all() { shopt -s globstar reset_dbs - execute_quiet "cp -f /cache/turbo_runtime_all.log spec/support/ || true" - execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/turbo_runtime_all.log spec modules/**/spec" - execute_quiet "cp -f spec/support/turbo_runtime_all.log /cache/ || true" + execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/runtime-logs/turbo_runtime_all.log spec modules/**/spec" cleanup } -export -f cleanup execute execute_quiet run_psql reset_dbs setup_tests run_units run_features +export -f cleanup execute execute_quiet run_psql reset_dbs setup_tests precompile_assets run_units run_features if [ "$1" == "setup-tests" ]; then shift diff --git a/docker/ci/postgresql.conf b/docker/ci/postgresql.conf index 666903da5f3..39e33488f1e 100644 --- a/docker/ci/postgresql.conf +++ b/docker/ci/postgresql.conf @@ -6,7 +6,10 @@ # Connections num: 200 # Data Storage: ssd +# https://www.postgresql.org/docs/current/non-durability.html fsync = off +synchronous_commit = off +checkpoint_timeout = 30min full_page_writes = off deadlock_timeout = 20s autovacuum = off