Improve CI cache support (#14022)

Improve CI cache support

.github/workflows/test-core.yml

@@ -28,47 +28,55 @@ jobs:
     env:
       DOCKER_BUILDKIT: 1
       LOCAL_DEV_CHECK: 1
-      CI_CACHE_PATH: "/tmp/cache"
       CI_RETRY_COUNT: 3
+      CAPYBARA_DOWNLOADED_FILE_DIR: /tmp/ci/downloads
       CAPYBARA_AWS_ACCESS_KEY_ID: "${{ secrets.CAPYBARA_AWS_ACCESS_KEY_ID }}"
       CAPYBARA_AWS_SECRET_ACCESS_KEY: "${{ secrets.CAPYBARA_AWS_SECRET_ACCESS_KEY }}"
     steps:
-    - uses: actions/checkout@v3
-    - name: cache
+    - uses: actions/checkout@v4
+    - name: Cache GEM
       uses: actions/cache@v3
       with:
-        path: /tmp/cache
-        key: ${{ runner.os }}-ci-cache-tests-all-${{ github.ref }}-${{ github.sha }}
+        path: cache/bundle
+        key: gem-${{ hashFiles('Gemfile.lock') }}
         restore-keys: |
-          ${{ runner.os }}-ci-cache-tests-all-${{ github.ref }}-
-          ${{ runner.os }}-ci-cache-tests-all-
-    - name: build
-      run: docker-compose -f docker-compose.ci.yml build --pull ci
-    - name: tests
-      run: |
-        sudo mkdir -p /tmp/cache && sudo chown -R 1000:1000 $CI_CACHE_PATH
-        ls -lrt $CI_CACHE_PATH/op-cache/ || true
-        docker-compose -f docker-compose.ci.yml run ci setup-tests run-units run-features
-    - name: cleanup
+          gem-
+    - name: Cache NPM
+      uses: actions/cache@v3
+      with:
+        path: cache/npm
+        key: npm-${{ hashFiles('frontend/package-lock.json') }}
+        restore-keys: |
+          npm-
+    - name: Cache ANGULAR
+      uses: actions/cache@v3
+      with:
+        path: cache/angular
+        key: angular-${{ hashFiles('frontend/package-lock.json') }}
+        restore-keys: |
+          angular-
+    - name: Cache TEST RUNTIME
+      uses: actions/cache@v3
+      with:
+        path: cache/runtime-logs
+        key: runtime-logs-${{ github.head_ref || github.ref }}-${{ github.ref }}
+        restore-keys: |
+          runtime-logs-${{ github.head_ref || github.ref }}-
+          runtime-logs-
+    - name: Build
+      run: bin/ci setup-tests
+    - name: APIv3 specification (OpenAPI 3.0)
+      run: bin/ci ./script/api/validate_spec
+    - name: Unit tests
+      run: bin/ci run-units
+    - name: Feature tests
+      run: bin/ci run-features
+    - name: Cleanup
       if: ${{ always() }}
       run: |
-        docker-compose -f docker-compose.ci.yml down --remove-orphans -t 10
-        sudo chown -R $(id -u):$(id -g) $CI_CACHE_PATH
-        ls -lrt spec/support/turbo*
-        ls -lrt $CI_CACHE_PATH/op-cache/ || true
-  api-spec:
-    name: APIv3 specification (OpenAPI 3.0)
-    if: github.repository == 'opf/openproject'
-    runs-on: [ubuntu-latest]
-    steps:
-    - uses: actions/checkout@v3
-    - uses: ruby/setup-ruby@v1
-      with:
-        bundler-cache: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: '14'
-    - run: ./script/api/validate_spec
+        ls -al cache/runtime-logs || true
+        ls -al cache/ || true
+        du -sh cache/* || true
 
 # github.head_ref is only availabe in PR context and if it is absent then github.run_id
 # is used . And github.run_id is unique for each workflow run. So, this option makes

Gemfile.lock

@@ -1206,4 +1206,4 @@ RUBY VERSION
    ruby 3.2.1p31
 
 BUNDLED WITH
-   2.4.7
+   2.4.6

bin/ci

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+export DOCKER_BUILDKIT=1
+export APP_USER_UID=$(id -u)
+export APP_USER_GID=$(id -g)
+export COMPOSE_FILE=docker-compose.ci.yml
+export RUBY_VERSION="$(cat .ruby-version)"
+export LOCAL_CACHE_PATH="./cache"
+
+mkdir -p $LOCAL_CACHE_PATH/{bundle,npm,angular,runtime-logs}
+
+if [ "$1" = "down" ]; then
+  exec docker compose down --remove-orphans --volumes
+else
+  exec docker compose run --rm --remove-orphans --build ci "$@"
+fi

docker-compose.ci.yml

@@ -3,20 +3,38 @@ version: "3.7"
 networks:
   testing:
 
+name: "openproject-ci"
+
 services:
+  db:
+    image: postgres:13
+    tmpfs:
+      - /var/lib/postgresql/data
+    volumes:
+      - ./docker/ci/postgresql.conf:/etc/postgresql/postgresql.conf
+    environment:
+      POSTGRES_PASSWORD: p4ssw0rd
   ci:
     build:
       context: .
       dockerfile: ./docker/ci/Dockerfile
+      args:
+        - APP_USER_UID
+        - APP_USER_GID
+        - RUBY_VERSION
     environment:
       CI_JOBS: "${CI_JOBS}"
       RSPEC_RETRY_RETRY_COUNT: "${CI_RETRY_COUNT:-3}"
       CAPYBARA_AWS_ACCESS_KEY_ID: "${CAPYBARA_AWS_ACCESS_KEY_ID}"
       CAPYBARA_AWS_SECRET_ACCESS_KEY: "${CAPYBARA_AWS_SECRET_ACCESS_KEY}"
+      PGHOST: db
     tmpfs:
       - "/tmp"
+    depends_on:
+      - db
     volumes:
-      # the spec folder is docker-ignored, so we need to mount it explicitly
-      - "./spec:/app/spec"
-      # used to store stuff that must persist between CI runs, e.g. turbo_tests runtime logs
-      - "${CI_CACHE_PATH:-/tmp}/op-cache:/cache"
+      - .:/app
+      - ${LOCAL_CACHE_PATH}/bundle:/usr/local/bundle
+      - ${LOCAL_CACHE_PATH}/npm:/app/.npm
+      - ${LOCAL_CACHE_PATH}/angular:/app/frontend/.angular/cache
+      - ${LOCAL_CACHE_PATH}/runtime-logs:/app/spec/support/runtime-logs

docker/ci/Dockerfile

@@ -1,45 +1,29 @@
 # syntax=docker/dockerfile:1.4
-FROM ruby:3.2.1-buster as gems
+ARG RUBY_VERSION
+FROM ruby:${RUBY_VERSION}-bullseye
 
-ENV BUNDLER_VERSION="2.4.7"
+ENV NODE_VERSION="16.17.0"
+ENV BUNDLER_VERSION="2.4.21"
+ENV DEBIAN_FRONTEND=noninteractive
 ENV BUNDLE_WITHOUT="development:production:docker"
-ENV BUNDLE_JOBS=8
 
-WORKDIR /app
-
-RUN gem install bundler --version "$BUNDLER_VERSION" --no-document
-
-COPY Gemfile Gemfile.modules Gemfile.lock ./
-COPY modules ./modules
-RUN \
-  --mount=type=cache,target=/usr/local/bundle/cache \
-  bundle install
-
-FROM node:16.17 as nodejs
-
-WORKDIR /app
-
-COPY package.json ./
-COPY frontend/package.json frontend/package-lock.json frontend/.npmrc ./frontend/
-RUN \
-  --mount=type=cache,target=/app/.npm \
-  JOBS=$(nproc) npm install
-
-FROM ruby:3.2.1-buster as final
-
-RUN wget --quiet -O- https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list
-
-RUN --mount=type=cache,target=/var/cache/apt \
-    apt-get update -qq && apt-get install -y \
-    postgresql-13 postgresql-client-13 time imagemagick default-jre-headless firefox-esr
+RUN apt-get update -qq && apt-get install -y postgresql-client time imagemagick default-jre-headless firefox-esr
 
 ENV CHROME_SOURCE_URL="https://dl.google.com/dl/linux/direct/google-chrome-stable_current_amd64.deb"
 RUN f="/tmp/chrome.deb"; wget --no-verbose -O $f $CHROME_SOURCE_URL && apt install -y $f && rm -f $f
 
+RUN curl -L https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz -o - | tar xJf - -C /usr/local --strip=1 && node --version
+
+RUN gem install bundler --version "$BUNDLER_VERSION" --no-document
+
 ENV APP_USER=dev
 ENV APP_PATH=/app
-RUN useradd -d $APP_PATH -m $APP_USER -s /bin/bash
+ARG APP_USER_UID
+ARG APP_USER_GID
+
+RUN groupadd --force --gid $APP_USER_GID $APP_USER
+RUN useradd -d $APP_PATH -m $APP_USER -s /bin/bash --uid $APP_USER_UID --gid $APP_USER_GID
+RUN chown -R $APP_USER:$APP_USER /usr/local/bundle
 
 ENV CI=true
 ENV RAILS_ENV=test
@@ -48,31 +32,9 @@ ENV CAPYBARA_DYNAMIC_BIND_IP=1
 ENV CAPYBARA_DOWNLOADED_FILE_DIR=/tmp
 # disable deprecations and other warnings in output
 ENV RUBYOPT="-W0"
-ENV PGVERSION=13
 
 WORKDIR $APP_PATH
-
-RUN mkdir -p /cache && chown $APP_USER:$APP_USER /cache
-
-COPY --from=gems /usr/local/bundle /usr/local/bundle
-COPY --from=nodejs /usr/local/bin/node /usr/local/bin/node
-COPY --from=nodejs /usr/local/lib/node_modules /usr/local/lib/node_modules
-COPY --from=nodejs /usr/local/include/node /usr/local/include/node
-RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm
-
-COPY --from=nodejs /app/node_modules ./node_modules
-COPY --from=nodejs /app/frontend/node_modules ./frontend/node_modules
-COPY --chown=$APP_USER:$APP_USER . .
-
 USER $APP_USER
-ENV DATABASE_URL="postgres://appuser:p4ssw0rd@127.0.0.1/appdb"
-ENV BUNDLE_WITHOUT="development:production:docker"
-
-RUN --mount=type=cache,target=/app/frontend/.angular/cache,uid=1000,gid=1000 DATABASE_URL=nulldb://db bin/rails openproject:plugins:register_frontend assets:precompile && \
-	# only keep most current angular cache since webpack is unable to cleanup after itself
-	find frontend/.angular/cache -type d -exec sh -c 'ls -dt "$1"/*/ | tail -n +2 | xargs rm -r' sh {} \;
-RUN cp -rp config/frontend_assets.manifest.json public/assets/frontend_assets.manifest.json
-RUN cp docker/ci/database.yml config/
 
 COPY ./docker/ci/entrypoint.sh /usr/sbin/entrypoint.sh
 

docker/ci/database.yml

@@ -1,2 +1,2 @@
 test:
-  url: <%= ENV['DATABASE_URL'].sub(/\/appdb$/, "/appdb#{ENV['TEST_ENV_NUMBER']}") %>
+  url: <%= ENV['DATABASE_URL'].sub(/\/postgres$/, "/appdb#{ENV['TEST_ENV_NUMBER']}") %>

docker/ci/entrypoint.sh

@@ -1,24 +1,27 @@
 #!/bin/bash
 set -e
 
-export PGBIN="/usr/lib/postgresql/$PGVERSION/bin"
+export PATH="/usr/lib/postgresql/$PGVERSION/bin:$PATH"
 export JOBS="${CI_JOBS:=$(nproc)}"
 # for parallel rspec
 export PARALLEL_TEST_PROCESSORS=$JOBS
 export PARALLEL_TEST_FIRST_IS_1=true
 export DISABLE_DATABASE_ENVIRONMENT_CHECK=1
-export NODE_OPTIONS="--max-old-space-size=8192"
+# export NODE_OPTIONS="--max-old-space-size=8192"
 export LOG_FILE=/tmp/op-output.log
+export PGUSER=${PGUSER:=postgres}
+export PGHOST=${PGHOST:=localhost}
+export PGPASSWORD=${PGPASSWORD:=p4ssw0rd}
+export DATABASE_URL="postgres://$PGUSER:$PGPASSWORD@$PGHOST/postgres"
 
 run_psql() {
-	$PGBIN/psql -v ON_ERROR_STOP=1 -U dev -h 127.0.0.1 "$@"
+	psql -v ON_ERROR_STOP=1 "$@"
 }
 
 cleanup() {
 	exit_code=$?
 	echo "CLEANUP"
 	rm -rf tmp/cache/parallel*
-	if [ -d tmp/features ]; then mv tmp/features spec/ ; fi
 
 	if [ ! $exit_code -eq "0" ]; then
 		echo "ERROR: exit code $exit_code"
@@ -49,10 +52,18 @@ reset_dbs() {
 	execute_quiet "cat db/structure.sql | run_psql -d appdb"
 	# create and load schema for test databases "appdb1" to "appdb$JOBS", far faster than using parallel_rspec tasks for that
 	for i in $(seq 1 $JOBS); do
-		execute_quiet "echo 'drop database if exists appdb$i ; create database appdb$i with template appdb owner appuser;' | run_psql -d postgres"
+		execute_quiet "echo 'drop database if exists appdb$i ; create database appdb$i with template appdb owner $PGUSER;' | run_psql -d postgres"
 	done
 }
 
+precompile_assets() {
+	execute "JOBS=8 npm install"
+	execute_quiet "DATABASE_URL=nulldb://db bin/rails openproject:plugins:register_frontend assets:precompile"
+	execute_quiet "cp -rp config/frontend_assets.manifest.json public/assets/frontend_assets.manifest.json"
+	# ls -al frontend/.angular/cache/
+	# find frontend/.angular/cache -type d -exec sh -c 'ls -dt "$1"/*/ | tail -n +2 | xargs rm -r' sh {} \;
+}
+
 setup_tests() {
 	echo "Preparing environment for running tests..."
 	for i in $(seq 1 $JOBS); do
@@ -60,49 +71,44 @@ setup_tests() {
 		execute_quiet "rm -rf '$folder' ; mkdir -p '$folder' ; chmod 1777 '$folder'"
 	done
 
-	if [ ! -d "/tmp/nulldb" ]; then
-		execute_quiet "$PGBIN/initdb -E UTF8 -D /tmp/nulldb"
-		execute_quiet "cp docker/ci/postgresql.conf /tmp/nulldb/"
-		execute_quiet "$PGBIN/pg_ctl -D /tmp/nulldb -l /dev/null -w start"
-		echo "create database appdb; create user appuser with superuser encrypted password 'p4ssw0rd'; grant all privileges on database appdb to appuser;" | run_psql -d postgres
-	fi
+	execute_quiet "cp docker/ci/database.yml config/"
+	execute_quiet "mkdir -p spec/support/runtime-logs/"
+
+	execute "BUNDLE_JOBS=4 bundle install"
+	execute_quiet "bundle clean --force"
 
 	# create test database "app" and dump schema because db/structure.sql is not checked in
-	execute_quiet "time bundle exec rails db:migrate db:schema:dump zeitwerk:check"
+	execute_quiet "time bundle exec rails db:create db:migrate db:schema:dump zeitwerk:check"
 
 	# pre-cache browsers and their drivers binaries
 	execute "$(bundle show selenium)/bin/linux/selenium-manager --browser chrome --debug"
 	execute "$(bundle show selenium)/bin/linux/selenium-manager --browser firefox --debug"
+
+	precompile_assets
 }
 
 run_units() {
 	shopt -s extglob
 	reset_dbs
-	execute_quiet "cp -f /cache/turbo_runtime_units.log spec/support/ || true"
-	execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/turbo_runtime_units.log spec/!(features) modules/**/spec/!(features)"
-	execute_quiet "cp -f spec/support/turbo_runtime_units.log /cache/ || true"
+	execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/runtime-logs/turbo_runtime_units.log spec/!(features) modules/**/spec/!(features)"
 	cleanup
 }
 
 run_features() {
 	shopt -s extglob
 	reset_dbs
-	execute_quiet "cp -f /cache/turbo_runtime_features.log spec/support/ || true"
-	execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/turbo_runtime_features.log spec/features modules/**/spec/features"
-	execute_quiet "cp -f spec/support/turbo_runtime_features.log /cache/ || true"
+	execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/runtime-logs/turbo_runtime_features.log spec/features modules/**/spec/features"
 	cleanup
 }
 
 run_all() {
 	shopt -s globstar
 	reset_dbs
-	execute_quiet "cp -f /cache/turbo_runtime_all.log spec/support/ || true"
-	execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/turbo_runtime_all.log spec modules/**/spec"
-	execute_quiet "cp -f spec/support/turbo_runtime_all.log /cache/ || true"
+	execute "time bundle exec turbo_tests --verbose -n $JOBS --runtime-log spec/support/runtime-logs/turbo_runtime_all.log spec modules/**/spec"
 	cleanup
 }
 
-export -f cleanup execute execute_quiet run_psql reset_dbs setup_tests run_units run_features
+export -f cleanup execute execute_quiet run_psql reset_dbs setup_tests precompile_assets run_units run_features
 
 if [ "$1" == "setup-tests" ]; then
 	shift

docker/ci/postgresql.conf

@@ -6,7 +6,10 @@
 # Connections num: 200
 # Data Storage: ssd
 
+# https://www.postgresql.org/docs/current/non-durability.html
 fsync = off
+synchronous_commit = off
+checkpoint_timeout = 30min
 full_page_writes = off
 deadlock_timeout = 20s
 autovacuum = off