Smaller CI image and parallelise setup commands (#14227)

.github/workflows/test-core.yml

@@ -50,15 +50,15 @@ jobs:
     - name: Cache NPM
       uses: actions/cache@v3
       with:
-        path: cache/npm
-        key: npm-${{ hashFiles('frontend/package-lock.json') }}
+        path: cache/node
+        key: node-${{ hashFiles('package.json', 'frontend/package-lock.json') }}
         restore-keys: |
-          npm-
+          node-
     - name: Cache ANGULAR
       uses: actions/cache@v3
       with:
         path: cache/angular
-        key: angular-${{ hashFiles('frontend/package-lock.json') }}
+        key: angular-${{ hashFiles('package.json', 'frontend/package-lock.json') }}
         restore-keys: |
           angular-
     - name: Cache TEST RUNTIME

bin/ci

@@ -9,10 +9,14 @@ export COMPOSE_FILE=docker-compose.ci.yml
 export RUBY_VERSION="$(cat .ruby-version)"
 export LOCAL_CACHE_PATH="./cache"
 
-mkdir -p $LOCAL_CACHE_PATH/{bundle,npm,angular,runtime-logs,buildx}
+mkdir -p $LOCAL_CACHE_PATH/{bundle,node/.npm,node/node_modules,node/frontend/node_modules,angular,runtime-logs}
 
 if [ "$1" = "down" ]; then
   exec docker compose down --remove-orphans --volumes
 else
-  exec docker compose run --rm --remove-orphans app "$@"
+  additional_args=""
+  if [ "$CI_REBUILD" = "true" ]; then
+    additional_args="$additional_args --build"
+  fi
+  exec docker compose run --rm --remove-orphans $additional_args app "$@"
 fi

docker-compose.ci.yml

@@ -12,8 +12,6 @@ services:
         - APP_USER_UID
         - APP_USER_GID
         - RUBY_VERSION
-      cache_from: ["type=local,src=${LOCAL_CACHE_PATH}/buildx"]
-      cache_to: ["type=local,dest=${LOCAL_CACHE_PATH}/buildx"]
     environment:
       CI_JOBS: "${CI_JOBS}"
       RSPEC_RETRY_RETRY_COUNT: "${CI_RETRY_COUNT:-3}"
@@ -23,7 +21,9 @@ services:
       - "/tmp"
     volumes:
       - .:/app
+      - ${LOCAL_CACHE_PATH}/node/.npm:/app/.npm
+      - ${LOCAL_CACHE_PATH}/node/node_modules:/app/node_modules
+      - ${LOCAL_CACHE_PATH}/node/frontend/node_modules:/app/frontend/node_modules
       - ${LOCAL_CACHE_PATH}/bundle:/usr/local/bundle
-      - ${LOCAL_CACHE_PATH}/npm:/app/.npm
       - ${LOCAL_CACHE_PATH}/angular:/app/frontend/.angular/cache
       - ${LOCAL_CACHE_PATH}/runtime-logs:/app/spec/support/runtime-logs

docker/ci/Dockerfile

@@ -14,11 +14,11 @@ RUN wget --quiet -O- https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-ke
 ENV CHROME_SOURCE_URL="https://dl.google.com/dl/linux/direct/google-chrome-stable_current_amd64.deb"
 RUN --mount=type=cache,target=/var/cache/apt export f="/tmp/chrome.deb" && wget --no-verbose -O $f $CHROME_SOURCE_URL && \
   apt-get update -qq && apt-get install -y "$f" postgresql-$PGVERSION postgresql-client-$PGVERSION time imagemagick default-jre-headless firefox-esr && \
-  rm -f "$f" && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+  rm -f "$f" && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/lib/postgresql && find /usr/share/locale/* -maxdepth 0 -type d ! -name 'en' -exec rm -rf {} \;
 
 RUN curl -L https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz -o - | tar xJf - -C /usr/local --strip=1 && node --version
 
-RUN gem install bundler --version "$BUNDLER_VERSION" --no-document
+RUN rm -rf /usr/local/bundle && gem install bundler --version "$BUNDLER_VERSION" --no-document
 
 ENV APP_USER=dev
 ENV APP_PATH=/app
@@ -40,7 +40,8 @@ ENV RUBYOPT="-W0"
 WORKDIR $APP_PATH
 USER $APP_USER
 
-COPY ./docker/ci/entrypoint.sh /usr/sbin/entrypoint.sh
+RUN mkdir -p ./docker/ci
+COPY ./docker/ci/entrypoint.sh ./docker/ci/entrypoint.sh
 
-ENTRYPOINT [ "/usr/sbin/entrypoint.sh" ]
+ENTRYPOINT [ "/app/docker/ci/entrypoint.sh" ]
 CMD ["setup-tests", "bash"]

docker/ci/entrypoint.sh

@@ -32,6 +32,28 @@ cleanup() {
 
 trap cleanup INT TERM EXIT
 
+declare -a pids=()
+
+run_background() {
+  # Run the command in the background
+  "$@" &
+
+  # Store the PID of the background process
+  local pid=$!
+  pids+=("$pid")
+}
+
+wait_for_background() {
+	for pid in "${pids[@]}"; do
+		wait "$pid"
+		# Check the exit status of each background process
+		if [ $? -ne 0 ]; then
+			echo "Command with PID $pid failed"
+			exit 1
+		fi
+	done
+}
+
 execute() {
 	BANNER=${BANNER:="[execute]"}
 	echo "$BANNER $@" >&2
@@ -64,12 +86,14 @@ reset_dbs() {
 	done
 }
 
-precompile_assets() {
-	execute "JOBS=8 time npm install"
+backend_stuff() {
+	# create test database "app" and dump schema because db/structure.sql is not checked in
+	execute_quiet "time bundle exec rails db:create db:migrate db:schema:dump zeitwerk:check"
+}
+
+frontend_stuff() {
 	execute_quiet "DATABASE_URL=nulldb://db time bin/rails openproject:plugins:register_frontend assets:precompile"
 	execute_quiet "cp -rp config/frontend_assets.manifest.json public/assets/frontend_assets.manifest.json"
-	# ls -al frontend/.angular/cache/
-	# find frontend/.angular/cache -type d -exec sh -c 'ls -dt "$1"/*/ | tail -n +2 | xargs rm -r' sh {} \;
 }
 
 setup_tests() {
@@ -80,20 +104,19 @@ setup_tests() {
 	done
 
 	execute_quiet "mkdir -p spec/support/runtime-logs/"
-
-	execute "BUNDLE_JOBS=4 bundle install"
-	execute_quiet "bundle clean --force"
-
-	create_db_cluster
 	execute_quiet "cp docker/ci/database.yml config/"
-	# create test database "app" and dump schema because db/structure.sql is not checked in
-	execute_quiet "time bundle exec rails db:create db:migrate db:schema:dump zeitwerk:check"
+	create_db_cluster
 
+	run_background execute "BUNDLE_JOBS=8 bundle install --quiet && bundle clean --force && echo BUNDLE DONE"
+	run_background execute "JOBS=8 time npm install --quiet && npm prune --quiet && echo NPM DONE"
+	wait_for_background
+
+	run_background backend_stuff
+	run_background frontend_stuff
 	# pre-cache browsers and their drivers binaries
-	execute "$(bundle show selenium)/bin/linux/selenium-manager --browser chrome --debug"
-	execute "$(bundle show selenium)/bin/linux/selenium-manager --browser firefox --debug"
-
-	precompile_assets
+	run_background $(bundle show selenium)/bin/linux/selenium-manager --browser chrome --debug
+	run_background $(bundle show selenium)/bin/linux/selenium-manager --browser firefox --debug
+	wait_for_background
 }
 
 run_units() {
@@ -117,7 +140,7 @@ run_all() {
 	cleanup
 }
 
-export -f cleanup execute execute_quiet run_psql reset_dbs setup_tests precompile_assets run_units run_features run_all
+export -f cleanup execute execute_quiet run_psql create_db_cluster reset_dbs setup_tests backend_stuff frontend_stuff run_units run_features run_all
 
 if [ "$1" == "setup-tests" ]; then
 	shift