From 32ce2600afbc14527d65a8a49ccd2ce9ddb20edb Mon Sep 17 00:00:00 2001 From: Cyril Rohr Date: Thu, 23 Nov 2023 15:07:17 +0100 Subject: [PATCH] Smaller CI image and parallelise setup commands (#14227) --- .github/workflows/test-core.yml | 8 ++--- bin/ci | 8 +++-- docker-compose.ci.yml | 6 ++-- docker/ci/Dockerfile | 9 +++--- docker/ci/entrypoint.sh | 55 +++++++++++++++++++++++---------- 5 files changed, 57 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test-core.yml b/.github/workflows/test-core.yml index 8805f13d636..fcbbf03fab8 100644 --- a/.github/workflows/test-core.yml +++ b/.github/workflows/test-core.yml @@ -50,15 +50,15 @@ jobs: - name: Cache NPM uses: actions/cache@v3 with: - path: cache/npm - key: npm-${{ hashFiles('frontend/package-lock.json') }} + path: cache/node + key: node-${{ hashFiles('package.json', 'frontend/package-lock.json') }} restore-keys: | - npm- + node- - name: Cache ANGULAR uses: actions/cache@v3 with: path: cache/angular - key: angular-${{ hashFiles('frontend/package-lock.json') }} + key: angular-${{ hashFiles('package.json', 'frontend/package-lock.json') }} restore-keys: | angular- - name: Cache TEST RUNTIME diff --git a/bin/ci b/bin/ci index ad9503b80f3..4dec632beb1 100755 --- a/bin/ci +++ b/bin/ci @@ -9,10 +9,14 @@ export COMPOSE_FILE=docker-compose.ci.yml export RUBY_VERSION="$(cat .ruby-version)" export LOCAL_CACHE_PATH="./cache" -mkdir -p $LOCAL_CACHE_PATH/{bundle,npm,angular,runtime-logs,buildx} +mkdir -p $LOCAL_CACHE_PATH/{bundle,node/.npm,node/node_modules,node/frontend/node_modules,angular,runtime-logs} if [ "$1" = "down" ]; then exec docker compose down --remove-orphans --volumes else - exec docker compose run --rm --remove-orphans app "$@" + additional_args="" + if [ "$CI_REBUILD" = "true" ]; then + additional_args="$additional_args --build" + fi + exec docker compose run --rm --remove-orphans $additional_args app "$@" fi \ No newline at end of file diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index b777cd8c02e..42913e5e194 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -12,8 +12,6 @@ services: - APP_USER_UID - APP_USER_GID - RUBY_VERSION - cache_from: ["type=local,src=${LOCAL_CACHE_PATH}/buildx"] - cache_to: ["type=local,dest=${LOCAL_CACHE_PATH}/buildx"] environment: CI_JOBS: "${CI_JOBS}" RSPEC_RETRY_RETRY_COUNT: "${CI_RETRY_COUNT:-3}" @@ -23,7 +21,9 @@ services: - "/tmp" volumes: - .:/app + - ${LOCAL_CACHE_PATH}/node/.npm:/app/.npm + - ${LOCAL_CACHE_PATH}/node/node_modules:/app/node_modules + - ${LOCAL_CACHE_PATH}/node/frontend/node_modules:/app/frontend/node_modules - ${LOCAL_CACHE_PATH}/bundle:/usr/local/bundle - - ${LOCAL_CACHE_PATH}/npm:/app/.npm - ${LOCAL_CACHE_PATH}/angular:/app/frontend/.angular/cache - ${LOCAL_CACHE_PATH}/runtime-logs:/app/spec/support/runtime-logs diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile index d5009f563b1..d44fb690c18 100644 --- a/docker/ci/Dockerfile +++ b/docker/ci/Dockerfile @@ -14,11 +14,11 @@ RUN wget --quiet -O- https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-ke ENV CHROME_SOURCE_URL="https://dl.google.com/dl/linux/direct/google-chrome-stable_current_amd64.deb" RUN --mount=type=cache,target=/var/cache/apt export f="/tmp/chrome.deb" && wget --no-verbose -O $f $CHROME_SOURCE_URL && \ apt-get update -qq && apt-get install -y "$f" postgresql-$PGVERSION postgresql-client-$PGVERSION time imagemagick default-jre-headless firefox-esr && \ - rm -f "$f" && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + rm -f "$f" && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /var/lib/postgresql && find /usr/share/locale/* -maxdepth 0 -type d ! -name 'en' -exec rm -rf {} \; RUN curl -L https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.xz -o - | tar xJf - -C /usr/local --strip=1 && node --version -RUN gem install bundler --version "$BUNDLER_VERSION" --no-document +RUN rm -rf /usr/local/bundle && gem install bundler --version "$BUNDLER_VERSION" --no-document ENV APP_USER=dev ENV APP_PATH=/app @@ -40,7 +40,8 @@ ENV RUBYOPT="-W0" WORKDIR $APP_PATH USER $APP_USER -COPY ./docker/ci/entrypoint.sh /usr/sbin/entrypoint.sh +RUN mkdir -p ./docker/ci +COPY ./docker/ci/entrypoint.sh ./docker/ci/entrypoint.sh -ENTRYPOINT [ "/usr/sbin/entrypoint.sh" ] +ENTRYPOINT [ "/app/docker/ci/entrypoint.sh" ] CMD ["setup-tests", "bash"] diff --git a/docker/ci/entrypoint.sh b/docker/ci/entrypoint.sh index e04a3627c3d..31038a159ad 100755 --- a/docker/ci/entrypoint.sh +++ b/docker/ci/entrypoint.sh @@ -32,6 +32,28 @@ cleanup() { trap cleanup INT TERM EXIT +declare -a pids=() + +run_background() { + # Run the command in the background + "$@" & + + # Store the PID of the background process + local pid=$! + pids+=("$pid") +} + +wait_for_background() { + for pid in "${pids[@]}"; do + wait "$pid" + # Check the exit status of each background process + if [ $? -ne 0 ]; then + echo "Command with PID $pid failed" + exit 1 + fi + done +} + execute() { BANNER=${BANNER:="[execute]"} echo "$BANNER $@" >&2 @@ -64,12 +86,14 @@ reset_dbs() { done } -precompile_assets() { - execute "JOBS=8 time npm install" +backend_stuff() { + # create test database "app" and dump schema because db/structure.sql is not checked in + execute_quiet "time bundle exec rails db:create db:migrate db:schema:dump zeitwerk:check" +} + +frontend_stuff() { execute_quiet "DATABASE_URL=nulldb://db time bin/rails openproject:plugins:register_frontend assets:precompile" execute_quiet "cp -rp config/frontend_assets.manifest.json public/assets/frontend_assets.manifest.json" - # ls -al frontend/.angular/cache/ - # find frontend/.angular/cache -type d -exec sh -c 'ls -dt "$1"/*/ | tail -n +2 | xargs rm -r' sh {} \; } setup_tests() { @@ -80,20 +104,19 @@ setup_tests() { done execute_quiet "mkdir -p spec/support/runtime-logs/" - - execute "BUNDLE_JOBS=4 bundle install" - execute_quiet "bundle clean --force" - - create_db_cluster execute_quiet "cp docker/ci/database.yml config/" - # create test database "app" and dump schema because db/structure.sql is not checked in - execute_quiet "time bundle exec rails db:create db:migrate db:schema:dump zeitwerk:check" + create_db_cluster + run_background execute "BUNDLE_JOBS=8 bundle install --quiet && bundle clean --force && echo BUNDLE DONE" + run_background execute "JOBS=8 time npm install --quiet && npm prune --quiet && echo NPM DONE" + wait_for_background + + run_background backend_stuff + run_background frontend_stuff # pre-cache browsers and their drivers binaries - execute "$(bundle show selenium)/bin/linux/selenium-manager --browser chrome --debug" - execute "$(bundle show selenium)/bin/linux/selenium-manager --browser firefox --debug" - - precompile_assets + run_background $(bundle show selenium)/bin/linux/selenium-manager --browser chrome --debug + run_background $(bundle show selenium)/bin/linux/selenium-manager --browser firefox --debug + wait_for_background } run_units() { @@ -117,7 +140,7 @@ run_all() { cleanup } -export -f cleanup execute execute_quiet run_psql reset_dbs setup_tests precompile_assets run_units run_features run_all +export -f cleanup execute execute_quiet run_psql create_db_cluster reset_dbs setup_tests backend_stuff frontend_stuff run_units run_features run_all if [ "$1" == "setup-tests" ]; then shift