From dfbfe24d0be1696a41304ebd90ea780d3d28334f Mon Sep 17 00:00:00 2001 From: Cyril Rohr Date: Wed, 19 Aug 2020 14:11:28 +0200 Subject: [PATCH] Unified docker image for BIM edition. Add pre/post hooks for other platforms. (#8553) Unified docker image for BIM edition. Add pre/post hooks for other platforms. --- .dockerignore | 2 + .github/workflows/docker.yml | 9 ++- Dockerfile | 99 +++++++++++------------------- docker/entrypoint.sh | 11 +++- docker/precompile-assets.sh | 23 ------- docker/setup/postinstall.sh | 59 ++++++++++++++++++ docker/setup/preinstall-on-prem.sh | 11 ++++ docker/setup/preinstall.sh | 82 +++++++++++++++++++++++++ docker/supervisord | 1 - 9 files changed, 207 insertions(+), 90 deletions(-) delete mode 100644 docker/precompile-assets.sh create mode 100755 docker/setup/postinstall.sh create mode 100755 docker/setup/preinstall-on-prem.sh create mode 100755 docker/setup/preinstall.sh diff --git a/.dockerignore b/.dockerignore index 2616f99e730..ebb3c917091 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +.github/ .git .dockerignore .bundle @@ -22,5 +23,6 @@ spec spec_legacy tmp frontend/node_modules +node_modules # travis vendor/bundle diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 599a9e8e8a4..060f15d5e97 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,11 +14,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@master - - name: Publish to registry + - name: Build & Push + id: build_and_push uses: elgohr/Publish-Docker-Github-Action@master with: - name: openproject/community-test + name: openproject/community username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} tag_semver: true cache: ${{ github.event_name != 'schedule' }} + - name: Test + run: | + docker run -d -p 8080:80 -e SUPERVISORD_LOG_LEVEL=debug -e SECRET_KEY_BASE=secret ${{ steps.build_and_push.outputs.digest }} + curl --fail -ks --retry 10 http://localhost:8080/api/v3 diff --git a/Dockerfile b/Dockerfile index bb3c231bd57..938f94003b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,83 +6,58 @@ RUN /tmp/build-pgloader && rm /tmp/build-pgloader FROM ruby:2.6-stretch MAINTAINER operations@openproject.com -ENV NODE_VERSION "10.15.0" -ENV BUNDLER_VERSION "2.0.2" -ENV BUNDLE_PATH__SYSTEM=false -ENV APP_USER app -ENV APP_PATH /app -ENV APP_DATA_PATH /var/openproject/assets -ENV APP_DATA_PATH_LEGACY /var/db/openproject -ENV PGDATA /var/openproject/pgdata -ENV PGDATA_LEGACY /var/lib/postgresql/9.6/main +# Allow platform-specific additions. Valid values are: on-prem,saas,bahn +ARG PLATFORM=on-prem +# Use OAuth token in case private gems need to be fetched +ARG GITHUB_OAUTH_TOKEN +ARG DEBIAN_FRONTEND=noninteractive -ENV DATABASE_URL postgres://openproject:openproject@127.0.0.1/openproject -ENV RAILS_ENV production -ENV HEROKU true -ENV RAILS_CACHE_STORE memcache -ENV OPENPROJECT_INSTALLATION__TYPE docker -ENV NEW_RELIC_AGENT_ENABLED false -ENV ATTACHMENTS_STORAGE_PATH $APP_DATA_PATH/files +ENV NODE_VERSION="12.18.3" +ENV BUNDLER_VERSION="2.0.2" +ENV BUNDLE_PATH__SYSTEM=false +ENV APP_USER=app +ENV APP_PATH=/app +ENV APP_DATA_PATH=/var/openproject/assets +ENV APP_DATA_PATH_LEGACY=/var/db/openproject +ENV PGBIN="/usr/lib/postgresql/9.6/bin" +ENV PGDATA=/var/openproject/pgdata +ENV PGDATA_LEGACY=/var/lib/postgresql/9.6/main + +ENV DATABASE_URL=postgres://openproject:openproject@127.0.0.1/openproject +ENV HEROKU=true +ENV RAILS_ENV=production +ENV RAILS_CACHE_STORE=memcache +ENV RAILS_GROUPS="production docker opf_plugins" +ENV OPENPROJECT_INSTALLATION__TYPE=docker +# Valid values are: standard,bim +ENV OPENPROJECT_EDITION=standard +ENV NEW_RELIC_AGENT_ENABLED=false +ENV ATTACHMENTS_STORAGE_PATH=$APP_DATA_PATH/files # Set a default key base, ensure to provide a secure value in production environments! -ENV SECRET_KEY_BASE OVERWRITE_ME +ENV SECRET_KEY_BASE=OVERWRITE_ME COPY --from=pgloader /usr/local/bin/pgloader-ccl /usr/local/bin/ -# install node + npm -RUN curl https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz | tar xzf - -C /usr/local --strip-components=1 - -RUN apt-get update -qq && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - postgresql-client \ - poppler-utils \ - unrtf \ - tesseract-ocr \ - catdoc \ - memcached \ - postfix \ - postgresql \ - apache2 \ - supervisor && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Set up pg defaults -RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/9.6/main/pg_hba.conf -RUN echo "listen_addresses='*'" >> /etc/postgresql/9.6/main/postgresql.conf -RUN echo "data_directory='$PGDATA'" >> /etc/postgresql/9.6/main/postgresql.conf -RUN rm -rf "$PGDATA_LEGACY" && rm -rf "$PGDATA" && mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" -RUN a2enmod proxy proxy_http && rm -f /etc/apache2/sites-enabled/000-default.conf - -# using /home/app since npm cache and other stuff will be put there when running npm install -# we don't want to pollute any locally-mounted directory -RUN useradd -d /home/$APP_USER -m $APP_USER - WORKDIR $APP_PATH -RUN gem install bundler --version "${bundler_version}" --no-document + +COPY docker/setup ./docker/setup +RUN ./docker/setup/preinstall.sh COPY Gemfile ./Gemfile COPY Gemfile.* ./ COPY modules ./modules -# OpenProject::Version is required by module versions in gemspecs -RUN mkdir -p lib/open_project -COPY lib/open_project/version.rb ./lib/open_project/ -RUN bundle install --deployment --path vendor/bundle --no-cache \ - --with="docker opf_plugins" --without="test development" --jobs=8 --retry=3 && \ +COPY vendor ./vendor +# some gemspec files of plugins require files in there, notably OpenProject::Version +COPY lib ./lib + +RUN bundle install --quiet --deployment --path vendor/bundle --no-cache \ + --with="$RAILS_GROUPS" --without="test development" --jobs=8 --retry=3 && \ rm -rf vendor/bundle/ruby/*/cache && rm -rf vendor/bundle/ruby/*/gems/*/spec && rm -rf vendor/bundle/ruby/*/gems/*/test # Finally, copy over the whole thing COPY . . -# Re-use packager database.yml -RUN cp ./packaging/conf/database.yml ./config/database.yml -# Add MySQL-to-Postgres migration script to path (used in entrypoint.sh) -RUN cp ./docker/mysql-to-postgres/bin/migrate-mysql-to-postgres /usr/local/bin/ -# Ensure OpenProject starts with the docker group of gems -RUN sed -i "s|Rails.groups(:opf_plugins)|Rails.groups(:opf_plugins, :docker)|" config/application.rb -# Ensure we can write in /tmp/op_uploaded_files (cf. #29112) -RUN mkdir -p /tmp/op_uploaded_files/ && chown -R $APP_USER:$APP_USER /tmp/op_uploaded_files/ - -# Handle the assets precompilation -RUN bash docker/precompile-assets.sh +RUN ./docker/setup/postinstall.sh # Expose ports for apache and postgres EXPOSE 80 5432 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index cbfa57e9c5d..80a4452107b 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -67,11 +67,18 @@ if [ "$(id -u)" = '0' ]; then mkdir -p "$APP_PATH/log" "$APP_PATH/tmp/pids" "$APP_PATH/files" chown "$APP_USER:$APP_USER" "$APP_PATH" chown -R "$APP_USER:$APP_USER" "$APP_PATH/log" "$APP_PATH/tmp" "$APP_PATH/files" "$APP_PATH/public" + + # allow to launch any command as root by prepending it with 'root' + if [ "$1" = "root" ]; then + shift + exec "$@" + fi + if [ "$1" = "./docker/supervisord" ] || [ "$1" = "./docker/proxy" ]; then exec "$@" - else - exec $APP_PATH/docker/gosu $APP_USER "$BASH_SOURCE" "$@" fi + + exec $APP_PATH/docker/gosu $APP_USER "$BASH_SOURCE" "$@" fi exec "$@" diff --git a/docker/precompile-assets.sh b/docker/precompile-assets.sh deleted file mode 100644 index 07d9aa72bfd..00000000000 --- a/docker/precompile-assets.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -e - -pushd "${APP_PATH}/frontend" - -export NG_CLI_ANALYTICS=ci # so angular cli doesn't block waiting for user input - -# Installing frontend dependencies -RAILS_ENV=production npm install - -popd - -# Bundle assets -DATABASE_URL='nulldb://nohost' RAILS_ENV=production bundle exec rake assets:precompile - -# Remove sprockets cache -rm -rf "$APP_PATH/tmp/cache/assets" - -# Remove node_modules and entire frontend -rm -rf "$APP_PATH/node_modules/" "$APP_PATH/frontend/node_modules/" - -# Clean cache in root -rm -rf /root/.npm diff --git a/docker/setup/postinstall.sh b/docker/setup/postinstall.sh new file mode 100755 index 00000000000..42911b8bdd6 --- /dev/null +++ b/docker/setup/postinstall.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -e + +display_error() { + echo " !--> ERROR on postinstall:" + tail -n 200 /tmp/dockerize.log + exit 1 +} + +echo " ---> POSTINSTALL" + +# Add MySQL-to-Postgres migration script to path (used in entrypoint.sh) +cp ./docker/mysql-to-postgres/bin/migrate-mysql-to-postgres /usr/local/bin/ + +# Ensure we can write in /tmp/op_uploaded_files (cf. #29112) +mkdir -p /tmp/op_uploaded_files/ && chown -R $APP_USER:$APP_USER /tmp/op_uploaded_files/ + +rm -f ./config/database.yml + +if test -f ./docker/setup/postinstall-$PLATFORM.sh ; then + echo " ---> Executing postinstall for $PLATFORM..." + ./docker/setup/postinstall-$PLATFORM.sh +fi + +echo " ---> Precompiling assets. This will take a while..." + +( + pushd "${APP_PATH}/frontend" + + export NG_CLI_ANALYTICS=ci # so angular cli doesn't block waiting for user input + + # Installing frontend dependencies + RAILS_ENV=production npm install + + popd + + # Bundle assets + su - postgres -c "$PGBIN/initdb -D /tmp/nulldb" + su - postgres -c "$PGBIN/pg_ctl -D /tmp/nulldb -l /dev/null -w start" + echo "create database assets; create user assets with encrypted password 'p4ssw0rd'; grant all privileges on database assets to assets;" | su - postgres -c psql + DATABASE_URL=postgres://assets:p4ssw0rd@127.0.0.1/assets RAILS_ENV=production bundle exec rake db:migrate db:schema:dump db:schema:cache:dump assets:precompile + + su - postgres -c "$PGBIN/pg_ctl -D /tmp/nulldb stop" + + rm -rf /tmp/nulldb + + # Remove sprockets cache + rm -rf "$APP_PATH/tmp/cache/assets" + + # Remove node_modules and entire frontend + rm -rf "$APP_PATH/node_modules/" "$APP_PATH/frontend/node_modules/" + + # Clean cache in root + rm -rf /root/.npm +) >/tmp/dockerize.log || display_error + +rm -f /tmp/dockerize.log +echo " OK." diff --git a/docker/setup/preinstall-on-prem.sh b/docker/setup/preinstall-on-prem.sh new file mode 100755 index 00000000000..5d75f67c98a --- /dev/null +++ b/docker/setup/preinstall-on-prem.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +apt-get install -y \ + memcached \ + postfix \ + apache2 \ + supervisor + +a2enmod proxy proxy_http +rm -f /etc/apache2/sites-enabled/000-default.conf diff --git a/docker/setup/preinstall.sh b/docker/setup/preinstall.sh new file mode 100755 index 00000000000..b9b11cd22e4 --- /dev/null +++ b/docker/setup/preinstall.sh @@ -0,0 +1,82 @@ +#!/bin/bash +set -e + +echo " ---> PREINSTALL" + +display_error() { + echo " !--> ERROR on preinstall:" + tail -n 200 /tmp/dockerize.log + exit 1 +} + +echo " ---> Setting up common dependencies. This will take a while..." +apt-get update -qq + +( + # install node + npm + curl -s https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-x64.tar.gz | \ + tar xzf - -C /usr/local --strip-components=1 + + apt-get install -y \ + apt-transport-https \ + postgresql-client \ + pandoc \ + poppler-utils \ + unrtf \ + tesseract-ocr \ + catdoc \ + postgresql + + # Set up pg defaults + echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/9.6/main/pg_hba.conf + echo "listen_addresses='*'" >> /etc/postgresql/9.6/main/postgresql.conf + echo "data_directory='$PGDATA'" >> /etc/postgresql/9.6/main/postgresql.conf + rm -rf "$PGDATA_LEGACY" && rm -rf "$PGDATA" && mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" + + # Specifics for BIM edition + wget -qO- https://packages.microsoft.com/keys/microsoft.asc | apt-key add - + wget -q https://packages.microsoft.com/config/debian/9/prod.list -O /etc/apt/sources.list.d/microsoft-prod.list + apt-get update -qq + apt-get install -y dotnet-runtime-3.1 + + tmpdir=$(mktemp -d) + cd $tmpdir + + # Install XKT converter + npm install xeokit/xeokit-gltf-to-xkt -g + + # Install COLLADA2GLTF + wget --quiet https://github.com/KhronosGroup/COLLADA2GLTF/releases/download/v2.1.5/COLLADA2GLTF-v2.1.5-linux.zip + unzip -q COLLADA2GLTF-v2.1.5-linux.zip + mv COLLADA2GLTF-bin "/usr/local/bin/COLLADA2GLTF" + + # IFCconvert + wget --quiet https://s3.amazonaws.com/ifcopenshell-builds/IfcConvert-v0.6.0-9bcd932-linux64.zip + unzip -q IfcConvert-v0.6.0-9bcd932-linux64.zip + mv IfcConvert "/usr/local/bin/IfcConvert" + + wget --quiet https://github.com/bimspot/xeokit-metadata/releases/download/1.0.0/xeokit-metadata-linux-x64.tar.gz + tar -zxvf xeokit-metadata-linux-x64.tar.gz + chmod +x xeokit-metadata-linux-x64/xeokit-metadata + cp -r xeokit-metadata-linux-x64/ "/usr/lib/xeokit-metadata" + ln -s /usr/lib/xeokit-metadata/xeokit-metadata /usr/local/bin/xeokit-metadata + + cd / + rm -rf $tmpdir + + gem install bundler --version "$BUNDLER_VERSION" --no-document + + useradd -d /home/$APP_USER -m $APP_USER + +) >/tmp/dockerize.log || display_error + +if test -f ./docker/setup/preinstall-$PLATFORM.sh ; then + echo " ---> Executing preinstall for $PLATFORM..." + ./docker/setup/preinstall-$PLATFORM.sh >/tmp/dockerize.log || display_error +fi + +apt-get clean +rm -rf /var/lib/apt/lists/* + +rm -f /tmp/dockerize.log +echo " OK." diff --git a/docker/supervisord b/docker/supervisord index fd2a1ecf83e..5e73ce7b0be 100755 --- a/docker/supervisord +++ b/docker/supervisord @@ -18,7 +18,6 @@ PGUSER=${PGUSER:=postgres} PGPASSWORD=${PGPASSWORD:=postgres} PG_STARTUP_WAIT_TIME=${PG_STARTUP_WAIT_TIME:=10} SUPERVISORD_LOG_LEVEL=${SUPERVISORD_LOG_LEVEL:=info} -PGBIN="/usr/lib/postgresql/9.6/bin" dbhost=$(ruby -ruri -e 'puts URI(ENV.fetch("DATABASE_URL")).host') pwfile=$(mktemp)