From 73ac53238fae768f2e1b0489d5123149bacfdd6f Mon Sep 17 00:00:00 2001 From: Eric Schubert Date: Mon, 5 Jun 2023 11:24:14 +0200 Subject: [PATCH] [#47624] reworked docker setup - https://community.openproject.org/work_packages/47624 - extracted port exposure from docker compose --- docker-compose.override.example.yml | 27 +++ docker-compose.yml | 27 +-- .../development-environment-docker/README.md | 178 ++++++++++++------ 3 files changed, 148 insertions(+), 84 deletions(-) create mode 100644 docker-compose.override.example.yml diff --git a/docker-compose.override.example.yml b/docker-compose.override.example.yml new file mode 100644 index 00000000000..3bc8e54edf1 --- /dev/null +++ b/docker-compose.override.example.yml @@ -0,0 +1,27 @@ +# This is an example for the `docker-compose.override.yml`. +# You can use this as a base for creating your own override file. + +services: + backend: + ports: + - "3000:3000" + + frontend: + ports: + - "4200:4200" + + storybook: + ports: + - '6006:6006' + + db: + ports: + - '5432:5432' + + db-test: + ports: + - '5433:5432' + + chrome: + ports: + - '5900:5900' diff --git a/docker-compose.yml b/docker-compose.yml index 1fed0d964e0..6cab7e57249 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.7" +version: '3.8' networks: network: @@ -45,6 +45,9 @@ x-op-backend: &backend OPENPROJECT_RAILS__RELATIVE__URL__ROOT: "${OPENPROJECT_RAILS__RELATIVE__URL__ROOT:-}" DATABASE_URL: postgresql://${DB_USERNAME:-postgres}:${DB_PASSWORD:-postgres}@${DB_HOST:-db}:${DB_PORT:-5432}/${DB_DATABASE:-openproject} OPENPROJECT_EDITION: ${OPENPROJECT_EDITION:-standard} + OPENPROJECT_WEB_MAX__THREADS: 1 + OPENPROJECT_WEB_MIN__THREADS: 1 + OPENPROJECT_WEB_WORKERS: 0 volumes: - ".:/home/dev/openproject" - "opdata:/var/openproject/assets" @@ -59,8 +62,6 @@ services: tty: true stdin_open: true command: run-app - ports: - - "3000:3000" depends_on: - db - cache @@ -79,9 +80,6 @@ services: volumes: - ".:/home/dev/openproject" - "${CKEDITOR_BUILD_DIR:-./frontend/src/vendor/ckeditor/}:/home/dev/openproject/frontend/src/vendor/ckeditor/" - ports: - - "4200:4200" - - "9999:9999" environment: PROXY_HOSTNAME: backend networks: @@ -95,8 +93,6 @@ services: command: "npm run storybook:serve" volumes: - ".:/home/dev/openproject" - ports: - - "6006:6006" networks: - network @@ -106,8 +102,6 @@ services: stop_grace_period: "3s" volumes: - "pgdata:/var/lib/postgresql/data" - ports: - - "5432:5432" environment: POSTGRES_USER: ${DB_USERNAME:-postgres} POSTGRES_PASSWORD: ${DB_PASSWORD:-postgres} @@ -128,8 +122,6 @@ services: stop_grace_period: "3s" volumes: - "pgdata-test:/var/lib/postgresql/data" - ports: - - "5433:5432" environment: POSTGRES_DB: openproject POSTGRES_USER: openproject @@ -196,8 +188,6 @@ services: - opera networks: - testing - ports: - - "4442-4444:4442-4444" chrome: image: selenium/node-chrome:4 @@ -206,8 +196,6 @@ services: - "downloads-test:/home/seluser/Downloads" networks: - testing - ports: - - "5900-5915:5900" shm_size: 2g environment: SE_EVENT_BUS_HOST: selenium-hub @@ -226,8 +214,6 @@ services: - "downloads-test:/home/seluser/Downloads" networks: - testing - ports: - - "5916-5931:5900" shm_size: 2g environment: SE_EVENT_BUS_HOST: selenium-hub @@ -246,8 +232,6 @@ services: - "downloads-test:/home/seluser/Downloads" networks: - testing - ports: - - "5932-5957:5900" environment: SE_EVENT_BUS_HOST: selenium-hub SE_EVENT_BUS_PUBLISH_PORT: 4442 @@ -257,6 +241,3 @@ services: # in case we want multiple sessions per container NODE_MAX_INSTANCES: "${CI_JOBS:-4}" NODE_MAX_SESSION: "${CI_JOBS:-4}" - - -# Volumes below here diff --git a/docs/development/development-environment-docker/README.md b/docs/development/development-environment-docker/README.md index 040d9415ccb..00be3e68fe9 100644 --- a/docs/development/development-environment-docker/README.md +++ b/docs/development/development-environment-docker/README.md @@ -12,21 +12,28 @@ And nothing else! To get right into it and just start the application you can just do the following: -``` +```bash git clone https://github.com/opf/openproject.git cd openproject -bin/compose setup -bin/compose start +cp .env.example .env +cp docker-compose.override.example.yml docker-compose.override.yml +docker compose run --rm backend setup +docker compose run --rm frontend npm install +docker compose up -d frontend ``` Once the containers are done booting you can access the application under `http://localhost:3000`. ### Tests -You can run tests using `bin/compose rspec`. You can run specific tests too. For instance: +You can run tests inside the `backend-test` container. You can run specific tests, too. -``` -bin/compose rspec spec/features/work_package_show_spec.rb +```bash +# Run all tests (not recommended) +docker compose run --rm backend-test bundle exec rspec + +# Run the specified test +docker compose run --rm backend-test bundle exec rspec spec/features/work_package_show_spec.rb ``` *** @@ -34,92 +41,130 @@ bin/compose rspec spec/features/work_package_show_spec.rb More details and options follow in the next section. -> **Note**: docker-compose needs access to at least 4GB of RAM. E.g. for Mac, this requires to [increase the default limit of the virtualized host.](https://docs.docker.com/docker-for-mac/) Signs of lacking memory include an "Exit status 137" in the frontend container. +> **Note**: docker compose needs access to at least 4GB of RAM. E.g. for Mac, this requires +> to [increase the default limit of the virtualized host](https://docs.docker.com/docker-for-mac/). +> Signs of lacking memory include an "Exit status 137" in the frontend container. ## Step-by-step Setup ### 1) Checkout the code -First you will need to checkout the code as usual. +First you will need to check out the code as usual. -``` +```bash git clone https://github.com/opf/openproject.git ``` -This will checkout the dev branch in `openproject`. **Change into that directory.** +This will check out the dev branch in `openproject`. **Change into that directory.** If you have OpenProject checked out already make sure that you do not have a `config/database.yml` -as that will interfere with the database connection inside of the docker containers. +as that will interfere with the database connection inside the docker containers. ### 2) Configure environment Copy the env example to `.env` -``` +```bash cp .env.example .env ``` -Afterwards, set the environment variables to your liking. `DEV_UID` and `DEV_GID` are required to be set so your project +Afterward, set the environment variables to your liking. `DEV_UID` and `DEV_GID` are required to be set so your project directory will not end up with files owned by root. -Both `docker-compose` and `bin/compose` will load the env from this file. +`docker compose` will load the env from this file. + +You also will want to create a `docker-compose.override.yml` file, which can contain the port exposure of you +containers. Those are excluded from the main compose file `docker-compose.yml` for sanity reasons. If any port is +already in use, `docker compose` won't start and as you cannot disable the exposed port in +the `docker-compose.override.yml` file, you would have to alter the original `docker-compose.yml`. + +There is an example you can use out of the box. + +```bash +cp docker-compose.override.example.yml docker-compose.override.yml +``` ### 3) Setup database and install dependencies -``` -# Start the database. It needs to be running to run migrations and seeders -bin/compose up -d db +```bash +# This will start the database as a dependency +# and then run the migrations and seeders, +# and will install all required server dependencies +docker compose run --rm backend setup -# Install frontend dependencies -bin/compose run frontend npm i - -# Install backend dependencies, migrate, and seed -bin/compose run backend setup +# This will install the web dependencies +docker compose run --rm frontend npm install ``` ### 4) Start the stack The docker compose file also has the test containers defined. The easiest way to start only the development stack, use -``` -bin/compose up frontend +```bash +docker compose up frontend ``` -To see the backend logs as well, use +If you want to see the backend logs, too. -``` -bin/compose up frontend backend +```bash +docker compose up frontend backend ``` -This starts only the frontend and backend containers and their dependencies. This excludes the workers, which are needed -to execute certain background actions. Nevertheless, for most interactions the worker jobs are not needed. If needed, -the workers can be started with the following command. Be aware that those process will consume a lot of the system's -resources. +Alternatively, if you do want to detach from the process you can use the `-d` option. +```bash +docker compose up -d frontend ``` -bin/compose up worker + +The logs can still be accessed like this. + +```bash +# Print the logs of the `frontend` service until the time you execute this command +docker compose logs frontend + +# Print the logs of the `backend` service and follow the log outputs +docker compose logs -f backend +``` + +Those commands only start the frontend and backend containers and their dependencies. This excludes the workers, which +are needed to execute certain background actions. Nevertheless, for most interactions the worker jobs are not needed. If +needed, the workers can be started with the following command. Be aware that this process will consume a lot of the +system's resources. + +```bash +# Start the worker and let them run continuously +docker compose up -d worker + +# Start the worker to work off all delayed jobs and shut it down afterwards +docker compose run --rm worker rake jobs:workoff ``` The testing containers are excluded as well, while they are harmless to start, but take up system resources again and -clog your logs while running. The delayed_job background worker reloads the application for every job in development mode. This is a know issue and documented here: https://github.com/collectiveidea/delayed_job/issues/823 +clog your logs while running. The delayed_job background worker reloads the application for every job in development +mode. This is a know issue and documented here: https://github.com/collectiveidea/delayed_job/issues/823 This process can take quite a long time on the first run where all gems are installed for the first time. However, these are cached in a docker volume. Meaning that from the 2nd run onwards it will start a lot quicker. -Wait until you see `frontend_1 | : Compiled successfully.` -and `backend_1 | => Rails 6.0.2.2 application starting in development http://0.0.0.0:3000` -in the logs. This means both frontend and backend have come up successfully. You can now access OpenProject +Wait until you see `✔ Compiled successfully.` in the frontend logs and the success message from Puma in the backend +logs. This means both frontend and backend have come up successfully. You can now access OpenProject under http://localhost:3000, and via the live-reloaded under http://localhost:4200. Again the first request to the server can take some time too. But subsequent requests will be a lot faster. Changes you make to the code will be picked up automatically. No need to restart the containers. -## Docker +### Storybook -You can stop the processes via Ctrl + C. You can also run everything in the background by adding the `-d` option as -in `bin/compose up -d`. In that case you'll still be able to see the logs using `docker logs` with the respective -container name. You can see the started containers using `docker ps`. +There is a service to launch the storybook of the SPOT design system in the local development environment. To run it, +simply use: + +```bash +# Start the worker and let them run continuously +docker compose up -d storybook +``` + +If you used the default overrides you will access the storybook now under `http://localhost:6006`. ### Volumes @@ -132,32 +177,33 @@ There are volumes for * the test database (`_pgdata-test`) * the test tmp directory (`_tmp-test`) -This means these will stay between runs even if you stop and restart the containers. If you want to reset the data you -can delete the docker volumes via `docker volume rm`. +This means these will stay between runs even if you stop (or remove) and restart the containers. If you want to reset +the data you can delete the docker volumes via `docker volume rm`. ## Running tests Start all linked containers and migrate the test database first: -``` -bin/compose up backend-test +```bash +docker compose up -d backend-test ``` -Afterwards, you can start the tests in the running `backend-test` container: +Afterward, you can start the tests in the running `backend-test` container: -``` -bin/compose run backend-test bundle exec rspec +```bash +docker compose exec backend-test bundle exec rspec ``` or for running a particular test -``` -bin/compose run backend-test bundle exec rspec path/to/some_spec.rb +```bash +docker compose exec backend-test bundle exec rspec path/to/some_spec.rb ``` Tests are ran within Selenium containers, on a small local Selenium grid. You can connect to the containers via VNC if -you want to see what the browsers are doing. `gvncviewer` on Linux is a good tool for this. Check out the docker-compose -file to see which port each browser container is exposed on. The password is `secret` for all. +you want to see what the browsers are doing. `gvncviewer` or `vinagre` on Linux is a good tool for this. Set any port in +the `docker-compose.override.yml` to access a container of a specific browser. As a default, the `chrome` container is +exposed on port 5900. The password is `secret` for all. ## Local files @@ -167,23 +213,33 @@ pull the latest changes. ## Debugging -It's common to just start a debugger within ruby code using `binding.pry`. This **does not work** with the application +It's common to just start a debugger within ruby code using `binding.pry`. This works as expected with the application running as shown above. -If you want to be able to do that, you can, however, simply run the following: +However, keep in mind that you won't see the pry console unless you attach to the container of the `backend` service. +The easiest way to do that is getting the container name from the docker compose list and attaching to it with the +standard docker command. -``` -bin/compose run +```bash +# Check all running services and their containers. +# As a default the `backend` container has the name `openproject-backend-1` +docker compose ps + +# Attach to the container +docker attach openproject-backend-1 ``` -If the frontend container is not running yet, it will be started. If the backend container is already running, it will -be stopped. Instead it will be started in the foreground. This way you can debug using pry just as if you had started -the server locally using `rails s`. You can stop it simply with Ctrl + C too. +Inside the `backend` container you have a standard rails console. If you attached to the container **after** you run +into your breakpoint, you won't the common lines pry will print before your prompt. + +To detach from the `backend` container without stopping it, you can use `CTRL+P, CTRL+Q`. Using `CTRL+C` works, too, but +it will close and restart the backend container. ## Updates When a dependency of the image or the base image itself is changed you may need rebuild the image. For instance when the -Ruby version is updated you may run into an error like the following when running `bin/compose setup`: +Ruby version is updated you may run into an error like the following when +running `docker compose run --rm backend setup`: ``` Your Ruby version is 2.7.6, but your Gemfile specified ~> 3.2.1 @@ -191,6 +247,6 @@ Your Ruby version is 2.7.6, but your Gemfile specified ~> 3.2.1 This means that the current image is out-dated. You can update it like this: -``` -bin/compose build --pull +```bash +docker compose build --pull ```