[28349] Download recent version of pandoc if needed (#6563)

[ci skip]

Gemfile

@@ -177,6 +177,8 @@ gem 'openproject-token', '~> 1.0.1'
 
 gem 'plaintext', '0.1.0'
 
+gem 'rest-client', '~> 2.0'
+
 group :test do
   gem 'rack-test', '~> 1.0.0'
   gem 'shoulda-context', '~> 1.2'

Gemfile.lock

@@ -701,6 +701,7 @@ DEPENDENCIES
   reform-rails (~> 0.1.7)
   request_store (~> 1.4.1)
   responders (~> 2.4)
+  rest-client (~> 2.0)
   retriable (~> 3.1.1)
   rinku (~> 2.0.4)
   roar (~> 1.1.0)

db/migrate/20180706150714_convert_to_markdown.rb

@@ -29,6 +29,25 @@
 class ConvertToMarkdown < ActiveRecord::Migration[5.1]
   def up
     setting = Setting.where(name: 'text_formatting').pluck(:value)
+    return unless setting && setting[0] == 'textile'
+
+    if ENV['OPENPROJECT_SKIP_TEXTILE_MIGRATION'].present?
+      warn <<~WARNING
+        Your instance is configured with Textile text formatting, this means you have likely been running OpenProject before 8.0.0
+
+        Since you have requested skip the textile migration, your data will NOT be converted. You can do this in a subsequent step:
+
+        $> bundle exec rails runner "OpenProject::TextFormatting::Formats::Markdown::TextileConverter.new.run!"
+
+        or in a packaged installation:
+
+        $> openproject run bundle exec rails runner "OpenProject::TextFormatting::Formats::Markdown::TextileConverter.new.run!"
+
+        For more information, please visit this page: https://www.openproject.org/textile-to-markdown-migration
+
+        WARNING
+      return
+    end
 
     if setting && setting[0] == 'textile'
       converter = OpenProject::TextFormatting::Formats::Markdown::TextileConverter.new

docs/user/text-formatting/README.md

@@ -4,8 +4,6 @@ Starting with version 8.0.0, OpenProject features a quasi-WYSIWYG editor, powere
 
 The CKEditor5 build in OpenProject supports basic text styles, such as bold and italic formatting, headings, inline code and quotes as well as inline image handling. Pasting content such as images or rich text is also supported, while unsupported styling will be stripped by the editor.
 
-
-
 ### Links
 
 Create hyperlinks by pressing the toolbar (optionally with some selected text), or by pressing `CTRL+k` to open a popup to enter the link href.

docs/user/textile-to-markdown-migration/README.md

@@ -0,0 +1,61 @@
+# OpenProject Textile to Markdown migration
+
+OpenProject 8.0.0. includes a switch away from Textile syntax formatting to Markdown. Pandoc is used for the conversion of all formattable texts in your existing OpenProject instance.
+
+
+
+## Applicable Instances
+
+All instances that have run OpenProject prior to 8.0.0. must be converted in order to maintain the integrity of all formattable resources such as wiki pages, work packages, meetings, posts, comments, journals and so on.
+
+The migration will be performed automatically during your upgrade to 8.0. You will find additional information to prevent or postpone the migration down below.
+
+
+
+## Dependencies
+
+We depend on `pandoc` (http://pandoc.org/) for the conversion of all formattable fields in OpenProject. It provides automated means to migrate between many input and output formats, in our case from Textile to GitHub-flavored Markdown.
+
+If you do not have an executable pandoc version of at least version 2.0 in your path, OpenProject will try download an AMD64 static linked binary for pandoc (Currently, this would be version 2.3.2). This version will be made available to OpenProject through `<OpenProject root>/vendor/pandoc` and is only used during that one-time migration step.
+
+If you want to force a specific version within your path, set the environment variable OPENPROJECT_PANDOC_PATH, e.g., `OPENPROJECT_PANDOC_PATH=/opt/my/pandoc/bin/pandoc`.
+
+
+
+## CommonMark
+
+Our Markdown parsers and formatters operate on the [CommonMark Markdown standard](https://commonmark.org/) with some suggested additions not yet part of the standard formalized in the [GitHub-flavored Markdown spec.](https://github.github.com/gfm/])
+
+
+
+## Skipping the migration
+
+If you want to skip the migration during the upgrade of 8.0. (e.g., because you want it to run asynchronously), please set the environment variable `OPENPROJECT_SKIP_TEXTILE_MIGRATION="true"` .
+
+This will print a warning and then continue with the migration. You can manually force the migration with the following command. **Warning:** Be careful not to execute this once you have already migrated to Markdown because the converter does not distinguish between input formats and simply iterates over all values.
+
+
+
+        $> bundle exec rails runner "OpenProject::TextFormatting::Formats::Markdown::TextileConverter.new.run!"
+or in a packaged installation:
+
+```
+    $> openproject run bundle exec rails runner "OpenProject::TextFormatting::Formats::Markdown::TextileConverter.new.run!"
+```
+
+
+
+## Markdown and WYSIWYG features
+
+With the migration of Markdown, we have introduced a quasi-WYSIWYG powerd by CKEditor5  that will make editing in all formattable fields of OpenProject much easier. The output format of that editor is still Markdown.
+
+
+
+For information regarding the features of Markdown and the capabilities of the CKEditor WYSIWYG editor built upon it, please see https://www.openproject.org/help/text-formatting/.
+
+
+
+## Textile in 8.0.0.
+
+OpenProject does no longer support Textile in 8.0.0 because it is infeasible to support both variants. Please reach out to us if you're interested in maintaining a Textile format as a plugin. 
+

lib/open_project/text_formatting/formats/markdown/pandoc_downloader.rb

@@ -0,0 +1,125 @@
+#-- encoding: UTF-8
+
+#-- copyright
+# OpenProject is a project management system.
+# Copyright (C) 2012-2018 the OpenProject Foundation (OPF)
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2017 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See docs/COPYRIGHT.rdoc for more details.
+#++
+
+require 'open3'
+require 'fileutils'
+require 'rest-client'
+
+module OpenProject::TextFormatting::Formats
+  module Markdown
+    module PandocDownloader
+      class << self
+        def check_or_download!
+          # Return if we have another version defined in ENV
+          if forced_pandoc_path
+            return compatible? forced_pandoc_path,
+                               "Environment variable OPENPROJECT_PANDOC_PATH set, but version is not executable by OpenProject or incompatible."
+          end
+
+          # Return if we have a compatible version
+          return if compatible?
+
+          warn <<~INFO
+            You have no compatible (>= 2.0) of pandoc in path. We're now trying to download a recent version for your amd64 linux
+            from '#{pandoc_amd64_tar}' to '#{vendored_pandoc_dir}'.
+
+            For more information, please visit this page: https://www.openproject.org/textile-to-markdown-migration
+          INFO
+
+          download!
+
+          raise "Failed to download pandoc version" unless compatible?
+        rescue StandardError => e
+          warn <<~WARNING
+            Error occurred while trying to find / download compatible pandoc version for your system:
+
+            #{e.message}
+          WARNING
+        end
+
+        ##
+        # Check if the given pandoc version is compatible
+        # Returns true/false
+        # Raises raise_msg if set and incompatible
+        def compatible?(path = pandoc_path, raise_msg = nil)
+          stdout, _, status = Open3.capture3(path, '--version')
+
+          if !status.success? && raise_msg.present?
+            raise raise_msg
+          end
+
+          status.success? && stdout.match(/^pandoc [23]\./i)
+        end
+
+        def forced_pandoc_path
+          ENV['OPENPROJECT_PANDOC_PATH']
+        end
+
+        def pandoc_path
+          vendored = Rails.root.join('vendor/pandoc/bin/pandoc').to_s
+
+          # Always return the vendored path if we have installed one
+          return vendored if File.executable?(vendored)
+
+          ENV.fetch('OPENPROJECT_PANDOC_PATH', 'pandoc')
+        end
+
+        def vendored_pandoc_dir
+          Rails.root.join('vendor/pandoc').to_s
+        end
+
+        def pandoc_amd64_tar
+          ENV.fetch('OPENPROJECT_PANDOC_TAR_DOWNLOAD', 'https://github.com/jgm/pandoc/releases/download/2.2.3.2/pandoc-2.2.3.2-linux.tar.gz')
+        end
+
+        private
+
+        def download!
+          response = RestClient::Request.execute method: :get,
+                                                 url: pandoc_amd64_tar,
+                                                 raw_response: true
+          tempfile = response.file
+
+          # Create vendor dir, this will however usually exist already
+          FileUtils.mkdir_p vendored_pandoc_dir
+
+          begin
+            # Extract downloaded tar into vendor
+            _, stderr_str, status = Open3.capture3('tar', 'xvzf', tempfile.path.to_s,
+                                                   '--strip-components', '1', '-C', vendored_pandoc_dir)
+            raise stderr_str unless status.success?
+          ensure
+            tempfile.unlink
+          end
+        end
+      end
+    end
+  end
+end

lib/open_project/text_formatting/formats/markdown/pandoc_wrapper.rb

@@ -36,6 +36,7 @@ module OpenProject::TextFormatting::Formats
       def initialize; end
 
       def execute!(stdin)
+        PandocDownloader.check_or_download!
         run_pandoc! pandoc_arguments, stdin_data: stdin
       end
 
@@ -99,7 +100,7 @@ module OpenProject::TextFormatting::Formats
       ##
       # Run pandoc through posix-spawn and raise if an exception occurred
       def run_pandoc!(command, stdin_data: nil, timeout: pandoc_timeout)
-        child = POSIX::Spawn::Child.new('pandoc', *command, input: stdin_data, timeout: timeout)
+        child = POSIX::Spawn::Child.new(PandocDownloader.pandoc_path, *command, input: stdin_data, timeout: timeout)
         raise child.err unless child.status.success?
 
         child.out

vendor/.gitignore

@@ -0,0 +1 @@
+pandoc/