Extract and use charset to properly encode attachments

lib/api/helpers/attachment_renderer.rb

@@ -103,7 +103,8 @@ module API
         if attachment.is_text?
           # Even if the text mime type might differ, always output plain text
           # so this doesn't get interpreted as e.g., a script or html file
-          "text/plain"
+          charset = attachment.charset.presence || Setting.attachment_default_charset
+          "text/plain; charset=#{charset}"
         elsif attachment.inlineable?
           attachment.content_type
         else

lib/open_project/content_type_detector.rb

@@ -92,19 +92,19 @@ module OpenProject
       @filename = filename
     end
 
-    # Returns a String describing the file's content type
-    def detect
-      if blank_name?
-        SENSIBLE_DEFAULT
-      elsif empty_file?
-        EMPTY_TYPE
-      elsif calculated_type_matches.any?
-        calculated_type_matches.first
-      else
-        type_from_file_command || SENSIBLE_DEFAULT
-      end.to_s
+    # Returns [mime_type, charset_or_nil], running the file command once.
+    def detect_with_charset
+      return [SENSIBLE_DEFAULT, nil] if blank_name?
+      return [EMPTY_TYPE, nil] if empty_file?
+
+      raw_mime, charset = FileCommandContentTypeDetector.new(@filename).detect
+      [resolve_mime(raw_mime), charset]
     end
 
+    # Detecting only the mime type is effectively the
+    # first argument of +detect_with_charset+
+    def detect = detect_with_charset.first
+
     private
 
     def empty_file?
@@ -121,12 +121,9 @@ module OpenProject
       MIME::Types.type_for(@filename).map(&:content_type)
     end
 
-    def calculated_type_matches
-      possible_types.select { |content_type| content_type == type_from_file_command }
-    end
-
-    def type_from_file_command
-      @type_from_file_command ||= FileCommandContentTypeDetector.new(@filename).detect
+    def resolve_mime(raw_mime)
+      matches = possible_types.select { |ct| ct == raw_mime }
+      (matches.first || raw_mime || SENSIBLE_DEFAULT).to_s
     end
   end
 end

lib/open_project/file_command_content_type_detector.rb

@@ -69,23 +69,31 @@ module OpenProject
       @filename = filename
     end
 
+    # Returns [mime_type, charset_or_nil], e.g.:
+    #   ["text/plain", "utf-8"]
+    #   ["image/png",  nil]
     def detect
-      type_from_file_command
+      @detect ||= parse_file_command
     end
 
     private
 
-    def type_from_file_command
+    def parse_file_command
       # On BSDs, `file` doesn't give a result code of 1 if the file doesn't exist.
       type, status = Open3.capture2("file", "-b", "--mime", "--", @filename)
+      return [SENSIBLE_DEFAULT, nil] if type.nil? || status.to_i > 0 || type.match(/\(.*?\)/)
 
-      if type.nil? || status.to_i > 0 || type.match(/\(.*?\)/)
-        type = SENSIBLE_DEFAULT
-      end
-      type.split(/[:;\s]+/)[0]
+      extract_mime_and_charset(type.strip)
     rescue StandardError => e
       Rails.logger.info { "Failed to get mime type from #{@filename}: #{e} #{e.message}" }
-      SENSIBLE_DEFAULT
+      [SENSIBLE_DEFAULT, nil]
+    end
+
+    def extract_mime_and_charset(type)
+      mime, charset_param = type.split(";", 2).map(&:strip)
+      charset = charset_param&.match(/\Acharset=(.+)\z/)&.[](1)
+      charset = nil if charset == "binary"
+      [mime, charset]
     end
   end
 end