Merged r11544, r11545, r11546, r11547, r11549 from trunk to 2.3-stable (#12641)

fix that diff outputs become ??? in some non ASCII words.

Contributed by Filou Centrinov.

git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/branches/2.3-stable@11551 e93f8b46-1217-0410-a6f0-8f06a7374b81

lib/redmine/unified_diff.rb

@@ -28,17 +28,9 @@ module Redmine
       lines = 0
       @truncated = false
       diff_table = DiffTable.new(diff_type, diff_style)
-      diff.each do |line|
-        line_encoding = nil
-        if line.respond_to?(:force_encoding)
-          line_encoding = line.encoding
-          # TODO: UTF-16 and Japanese CP932 which is imcompatible with ASCII
-          #       In Japan, diffrence between file path encoding
-          #       and file contents encoding is popular.
-          line.force_encoding('ASCII-8BIT')
-        end
-        unless diff_table.add_line line
-          line.force_encoding(line_encoding) if line_encoding
+      diff.each do |line_raw|
+        line = Redmine::CodesetUtil.to_utf8_by_setting(line_raw)
+        unless diff_table.add_line(line)
           self << diff_table if diff_table.length > 0
           diff_table = DiffTable.new(diff_type, diff_style)
         end
@@ -207,10 +199,16 @@ module Redmine
         while starting < max && line_left[starting] == line_right[starting]
           starting += 1
         end
+        while line_left[starting].ord.between?(128, 191) && starting > 0
+          starting -= 1
+        end
         ending = -1
         while ending >= -(max - starting) && line_left[ending] == line_right[ending]
           ending -= 1
         end
+        while line_left[ending].ord.between?(128, 191) && ending > -1
+          ending -= 1
+        end
         unless starting == 0 && ending == -1
           [starting, ending]
         end
@@ -268,6 +266,12 @@ module Redmine
     private
 
     def line_to_html(line, offsets)
+      html = line_to_html_raw(line, offsets)
+      html.force_encoding('UTF-8') if html.respond_to?(:force_encoding)
+      html
+    end
+
+    def line_to_html_raw(line, offsets)
       if offsets
         s = ''
         unless offsets.first == 0