Adds support for commit logs reencoding to UTF-8 before insertion in the database (#834, #917, #1663).

Source encoding of commit logs can be selected in Application settings -> Repositories.

git-svn-id: http://redmine.rubyforge.org/svn/trunk@1767 e93f8b46-1217-0410-a6f0-8f06a7374b81

app/models/changeset.rb

@@ -15,6 +15,8 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
+require 'iconv'
+
 class Changeset < ActiveRecord::Base
   belongs_to :repository
   has_many :changes, :dependent => :delete_all
@@ -43,7 +45,7 @@ class Changeset < ActiveRecord::Base
   end
   
   def comments=(comment)
-    write_attribute(:comments, comment.strip)
+    write_attribute(:comments, to_utf8(comment.to_s.strip))
   end
 
   def committed_on=(date)
@@ -131,4 +133,19 @@ class Changeset < ActiveRecord::Base
   def next
     @next ||= Changeset.find(:first, :conditions => ['id > ? AND repository_id = ?', self.id, self.repository_id], :order => 'id ASC')
   end
+  
+  private
+  
+  def to_utf8(str)
+    return str if /\A[\r\n\t\x20-\x7e]*\Z/n.match(str) # for us-ascii
+    encoding = Setting.commit_logs_encoding.to_s.strip
+    unless encoding.blank? || encoding == 'UTF-8'
+      begin
+        return Iconv.conv('UTF-8', encoding, str)
+      rescue Iconv::Failure
+        # do nothing here
+      end
+    end
+    str
+  end
 end

app/models/setting.rb

@@ -33,6 +33,45 @@ class Setting < ActiveRecord::Base
     '%H:%M',
     '%I:%M %p'
     ]
+    
+  ENCODINGS = %w(US-ASCII
+                  windows-1250
+                  windows-1251
+                  windows-1252
+                  windows-1253
+                  windows-1254
+                  windows-1255
+                  windows-1256
+                  windows-1257
+                  windows-1258
+                  windows-31j
+                  ISO-2022-JP
+                  ISO-2022-KR
+                  ISO-8859-1
+                  ISO-8859-2
+                  ISO-8859-3
+                  ISO-8859-4
+                  ISO-8859-5
+                  ISO-8859-6
+                  ISO-8859-7
+                  ISO-8859-8
+                  ISO-8859-9
+                  ISO-8859-13
+                  ISO-8859-15
+                  KOI8-R
+                  UTF-8
+                  UTF-16
+                  UTF-16BE
+                  UTF-16LE
+                  EUC-JP
+                  Shift_JIS
+                  GB18030
+                  GBK
+                  ISCII91
+                  EUC-KR
+                  Big5
+                  Big5-HKSCS
+                  TIS-620)
   
   cattr_accessor :available_settings
   @@available_settings = YAML::load(File.open("#{RAILS_ROOT}/config/settings.yml"))

app/views/settings/_repositories.rhtml

@@ -16,6 +16,9 @@
 
 <p><label><%= l(:setting_repositories_encodings) %></label>
 <%= text_field_tag 'settings[repositories_encodings]', Setting.repositories_encodings, :size => 60 %><br /><em><%= l(:text_comma_separated) %></em></p>
+
+<p><label><%= l(:setting_commit_logs_encoding) %></label>
+<%= select_tag 'settings[commit_logs_encoding]', options_for_select(Setting::ENCODINGS, Setting.commit_logs_encoding) %></p>
 </div>
 
 <fieldset class="box tabular settings"><legend><%= l(:text_issues_ref_in_commit_messages) %></legend>