Update requests lib

libs/requests/packages/charade/__init__.py

@@ -30,3 +30,37 @@ def detect(aBuf):
     u.feed(aBuf)
     u.close()
     return u.result
+
+def _description_of(path):
+    """Return a string describing the probable encoding of a file."""
+    from charade.universaldetector import UniversalDetector
+
+    u = UniversalDetector()
+    for line in open(path, 'rb'):
+        u.feed(line)
+    u.close()
+    result = u.result
+    if result['encoding']:
+        return '%s: %s with confidence %s' % (path,
+                                              result['encoding'],
+                                              result['confidence'])
+    else:
+        return '%s: no result' % path
+
+
+def charade_cli():
+    """
+    Script which takes one or more file paths and reports on their detected
+    encodings
+
+    Example::
+
+        % chardetect.py somefile someotherfile
+        somefile: windows-1252 with confidence 0.5
+        someotherfile: ascii with confidence 1.0
+
+    """
+    from sys import argv
+    for path in argv[1:]:
+        print(_description_of(path))
+        

libs/requests/packages/charade/__main__.py

@@ -0,0 +1,7 @@
+'''
+support ';python -m charade <file1> [file2] ...' package execution syntax (2.7+)
+'''
+
+from charade import charade_cli
+
+charade_cli()

libs/requests/packages/charade/jpcntx.py

@@ -169,7 +169,7 @@ class JapaneseContextAnalysis:
     def get_confidence(self):
         # This is just one way to calculate confidence. It works well for me.
         if self._mTotalRel > MINIMUM_DATA_THRESHOLD:
-            return (self._mTotalRel - self._mRelSample[0]) / self._mTotalRel
+            return float(self._mTotalRel - self._mRelSample[0]) / self._mTotalRel
         else:
             return DONT_KNOW
 

libs/requests/packages/charade/latin1prober.py

@@ -129,7 +129,7 @@ class Latin1Prober(CharSetProber):
         if total < 0.01:
             confidence = 0.0
         else:
-            confidence = ((self._mFreqCounter[3] / total)
+            confidence = ((float(self._mFreqCounter[3]) / total)
                           - (self._mFreqCounter[1] * 20.0 / total))
         if confidence < 0.0:
             confidence = 0.0

libs/requests/packages/charade/universaldetector.py

@@ -74,12 +74,10 @@ class UniversalDetector:
             if aBuf[:3] == codecs.BOM:
                 # EF BB BF  UTF-8 with BOM
                 self.result = {'encoding': "UTF-8", 'confidence': 1.0}
-            elif aBuf[:4] == codecs.BOM_UTF32_LE:
+            elif aBuf[:4] in (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE):
                 # FF FE 00 00  UTF-32, little-endian BOM
-                self.result = {'encoding': "UTF-32LE", 'confidence': 1.0}
-            elif aBuf[:4] == codecs.BOM_UTF32_BE:
                 # 00 00 FE FF  UTF-32, big-endian BOM
-                self.result = {'encoding': "UTF-32BE", 'confidence': 1.0}
+                self.result = {'encoding': "UTF-32", 'confidence': 1.0}
             elif aBuf[:4] == b'\xFE\xFF\x00\x00':
                 # FE FF 00 00  UCS-4, unusual octet order BOM (3412)
                 self.result = {
@@ -92,12 +90,10 @@ class UniversalDetector:
                     'encoding': "X-ISO-10646-UCS-4-2143",
                     'confidence': 1.0
                 }
-            elif aBuf[:2] == codecs.BOM_LE:
+            elif aBuf[:2] == codecs.BOM_LE or aBuf[:2] == codecs.BOM_BE:
                 # FF FE  UTF-16, little endian BOM
-                self.result = {'encoding': "UTF-16LE", 'confidence': 1.0}
-            elif aBuf[:2] == codecs.BOM_BE:
                 # FE FF  UTF-16, big endian BOM
-                self.result = {'encoding': "UTF-16BE", 'confidence': 1.0}
+                self.result = {'encoding': "UTF-16", 'confidence': 1.0}
 
         self._mGotData = True
         if self.result['encoding'] and (self.result['confidence'] > 0.0):