Use custom urlencode function. fixes #121

couchpotato/core/downloaders/sabnzbd/main.py

@@ -1,9 +1,9 @@
 from couchpotato.core.downloaders.base import Downloader
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import cleanHost
 from couchpotato.core.logger import CPLog
 from inspect import isfunction
 from tempfile import mkstemp
-from urllib import urlencode
 import base64
 import os
 import re
@@ -58,7 +58,7 @@ class Sabnzbd(Downloader):
         if pp:
             params['script'] = pp_script_fn
 
-        url = cleanHost(self.conf('host')) + "api?" + urlencode(params)
+        url = cleanHost(self.conf('host')) + "api?" + tryUrlencode(params)
 
         try:
             if params.get('mode') is 'addfile':

couchpotato/core/helpers/encoding.py

@@ -1,5 +1,6 @@
 from couchpotato.core.logger import CPLog
 from string import ascii_letters, digits
+from urllib import quote_plus
 import re
 import unicodedata
 
@@ -19,7 +20,7 @@ def simplifyString(original):
 
 def toUnicode(original, *args):
     try:
-        if type(original) is unicode:
+        if isinstance(original, unicode):
             return original
         else:
             try:
@@ -35,7 +36,7 @@ def toUnicode(original, *args):
         return toUnicode(ascii_text)
 
 def ek(original, *args):
-    if type(original) in [str, unicode]:
+    if isinstance(original, (str, unicode)):
         try:
             from couchpotato.environment import Env
             return original.decode(Env.get('encoding'))
@@ -53,3 +54,19 @@ def isInt(value):
 
 def stripAccents(s):
     return ''.join((c for c in unicodedata.normalize('NFD', toUnicode(s)) if unicodedata.category(c) != 'Mn'))
+
+def tryUrlencode(s):
+    new = u''
+    if isinstance(s, (dict)):
+        for key, value in s.iteritems():
+            new += u'&%s=%s' % (key, tryUrlencode(value))
+
+        return new[1:]
+    else:
+        for letter in toUnicode(s):
+            try:
+                new += quote_plus(letter)
+            except:
+                new += letter
+
+    return new

couchpotato/core/helpers/request.py

@@ -1,3 +1,4 @@
+from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.helpers.variable import natcmp
 from flask.globals import current_app
 from flask.helpers import json
@@ -25,7 +26,7 @@ def getParams():
 
             for item in nested:
                 if item is nested[-1]:
-                    current[item] = unquote_plus(value)
+                    current[item] = toUnicode(unquote_plus(value)).encode('utf-8')
                 else:
                     try:
                         current[item]
@@ -34,7 +35,7 @@ def getParams():
 
                     current = current[item]
         else:
-            temp[param] = unquote_plus(value)
+            temp[param] = toUnicode(unquote_plus(value)).encode('utf-8')
 
     return dictToList(temp)
 
@@ -56,7 +57,7 @@ def dictToList(params):
 
 def getParam(attr, default = None):
     try:
-        return unquote_plus(getattr(flask.request, 'args').get(attr, default))
+        return toUnicode(unquote_plus(getattr(flask.request, 'args').get(attr, default))).encode('utf-8')
     except:
         return None
 

couchpotato/core/notifications/nmj/main.py

@@ -1,12 +1,11 @@
 from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.request import getParams, jsonified
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
-from couchpotato.environment import Env
 import re
 import telnetlib
-import urllib
 
 try:
     import xml.etree.cElementTree as etree
@@ -90,7 +89,7 @@ class NMJ(Notification):
             'arg2': 'background',
             'arg3': '',
         }
-        params = urllib.urlencode(params)
+        params = tryUrlencode(params)
         UPDATE_URL = 'http://%(host)s:8008/metadata_database?%(params)s'
         updateUrl = UPDATE_URL % {'host': host, 'params': params}
 

couchpotato/core/notifications/prowl/main.py

@@ -1,8 +1,7 @@
-from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 from httplib import HTTPSConnection
-from urllib import urlencode
 
 log = CPLog(__name__)
 
@@ -24,7 +23,7 @@ class Prowl(Notification):
         http_handler.request('POST',
             '/publicapi/add',
             headers = {'Content-type': 'application/x-www-form-urlencoded'},
-            body = urlencode(data)
+            body = tryUrlencode(data)
         )
         response = http_handler.getresponse()
         request_status = response.status

couchpotato/core/notifications/twitter/main.py

@@ -1,4 +1,5 @@
 from couchpotato.api import addApiView
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.request import jsonified, getParam
 from couchpotato.core.helpers.variable import cleanHost
 from couchpotato.core.logger import CPLog
@@ -7,7 +8,6 @@ from flask.helpers import url_for
 from pytwitter import Api, parse_qsl
 from werkzeug.utils import redirect
 import oauth2
-import urllib
 
 log = CPLog(__name__)
 
@@ -56,7 +56,7 @@ class Twitter(Notification):
         oauth_consumer = oauth2.Consumer(self.consumer_key, self.consumer_secret)
         oauth_client = oauth2.Client(oauth_consumer)
 
-        resp, content = oauth_client.request(self.urls['request'], 'POST', body = urllib.urlencode({'oauth_callback': callback_url}))
+        resp, content = oauth_client.request(self.urls['request'], 'POST', body = tryUrlencode({'oauth_callback': callback_url}))
 
         if resp['status'] != '200':
             log.error('Invalid response from Twitter requesting temp token: %s' % resp['status'])

couchpotato/core/notifications/xbmc/main.py

@@ -1,7 +1,7 @@
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 import base64
-import urllib
 
 log = CPLog(__name__)
 
@@ -21,7 +21,7 @@ class XBMC(Notification):
 
     def send(self, command, host):
 
-        url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, urllib.urlencode(command))
+        url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, tryUrlencode(command))
 
         headers = {}
         if self.conf('password'):

couchpotato/core/plugins/base.py

@@ -1,5 +1,6 @@
 from couchpotato import addView
 from couchpotato.core.event import fireEvent, addEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import getExt
 from couchpotato.core.logger import CPLog
 from couchpotato.environment import Env
@@ -11,10 +12,8 @@ import glob
 import math
 import os.path
 import re
-import socket
 import time
 import traceback
-import urllib
 import urllib2
 
 log = CPLog(__name__)
@@ -114,7 +113,6 @@ class Plugin(object):
                 del self.http_failed_disabled[host]
 
         self.wait(host)
-
         try:
 
             if multipart:
@@ -127,7 +125,7 @@ class Plugin(object):
                 data = opener.open(request, timeout = timeout).read()
             else:
                 log.info('Opening url: %s, params: %s' % (url, [x for x in params.iterkeys()]))
-                data = urllib.urlencode(params) if len(params) > 0 else None
+                data = tryUrlencode(params) if len(params) > 0 else None
                 request = urllib2.Request(url, data, headers)
 
                 data = urllib2.urlopen(request, timeout = timeout).read()

couchpotato/core/plugins/library/main.py

@@ -1,6 +1,7 @@
 from couchpotato import get_session
 from couchpotato.core.event import addEvent, fireEventAsync, fireEvent
-from couchpotato.core.helpers.encoding import toUnicode, simplifyString
+from couchpotato.core.helpers.encoding import toUnicode, simplifyString, \
+    tryUrlencode
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.settings.model import Library, LibraryTitle, File
@@ -45,7 +46,7 @@ class LibraryPlugin(Plugin):
         # Update library info
         if update_after is not False:
             handle = fireEventAsync if update_after is 'async' else fireEvent
-            handle('library.update', identifier = l.identifier, default_title = attrs.get('title', ''))
+            handle('library.update', identifier = l.identifier, default_title = toUnicode(attrs.get('title', '')))
 
         return l.to_dict(self.default_dict)
 
@@ -86,10 +87,11 @@ class LibraryPlugin(Plugin):
             for title in titles:
                 if not title:
                     continue
+                title = toUnicode(title)
                 t = LibraryTitle(
-                    title = toUnicode(title),
+                    title = title,
                     simple_title = self.simplifyTitle(title),
-                    default = title.lower() == default_title.lower() or (default_title is '' and titles[0] == title)
+                    default = title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == u'' and toUnicode(titles[0]) == title)
                 )
                 library.titles.append(t)
 

couchpotato/core/plugins/movie/main.py

@@ -1,8 +1,9 @@
 from couchpotato import get_session
 from couchpotato.api import addApiView
 from couchpotato.core.event import fireEvent, fireEventAsync, addEvent
-from couchpotato.core.helpers.encoding import toUnicode
-from couchpotato.core.helpers.request import getParams, jsonified
+from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
+    simplifyString
+from couchpotato.core.helpers.request import getParams, jsonified, getParam
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.settings.model import Movie, Library, LibraryTitle
@@ -10,7 +11,6 @@ from couchpotato.environment import Env
 from sqlalchemy.orm import joinedload_all
 from sqlalchemy.sql.expression import or_, asc, not_
 from string import ascii_lowercase
-from urllib import urlencode
 
 log = CPLog(__name__)
 
@@ -225,7 +225,6 @@ class MoviePlugin(Plugin):
             if title.default: default_title = title.title
 
         if movie:
-            #addEvent('library.update.after', )
             fireEventAsync('library.update', identifier = movie.library.identifier, default_title = default_title, force = True)
             fireEventAsync('searcher.single', movie.to_dict(self.default_dict))
 
@@ -235,12 +234,12 @@ class MoviePlugin(Plugin):
 
     def search(self):
 
-        params = getParams()
-        cache_key = '%s/%s' % (__name__, urlencode(params))
+        q = getParam('q')
+        cache_key = u'%s/%s' % (__name__, simplifyString(q))
         movies = Env.get('cache').get(cache_key)
 
         if not movies:
-            movies = fireEvent('movie.search', q = params.get('q'), merge = True)
+            movies = fireEvent('movie.search', q = q, merge = True)
             Env.get('cache').set(cache_key, movies)
 
         return jsonified({
@@ -335,7 +334,7 @@ class MoviePlugin(Plugin):
             # Default title
             if params.get('default_title'):
                 for title in m.library.titles:
-                    title.default = params.get('default_title').lower() == title.title.lower()
+                    title.default = toUnicode(params.get('default_title', '')).lower() == toUnicode(title.title).lower()
 
             db.commit()
 

couchpotato/core/providers/movie/imdbapi/main.py

@@ -1,8 +1,8 @@
 from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import tryInt, tryFloat
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.movie.base import MovieProvider
-from urllib import urlencode
 import json
 import re
 import traceback
@@ -27,8 +27,11 @@ class IMDBAPI(MovieProvider):
 
         name_year = fireEvent('scanner.name_year', q, single = True)
 
+        if not name_year.get('name'):
+            return []
+
         cache_key = 'imdbapi.cache.%s' % q
-        cached = self.getCache(cache_key, self.urls['search'] % urlencode({'t': name_year.get('name'), 'y': name_year.get('year')}))
+        cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}))
 
         if cached:
             result = self.parseMovie(cached)

couchpotato/core/providers/nzb/moovee/main.py

@@ -1,8 +1,8 @@
 from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
 from dateutil.parser import parse
-from urllib import quote_plus
 import re
 import time
 
@@ -27,7 +27,7 @@ class Moovee(NZBProvider):
             return results
 
         q = '%s %s' % (movie['library']['titles'][0]['title'], quality.get('identifier'))
-        url = self.urls['search'] % quote_plus(q)
+        url = self.urls['search'] % tryUrlencode(q)
 
         cache_key = 'moovee.%s' % q
         data = self.getCache(cache_key, url)

couchpotato/core/providers/nzb/mysterbin/main.py

@@ -1,11 +1,10 @@
 from BeautifulSoup import BeautifulSoup
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
-import urllib
 
 log = CPLog(__name__)
 
@@ -40,8 +39,8 @@ class Mysterbin(NZBProvider):
             'nopasswd': 'on',
         }
 
-        cache_key = 'mysterbin.%s' % q
-        data = self.getCache(cache_key, self.urls['search'] % urllib.urlencode(params))
+        cache_key = 'mysterbin.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
+        data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params))
         if data:
 
             try:

couchpotato/core/providers/nzb/newzbin/main.py

@@ -1,9 +1,9 @@
 from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
 from dateutil.parser import parse
-from urllib import urlencode
 import base64
 import time
 import xml.etree.ElementTree as XMLTree
@@ -45,7 +45,7 @@ class Newzbin(NZBProvider, RSS):
         format_id = self.getFormatId(type)
         cat_id = self.getCatId(type)
 
-        arguments = urlencode({
+        arguments = tryUrlencode({
             'searchaction': 'Search',
             'u_url_posts_only': '0',
             'u_show_passworded': '0',

couchpotato/core/providers/nzb/newznab/main.py

@@ -1,10 +1,10 @@
 from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import cleanHost
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
 from dateutil.parser import parse
-from urllib import urlencode
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -48,7 +48,7 @@ class Newznab(NZBProvider, RSS):
         if self.isDisabled(host) or not self.isAvailable(self.getUrl(host['host'], self.urls['search'])):
             return results
 
-        arguments = urlencode({
+        arguments = tryUrlencode({
             't': self.cat_backup_id,
             'r': host['api_key'],
             'i': 58,
@@ -81,7 +81,7 @@ class Newznab(NZBProvider, RSS):
             return results
 
         cat_id = self.getCatId(quality['identifier'])
-        arguments = urlencode({
+        arguments = tryUrlencode({
             'imdbid': movie['library']['identifier'].replace('tt', ''),
             'cat': cat_id[0],
             'apikey': host['api_key'],

couchpotato/core/providers/nzb/nzbclub/main.py

@@ -1,6 +1,5 @@
-from BeautifulSoup import BeautifulSoup, SoupStrainer
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
@@ -8,7 +7,6 @@ from couchpotato.core.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
 import time
-import urllib
 import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
@@ -41,8 +39,8 @@ class NZBClub(NZBProvider, RSS):
             'ns': 1,
         }
 
-        cache_key = 'nzbclub.%s' % q
-        data = self.getCache(cache_key, self.urls['search'] % urllib.urlencode(params))
+        cache_key = 'nzbclub.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
+        data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params))
         if data:
             try:
                 try:

couchpotato/core/providers/nzb/nzbindex/main.py

@@ -1,12 +1,11 @@
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
-from urllib import urlencode
 import re
 import time
 import xml.etree.ElementTree as XMLTree
@@ -30,7 +29,7 @@ class NzbIndex(NZBProvider, RSS):
             return results
 
         q = '%s %s %s' % (movie['library']['titles'][0]['title'], movie['library']['year'], quality.get('identifier'))
-        arguments = urlencode({
+        arguments = tryUrlencode({
             'q': q,
             'age': Env.setting('retention', 'nzb'),
             'sort': 'agedesc',
@@ -43,7 +42,7 @@ class NzbIndex(NZBProvider, RSS):
         })
         url = "%s?%s" % (self.urls['api'], arguments)
 
-        cache_key = 'nzbindex.%s' % q
+        cache_key = 'nzbindex.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
 
         data = self.getCache(cache_key, url)
         if data:

couchpotato/core/providers/nzb/nzbmatrix/main.py

@@ -1,10 +1,10 @@
 from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
-from urllib import urlencode
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -37,7 +37,7 @@ class NZBMatrix(NZBProvider, RSS):
 
         cat_ids = ','.join(['%s' % x for x in self.getCatId(quality.get('identifier'))])
 
-        arguments = urlencode({
+        arguments = tryUrlencode({
             'term': movie['library']['identifier'],
             'subcat': cat_ids,
             'username': self.conf('username'),

couchpotato/core/providers/nzb/nzbs/main.py

@@ -1,10 +1,9 @@
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.encoding import simplifyString
+from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
 from dateutil.parser import parse
-from urllib import urlencode
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -36,7 +35,7 @@ class Nzbs(NZBProvider, RSS):
             return results
 
         cat_id = self.getCatId(quality.get('identifier'))
-        arguments = urlencode({
+        arguments = tryUrlencode({
             'action':'search',
             'q': simplifyString(movie['library']['titles'][0]['title']),
             'catid': cat_id[0],

couchpotato/core/providers/nzb/x264/main.py

@@ -1,8 +1,8 @@
 from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
-from urllib import quote_plus
 import re
 
 log = CPLog(__name__)
@@ -26,9 +26,9 @@ class X264(NZBProvider):
             return results
 
         q = '%s %s %s' % (movie['library']['titles'][0]['title'], movie['library']['year'], quality.get('identifier'))
-        url = self.urls['search'] % quote_plus(q)
+        url = self.urls['search'] % tryUrlencode(q)
 
-        cache_key = 'x264.%s' % q
+        cache_key = 'x264.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
         data = self.getCache(cache_key, url)
         if data:
             match = re.compile(self.regex, re.DOTALL).finditer(data)

couchpotato/core/providers/torrent/kickasstorrents/main.py

@@ -36,7 +36,7 @@ class KickAssTorrents(TorrentProvider):
         if self.isDisabled() or not self.isAvailable(self.urls['test']):
             return results
 
-        cache_key = 'kickasstorrents.%s' % movie['library']['identifier']
+        cache_key = 'kickasstorrents.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
         data = self.getCache(cache_key, self.urls['search'] % (movie['library']['titles'][0]['title'], movie['library']['identifier'].replace('tt', '')))
         if data:
 

couchpotato/core/providers/trailer/hdtrailers/main.py

@@ -1,9 +1,9 @@
 from BeautifulSoup import SoupStrainer, BeautifulSoup
+from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import mergeDicts
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.trailer.base import TrailerProvider
 from string import letters, digits
-from urllib import urlencode
 import re
 
 log = CPLog(__name__)
@@ -44,7 +44,7 @@ class HDTrailers(TrailerProvider):
 
         movie_name = group['library']['titles'][0]['title']
 
-        url = "%s?%s" % (self.url['backup'], urlencode({'s':movie_name}))
+        url = "%s?%s" % (self.url['backup'], tryUrlencode({'s':movie_name}))
         data = self.getCache('hdtrailers.alt.%s' % group['library']['identifier'], url)
 
         try: