Use requests lib for openurl

couchpotato/core/providers/base.py

@@ -5,12 +5,10 @@ from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
 from urlparse import urlparse
-import cookielib
 import json
 import re
 import time
 import traceback
-import urllib2
 import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
@@ -95,7 +93,7 @@ class Provider(Plugin):
 
     def getHTMLData(self, url, **kwargs):
 
-        cache_key = '%s%s' % (md5(url), md5('%s' % kwargs.get('params', {})))
+        cache_key = '%s%s' % (md5(url), md5('%s' % kwargs.get('data', {})))
         return self.getCache(cache_key, url, **kwargs)
 
 
@@ -111,8 +109,7 @@ class YarrProvider(Provider):
     sizeMb = ['mb', 'mib']
     sizeKb = ['kb', 'kib']
 
-    login_opener = None
-    last_login_check = 0
+    last_login_check = None
 
     def __init__(self):
         addEvent('provider.enabled_protocols', self.getEnabledProtocol)
@@ -129,35 +126,30 @@ class YarrProvider(Provider):
 
         # Check if we are still logged in every hour
         now = time.time()
-        if self.login_opener and self.last_login_check < (now - 3600):
+        if self.last_login_check and self.last_login_check < (now - 3600):
             try:
-                output = self.urlopen(self.urls['login_check'], opener = self.login_opener)
+                output = self.urlopen(self.urls['login_check'])
                 if self.loginCheckSuccess(output):
                     self.last_login_check = now
                     return True
-                else:
-                    self.login_opener = None
-            except:
-                self.login_opener = None
+            except: pass
+            self.last_login_check = None
 
-        if self.login_opener:
+        if self.last_login_check:
             return True
 
         try:
-            cookiejar = cookielib.CookieJar()
-            opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
-            output = self.urlopen(self.urls['login'], params = self.getLoginParams(), opener = opener)
+            output = self.urlopen(self.urls['login'], data = self.getLoginParams())
 
             if self.loginSuccess(output):
                 self.last_login_check = now
-                self.login_opener = opener
                 return True
 
             error = 'unknown'
         except:
             error = traceback.format_exc()
 
-        self.login_opener = None
+        self.last_login_check = None
         log.error('Failed to login %s: %s', (self.getName(), error))
         return False
 
@@ -171,12 +163,12 @@ class YarrProvider(Provider):
         try:
             if not self.login():
                 log.error('Failed downloading from %s', self.getName())
-            return self.urlopen(url, opener = self.login_opener)
+            return self.urlopen(url)
         except:
             log.error('Failed downloading from %s: %s', (self.getName(), traceback.format_exc()))
 
     def getLoginParams(self):
-        return ''
+        return {}
 
     def download(self, url = '', nzb_id = ''):
         try:

couchpotato/core/providers/info/couchpotatoapi/main.py

@@ -97,7 +97,7 @@ class CouchPotatoApi(MovieProvider):
         if not ignore: ignore = []
         if not movies: movies = []
 
-        suggestions = self.getJsonData(self.urls['suggest'], params = {
+        suggestions = self.getJsonData(self.urls['suggest'], data = {
             'movies': ','.join(movies),
             'ignore': ','.join(ignore),
         }, headers = self.getRequestHeaders())

couchpotato/core/providers/nzb/binsearch/main.py

@@ -90,13 +90,13 @@ class BinSearch(NZBProvider):
 
     def download(self, url = '', nzb_id = ''):
 
-        params = {
+        data = {
             'action': 'nzb',
             nzb_id: 'on'
         }
 
         try:
-            return self.urlopen(url, params = params, show_error = False)
+            return self.urlopen(url, data = data, show_error = False)
         except:
             log.error('Failed getting nzb from %s: %s', (self.getName(), traceback.format_exc()))
 

couchpotato/core/providers/torrent/bithdtv/main.py

@@ -31,7 +31,7 @@ class BiTHDTV(TorrentProvider):
 
         url = "%s&%s" % (self.urls['search'], arguments)
 
-        data = self.getHTMLData(url, opener = self.login_opener)
+        data = self.getHTMLData(url)
 
         if data:
             # Remove BiT-HDTV's output garbage so outdated BS4 versions successfully parse the HTML
@@ -68,10 +68,10 @@ class BiTHDTV(TorrentProvider):
                 log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
 
     def getLoginParams(self):
-        return tryUrlencode({
+        return {
             'username': self.conf('username'),
             'password': self.conf('password'),
-        })
+        }
 
     def getMoreInfo(self, item):
         full_description = self.getCache('bithdtv.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)

couchpotato/core/providers/torrent/bitsoup/main.py

@@ -28,7 +28,7 @@ class Bitsoup(TorrentProvider):
         })
         url = "%s&%s" % (self.urls['search'], arguments)
 
-        data = self.getHTMLData(url, opener = self.login_opener)
+        data = self.getHTMLData(url)
 
         if data:
             html = BeautifulSoup(data)
@@ -73,11 +73,11 @@ class Bitsoup(TorrentProvider):
 
 
     def getLoginParams(self):
-        return tryUrlencode({
+        return {
             'username': self.conf('username'),
             'password': self.conf('password'),
             'ssl': 'yes',
-        })
+        }
 
 
     def loginSuccess(self, output):

couchpotato/core/providers/torrent/hdbits/main.py

@@ -1,5 +1,4 @@
 from bs4 import BeautifulSoup
-from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.torrent.base import TorrentProvider
@@ -23,7 +22,7 @@ class HDBits(TorrentProvider):
 
     def _search(self, movie, quality, results):
 
-        data = self.getJsonData(self.urls['search'] % movie['library']['identifier'], opener = self.login_opener)
+        data = self.getJsonData(self.urls['search'] % movie['library']['identifier'])
 
         if data:
             try:
@@ -42,15 +41,17 @@ class HDBits(TorrentProvider):
                 log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
 
     def getLoginParams(self):
-        data = self.getHTMLData('https://hdbits.org/login')
+        data = self.getHTMLData('https://hdbits.org/login', cache_timeout = 0)
+
         bs = BeautifulSoup(data)
         secret = bs.find('input', attrs = {'name': 'lol'})['value']
 
-        return tryUrlencode({
+        return {
             'uname': self.conf('username'),
             'password': self.conf('password'),
+            'returnto': '/',
             'lol': secret
-        })
+        }
 
     def loginSuccess(self, output):
         return '/logout.php' in output.lower()

couchpotato/core/providers/torrent/ilovetorrents/main.py

@@ -42,7 +42,7 @@ class ILoveTorrents(TorrentProvider):
             search_url = self.urls['search'] % (movieTitle, page, cats[0])
             page += 1
 
-            data = self.getHTMLData(search_url, opener = self.login_opener)
+            data = self.getHTMLData(search_url)
             if data:
                 try:
                     soup = BeautifulSoup(data)
@@ -96,11 +96,11 @@ class ILoveTorrents(TorrentProvider):
                     log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
 
     def getLoginParams(self):
-        return tryUrlencode({
+        return {
             'username': self.conf('username'),
             'password': self.conf('password'),
             'submit': 'Welcome to ILT',
-        })
+        }
 
     def getMoreInfo(self, item):
         cache_key = 'ilt.%s' % item['id']
@@ -109,7 +109,7 @@ class ILoveTorrents(TorrentProvider):
         if not description:
 
             try:
-                full_description = self.getHTMLData(item['detail_url'], opener = self.login_opener)
+                full_description = self.getHTMLData(item['detail_url'])
                 html = BeautifulSoup(full_description)
                 nfo_pre = html.find('td', attrs = {'class':'main'}).findAll('table')[1]
                 description = toUnicode(nfo_pre.text) if nfo_pre else ''

couchpotato/core/providers/torrent/iptorrents/main.py

@@ -37,7 +37,7 @@ class IPTorrents(TorrentProvider):
         while current_page <= pages and not self.shuttingDown():
 
             url = self.urls['search'] % (self.getCatId(quality['identifier'])[0], freeleech, tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), current_page)
-            data = self.getHTMLData(url, opener = self.login_opener)
+            data = self.getHTMLData(url)
 
             if data:
                 html = BeautifulSoup(data)
@@ -90,11 +90,11 @@ class IPTorrents(TorrentProvider):
             current_page += 1
 
     def getLoginParams(self):
-        return tryUrlencode({
+        return {
             'username': self.conf('username'),
             'password': self.conf('password'),
             'login': 'submit',
-        })
+        }
 
     def loginSuccess(self, output):
         return 'don\'t have an account' not in output.lower()

couchpotato/core/providers/torrent/passthepopcorn/main.py

@@ -65,7 +65,7 @@ class PassThePopcorn(TorrentProvider):
         })
 
         url = '%s?json=noredirect&%s' % (self.urls['torrent'], tryUrlencode(params))
-        res = self.getJsonData(url, opener = self.login_opener)
+        res = self.getJsonData(url)
 
         try:
             if not 'Movies' in res:
@@ -188,13 +188,13 @@ class PassThePopcorn(TorrentProvider):
         return self.unicodeToASCII(self.htmlToUnicode(text))
 
     def getLoginParams(self):
-        return tryUrlencode({
+        return {
              'username': self.conf('username'),
              'password': self.conf('password'),
              'passkey': self.conf('passkey'),
              'keeplogged': '1',
              'login': 'Login'
-        })
+        }
 
     def loginSuccess(self, output):
         try:

couchpotato/core/providers/torrent/sceneaccess/main.py

@@ -45,7 +45,7 @@ class SceneAccess(TorrentProvider):
         url = "%s&%s" % (url, arguments)
 
 
-        data = self.getHTMLData(url, opener = self.login_opener)
+        data = self.getHTMLData(url)
 
         if data:
             html = BeautifulSoup(data)
@@ -78,11 +78,11 @@ class SceneAccess(TorrentProvider):
                 log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
 
     def getLoginParams(self):
-        return tryUrlencode({
+        return {
             'username': self.conf('username'),
             'password': self.conf('password'),
             'submit': 'come on in',
-        })
+        }
 
     def getMoreInfo(self, item):
         full_description = self.getCache('sceneaccess.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)

couchpotato/core/providers/torrent/torrentbytes/main.py

@@ -35,7 +35,7 @@ class TorrentBytes(TorrentProvider):
     def _searchOnTitle(self, title, movie, quality, results):
 
         url = self.urls['search'] % (tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), self.getCatId(quality['identifier'])[0])
-        data = self.getHTMLData(url, opener = self.login_opener)
+        data = self.getHTMLData(url)
 
         if data:
             html = BeautifulSoup(data)
@@ -69,11 +69,11 @@ class TorrentBytes(TorrentProvider):
                 log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
 
     def getLoginParams(self):
-        return tryUrlencode({
+        return {
             'username': self.conf('username'),
             'password': self.conf('password'),
             'login': 'submit',
-        })
+        }
 
     def loginSuccess(self, output):
         return 'logout.php' in output.lower() or 'Welcome' in output.lower()

couchpotato/core/providers/torrent/torrentday/main.py

@@ -1,4 +1,3 @@
-from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.torrent.base import TorrentProvider
@@ -30,7 +29,7 @@ class TorrentDay(TorrentProvider):
 
         q = '"%s %s"' % (title, movie['library']['year'])
 
-        params = {
+        data = {
             '/browse.php?': None,
             'cata': 'yes',
             'jxt': 8,
@@ -38,7 +37,7 @@ class TorrentDay(TorrentProvider):
             'search': q,
         }
 
-        data = self.getJsonData(self.urls['search'], params = params, opener = self.login_opener)
+        data = self.getJsonData(self.urls['search'], data = data)
         try: torrents = data.get('Fs', [])[0].get('Cn', {}).get('torrents', [])
         except: return
 
@@ -54,11 +53,13 @@ class TorrentDay(TorrentProvider):
             })
 
     def getLoginParams(self):
-        return tryUrlencode({
+        return {
             'username': self.conf('username'),
             'password': self.conf('password'),
+            'submit.x': 18,
+            'submit.y': 11,
             'submit': 'submit',
-        })
+        }
 
     def loginSuccess(self, output):
         return 'Password not correct' not in output

couchpotato/core/providers/torrent/torrentleech/main.py

@@ -36,7 +36,7 @@ class TorrentLeech(TorrentProvider):
     def _searchOnTitle(self, title, movie, quality, results):
 
         url = self.urls['search'] % (tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), self.getCatId(quality['identifier'])[0])
-        data = self.getHTMLData(url, opener = self.login_opener)
+        data = self.getHTMLData(url)
 
         if data:
             html = BeautifulSoup(data)
@@ -68,12 +68,12 @@ class TorrentLeech(TorrentProvider):
                 log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
 
     def getLoginParams(self):
-        return tryUrlencode({
+        return {
             'username': self.conf('username'),
             'password': self.conf('password'),
             'remember_me': 'on',
             'login': 'submit',
-        })
+        }
 
     def loginSuccess(self, output):
         return '/user/account/logout' in output.lower() or 'welcome back' in output.lower()

couchpotato/core/providers/torrent/torrentshack/main.py

@@ -34,7 +34,7 @@ class TorrentShack(TorrentProvider):
         scene_only = '1' if self.conf('scene_only') else ''
 
         url = self.urls['search'] % (tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), scene_only, self.getCatId(quality['identifier'])[0])
-        data = self.getHTMLData(url, opener = self.login_opener)
+        data = self.getHTMLData(url)
 
         if data:
             html = BeautifulSoup(data)
@@ -65,12 +65,12 @@ class TorrentShack(TorrentProvider):
                 log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
 
     def getLoginParams(self):
-        return tryUrlencode({
+        return {
             'username': self.conf('username'),
             'password': self.conf('password'),
             'keeplogged': '1',
             'login': 'Login',
-        })
+        }
 
     def loginSuccess(self, output):
         return 'logout.php' in output.lower()