Torrent provider cleanup

couchpotato/core/plugins/base.py

@@ -95,7 +95,7 @@ class Plugin(object):
         return False
 
     # http request
-    def urlopen(self, url, timeout = 30, params = {}, headers = {}, multipart = False, show_error = True):
+    def urlopen(self, url, timeout = 30, params = {}, headers = {}, opener = None, multipart = False, show_error = True):
 
         # Fill in some headers
         if not headers.get('Referer'):
@@ -130,6 +130,9 @@ class Plugin(object):
                 data = tryUrlencode(params) if len(params) > 0 else None
                 request = urllib2.Request(url, data, headers)
 
+                if opener:
+                    data = opener.open(request, timeout = timeout).read()
+                else:
                     data = urllib2.urlopen(request, timeout = timeout).read()
 
             self.http_failed_request[host] = 0
@@ -215,17 +218,6 @@ class Plugin(object):
                     cache_timeout = kwargs.get('cache_timeout')
                     del kwargs['cache_timeout']
 
-                opener = None
-                if kwargs.get('opener'):
-                    opener = kwargs.get('opener')
-                    del kwargs['opener']
-                
-                if opener:
-                    log.info('Opening url: %s', url)
-                    f = opener.open(url)
-                    data = f.read()
-                    f.close()
-                else:
                 data = self.urlopen(url, **kwargs)
 
                 if data:

couchpotato/core/providers/nzb/nzbindex/main.py

@@ -10,6 +10,7 @@ from couchpotato.environment import Env
 from dateutil.parser import parse
 import re
 import time
+import traceback
 import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
@@ -100,8 +101,8 @@ class NzbIndex(NZBProvider, RSS):
                         self.found(new)
 
                 return results
-            except SyntaxError:
+            except:
-                log.error('Failed to parse XML response from NZBMatrix.com')
+                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
 
         return results
 

couchpotato/core/providers/torrent/base.py

@@ -1,31 +1,55 @@
-from couchpotato.core.providers.base import YarrProvider
+from couchpotato.core.helpers.variable import getImdb
 from couchpotato.core.logger import CPLog
-import urllib2
+from couchpotato.core.providers.base import YarrProvider
 import cookielib
+import traceback
+import urllib2
 
 log = CPLog(__name__)
 
 
 class TorrentProvider(YarrProvider):
-    type = 'torrent'
 
-    def login(self, params):
+    type = 'torrent'
+    login_opener = None
+
+    def imdbMatch(self, url, imdbId):
+        if getImdb(url) == imdbId:
+            return True
+
+        if url[:4] == 'http':
+            try:
+                data = self.urlopen(url)
+            except IOError:
+                log.error('Failed to open %s.', url)
+                return False
+
+            return getImdb(data) == imdbId
+
+        return False
+
+    def login(self):
 
         try:
             cookiejar = cookielib.CookieJar()
             opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
             urllib2.install_opener(opener)
-            f = opener.open(self.urls['login'], params)
+            f = opener.open(self.urls['login'], self.getLoginParam())
-            loginData = f.read()
+            f.read()
             f.close()
-        
+            self.login_opener = opener
+            return True
         except:
-            log.error('Failed to login.')
+            log.error('Failed to login %s: %s', (self.getName(), traceback.format_exc()))
 
-        return opener  
+        return False
 
     def download(self, url = '', nzb_id = ''):
-        loginParams = self.getLoginParams()
+
-        self.login(params = loginParams)
+        try:
-        torrent = self.urlopen(url)
+            if not self.login_opener and not self.login():
-        return torrent
+                log.error('Failed downloading from %s', self.getName())
+
+            return self.urlopen(url, opener = self.login_opener)
+        except:
+            log.error('Failed downloading from %s: %s', (self.getName(), traceback.format_exc()))

couchpotato/core/providers/torrent/sceneaccess/main.py

@@ -1,17 +1,12 @@
 from bs4 import BeautifulSoup
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.variable import tryInt, getTitle
+from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode
+from couchpotato.core.helpers.variable import getTitle, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.torrent.base import TorrentProvider
-import StringIO
+from urllib import quote_plus
-import gzip
-import re
 import traceback
 import urllib
-import urllib2
-import cookielib
-from urllib import quote_plus
-from urllib2 import URLError
 
 log = CPLog(__name__)
 
@@ -22,7 +17,7 @@ class SceneAccess(TorrentProvider):
         'test': 'https://www.sceneaccess.eu/',
         'login' : 'https://www.sceneaccess.eu/login',
         'detail': 'https://www.sceneaccess.eu/details?id=%s',
-        'search': 'https://www.sceneaccess.eu/browse?search=%s&method=2&c%d=%d',
+        'search': 'https://www.sceneaccess.eu/browse?method=2&c%d=%d',
         'download': 'https://www.sceneaccess.eu/%s',
     }
 
@@ -34,88 +29,74 @@ class SceneAccess(TorrentProvider):
 
     http_time_between_calls = 1 #seconds
 
-    def getLoginParams(self):
-        loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), submit='come on in'))
-        return loginParams
-
     def search(self, movie, quality):
 
         results = []
         if self.isDisabled():
             return results
 
-        cache_key = 'sceneaccess.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
+        q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
-        searchUrl =  self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0], self.getCatId(quality['identifier'])[0])
+        arguments = tryUrlencode({
-        loginParams = self.getLoginParams()
+            'search': q,
+        })
+        url = "%s&%s" % (self.urls['search'], arguments)
+        url = url % (
+           self.getCatId(quality['identifier'])[0],
+           self.getCatId(quality['identifier'])[0]
+        )
 
-        opener = self.login(params = loginParams)
+        # Do login for the cookies
-        if not opener:
+        if not self.login_opener and not self.login():
-            log.info("Couldn't login at SceneAccess")
             return results
 
-        data = self.getCache(cache_key, searchUrl, opener = opener)
+        cache_key = 'sceneaccess.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
+        data = self.getCache(cache_key, url, opener = self.login_opener)
 
         if data:
             html = BeautifulSoup(data)
 
-        else:
-            log.info("No results found at SceneAccess")
-
             try:
                 resultsTable = html.find('table', attrs = {'id' : 'torrents-table'})
                 entries = resultsTable.findAll('tr', attrs = {'class' : 'tt_row'})
                 for result in entries:
+
+                    link = result.find('td', attrs = {'class' : 'ttr_name'}).find('a')
+                    url = result.find('td', attrs = {'class' : 'td_dl'}).find('a')
+                    leechers = result.find('td', attrs = {'class' : 'ttr_leechers'}).find('a')
+
                     new = {
+                        'id': link['href'].replace('details?id=', ''),
                         'type': 'torrent',
                         'check_nzb': False,
                         'description': '',
                         'provider': self.getName(),
+                        'name': link['title'],
+                        'url': self.urls['download'] % url['href'],
+                        'size': self.parseSize(result.find('td', attrs = {'class' : 'ttr_size'}).contents[0]),
+                        'seeders': tryInt(result.find('td', attrs = {'class' : 'ttr_seeders'}).find('a').string),
+                        'leechers': tryInt(leechers.string) if leechers else 0,
+                        'download': self.download,
                     }
 
-                link = result.find('td', attrs = {'class' : 'ttr_name'}).find('a')
+                    imdb_results = self.imdbMatch(self.urls['detail'] % new['id'], movie['library']['identifier'])
-                new['name'] = link['title']
-                new['id'] = link['href'].replace('details?id=', '')
-                url = result.find('td', attrs = {'class' : 'td_dl'}).find('a')
-                new['url'] = self.urls['download'] % url['href']
-                new['size'] = self.parseSize(result.find('td', attrs = {'class' : 'ttr_size'}).contents[0])
-                new['seeders'] = int(result.find('td', attrs = {'class' : 'ttr_seeders'}).find('a').string)
-                leechers = result.find('td', attrs = {'class' : 'ttr_leechers'}).find('a')
-                if leechers:
-                    new['leechers'] = int(leechers.string)
-                else:
-                    new['leechers'] = 0
-            
-                details = self.urls['detail'] % new['id']
-                imdb_results = self.imdbMatch(details, movie['library']['identifier'])
 
                     new['score'] = fireEvent('score.calculate', new, movie, single = True)
                     is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
                                                      imdb_results = imdb_results, single_category = False, single = True)
 
                     if is_correct_movie:
-                    new['download'] = self.download
                         results.append(new)
                         self.found(new)
-            return results
 
+                return results
             except:
-            log.info("No results found at SceneAccess")
+                log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
+
         return []
 
-
+    def getLoginParams(self, params):
-    def imdbMatch(self, url, imdbId):
+        return tryUrlencode({
-        try:
+            'username': self.conf('username'),
-            data = urllib2.urlopen(url).read()
+            'password': self.conf('password'),
-            pass
+            'submit': 'come on in',
-        except IOError:
+        })
-            log.error('Failed to open %s.' % url)
-            return False
-
-        html = BeautifulSoup(data)
-        imdbDiv = html.find('span', attrs = {'class':'i_link'})
-        imdbDiv = str(imdbDiv).decode("utf-8", "replace")
-        imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId)
-
-        if 'imdb.com/title/' + imdbId in imdbDiv or 'imdb.com/title/' + imdbIdAlt in imdbDiv:
-            return True
-        return False

couchpotato/core/providers/torrent/scenehd/main.py

@@ -1,18 +1,11 @@
 from bs4 import BeautifulSoup
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.variable import tryInt, getTitle
+from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode
+from couchpotato.core.helpers.variable import getTitle, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.torrent.base import TorrentProvider
-import StringIO
-import gzip
-import re
 import traceback
 import urllib
-import urllib2
-import cookielib
-from urllib import quote_plus
-from urllib2 import URLError
-
 
 log = CPLog(__name__)
 
@@ -23,89 +16,87 @@ class SceneHD(TorrentProvider):
         'test': 'http://scenehd.org/',
         'login' : 'http://scenehd.org/takelogin.php',
         'detail': 'http://scenehd.org/details.php?id=%s',
-        'search': 'http://scenehd.org/browse.php?ajax&search=%s',
+        'search': 'http://scenehd.org/browse.php?ajax',
         'download': 'http://scenehd.org/download.php?id=%s',
     }
 
     http_time_between_calls = 1 #seconds
 
-    def getLoginParams(self):
-        loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), ssl='yes'))
-        return loginParams
-
     def search(self, movie, quality):
 
         results = []
         if self.isDisabled():
             return results
 
-        cache_key = 'scenehd.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
+        q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
-        searchUrl =  self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']))
+        arguments = tryUrlencode({
-        loginParams = self.getLoginParams()
+            'search': q,
+        })
+        url = "%s&%s" % (self.urls['search'], arguments)
 
-        opener = self.login(params = loginParams)
+        # Cookie login
-        if not opener:
+        if not self.login_opener and not self.login():
-            log.error("Couldn't login at SceneHD")
             return results
 
-        data = self.getCache(cache_key, searchUrl, opener = opener)
+        cache_key = 'scenehd.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
+        data = self.getCache(cache_key, url, opener = self.login_opener)
 
         if data:
             html = BeautifulSoup(data)
 
-        else:
-            log.info("No results found at SceneHD")
-        
             try:
-            resultsTable = html.findAll('table')[6]
+                resultsTable = html.find_all('table')[6]
-            entries = resultsTable.findAll('tr')
+                entries = resultsTable.find_all('tr')
                 for result in entries[1:]:
+
+                    all_cells = result.find_all('td')
+
+                    detail_link = all_cells[2].find('a')
+                    details = detail_link['href']
+                    id = details.replace('details.php?id=', '')
+
+                    leechers = all_cells[11].find('a')
+                    if leechers:
+                        leechers = leechers.string
+                    else:
+                        leechers = all_cells[11].string
+
                     new = {
+                        'id': id,
+                        'name': detail_link['title'],
                         'type': 'torrent',
                         'check_nzb': False,
                         'description': '',
                         'provider': self.getName(),
+                        'size': self.parseSize(all_cells[7].string),
+                        'seeders': tryInt(all_cells[10].find('a').string),
+                        'leechers': tryInt(leechers),
+                        'url': self.urls['download'] % id,
+                        'download': self.download,
                     }
 
-                allCells = result.findAll('td')
+                    imdb_link = all_cells[1].find('a')
-                new['size'] = self.parseSize(allCells[7].string.replace('GiB', 'GB'))
+                    imdb_results = self.imdbMatch(imdb_link['href'], movie['library']['identifier']) if imdb_link else False
-                new['seeders'] = allCells[10].find('a').string
-                leechers = allCells[11].find('a')
-                if leechers:
-                    new['leechers'] = leechers.string
-                else:
-                    new['leechers'] = allCells[11].string
 
-                detailLink = allCells[2].find('a')
-                details = detailLink['href']
-                new['id'] = details.replace('details.php?id=', '')
-                new['name'] = detailLink['title']
-            
-                imdbLink = allCells[1].find('a')
-                imdb_results = False
-
-                if imdbLink:
-                    imdbFound = imdbLink['href'].replace('http://www.imdb.com/title/','').rstrip('/')
-                    imdb_results = self.imdbMatch(imdbFound, movie['library']['identifier'])
-               
-                new['url'] = self.urls['download'] % new['id']
                     new['score'] = fireEvent('score.calculate', new, movie, single = True)
                     is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
                                                      imdb_results = imdb_results, single_category = False, single = True)
 
                     if is_correct_movie:
-                    new['download'] = self.download
                         results.append(new)
                         self.found(new)
+
                 return results
 
             except:
-            log.info("No results found at SceneHD")
+                log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
+
         return []
 
-    def imdbMatch(self, imdbFound, imdbId):
-        imdbIdAlt = re.sub('tt[0]*', 'tt', imdbFound)
-        if imdbFound == imdbId or imdbIdAlt == imdbId:
-            return True
-        return False
 
+    def getLoginParams(self, params):
+        return tryUrlencode({
+            'username': self.conf('username'),
+            'password': self.conf('password'),
+            'ssl': 'yes',
+        })

couchpotato/core/providers/torrent/torrentleech/main.py

@@ -1,18 +1,11 @@
 from bs4 import BeautifulSoup
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.variable import tryInt, getTitle
+from couchpotato.core.helpers.encoding import tryUrlencode
+from couchpotato.core.helpers.variable import getTitle, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.torrent.base import TorrentProvider
-import StringIO
-import gzip
-import re
-import traceback
-import urllib
-import urllib2
-import cookielib
 from urllib import quote_plus
-from urllib2 import URLError
+import traceback
-import sys
 
 
 log = CPLog(__name__)
@@ -40,81 +33,66 @@ class TorrentLeech(TorrentProvider):
 
     http_time_between_calls = 1 #seconds
 
-    def getLoginParams(self):
-        loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), remember_me='on', login='submit'))
-        return loginParams
-
     def search(self, movie, quality):
 
         results = []
         if self.isDisabled():
             return results
 
-        cache_key = 'torrentleech.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
+        # Cookie login
-        searchUrl =  self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
+        if not self.login_opener and not self.login():
-        loginParams = self.getLoginParams()
-      
-        opener = self.login(params = loginParams)
-        if not opener:
-            log.info("Couldn't login at Torrentleech")
             return results
 
-        data = self.getCache(cache_key, searchUrl, opener = opener)
+        cache_key = 'torrentleech.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
+        url = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':', '') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
+        data = self.getCache(cache_key, url, opener = self.login_opener)
 
         if data:
             html = BeautifulSoup(data)
 
-        else:
-            log.info("No results found at Torrentleech")
-
             try:
-            resultsTable = html.find('table', attrs = {'id' : 'torrenttable'})            
+                result_table = html.find('table', attrs = {'id' : 'torrenttable'})
-            entries = resultsTable.findAll('tr')
+                entries = result_table.find_all('tr')
+
                 for result in entries[1:]:
+
+                    link = result.find('td', attrs = {'class' : 'name'}).find('a')
+                    url = result.find('td', attrs = {'class' : 'quickdownload'}).find('a')
+
                     new = {
+                        'id': link['href'].replace('/torrent/', ''),
+                        'name': link.string,
                         'type': 'torrent',
                         'check_nzb': False,
                         'description': '',
                         'provider': self.getName(),
+                        'url': self.urls['download'] % url['href'],
+                        'download': self.download,
+                        'size': self.parseSize(result.find_all('td')[4].string),
+                        'seeders': tryInt(result.find('td', attrs = {'class' : 'seeders'}).string),
+                        'leechers': tryInt(result.find('td', attrs = {'class' : 'leechers'}).string),
                     }
 
-                link = result.find('td', attrs = {'class' : 'name'}).find('a')
+                    imdb_results = self.imdbMatch(self.urls['detail'] % new['id'], movie['library']['identifier'])
-                new['name'] = link.string
-                new['id'] = link['href'].replace('/torrent/', '')
-                url = result.find('td', attrs = {'class' : 'quickdownload'}).find('a')
-                new['url'] = self.urls['download'] % url['href']
-                new['size'] = self.parseSize(result.findAll('td')[4].string)
-                new['seeders'] = int(result.find('td', attrs = {'class' : 'seeders'}).string)
-                new['leechers'] = int(result.find('td', attrs = {'class' : 'leechers'}).string)   
-                
-                details = self.urls['detail'] % new['id']
-                imdb_results = self.imdbMatch(details, movie['library']['identifier'])
 
                     new['score'] = fireEvent('score.calculate', new, movie, single = True)
                     is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
                                                     imdb_results = imdb_results, single_category = False, single = True)
 
                     if is_correct_movie:
-                    new['download'] = self.download
                         results.append(new)
                         self.found(new)
-            return results
 
+                return results
             except:
-            log.info("No results found at TorrentLeech")
+                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
+
         return []
 
-
+    def getLoginParams(self):
-    def imdbMatch(self, url, imdbId):
+        return tryUrlencode({
-        try:
+            'username': self.conf('username'),
-            data = urllib2.urlopen(url).read()
+            'password': self.conf('password'),
-            pass
+            'remember_me': 'on',
-        except IOError:
+            'login': 'submit',
-            log.error('Failed to open %s.' % url)
+        })
-            return False
-
-        imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId)
-        data = unicode(data, errors='ignore')
-        if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' + imdbIdAlt in data:
-            return True
-        return False