Torrent provider cleanup

couchpotato/core/downloaders/transmission/__init__.py

@@ -34,6 +34,7 @@ config = [{
                 {
                     'name': 'paused',
                     'type': 'bool',
+                    'default': False,
                     'description': 'Add the torrent paused.',
                 },
                 {

couchpotato/core/downloaders/transmission/main.py

@@ -1,98 +1,124 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
 from base64 import b64encode
 from couchpotato.core.downloaders.base import Downloader
 from couchpotato.core.helpers.encoding import isInt
 from couchpotato.core.logger import CPLog
-
-import urllib2
 import httplib
 import json
 import re
+import urllib2
+
+log = CPLog(__name__)
+
+
+class Transmission(Downloader):
+
+    type = ['torrent', 'torrent_magnet']
+    log = CPLog(__name__)
+
+    def download(self, data, movie, manual = False, filedata = None):
+
+        if self.isDisabled(manual) or not self.isCorrectType(data.get('type')):
+            return
+
+        log.debug('Sending "%s" (%s) to Transmission.', (data.get('name'), data.get('type')))
+
+        # Load host from config and split out port.
+        host = self.conf('host').split(':')
+        if not isInt(host[1]):
+            log.error('Config properties are not filled in correctly, port is missing.')
+            return False
+
+        # Set parameters for Transmission
+        params = {
+            'paused': self.conf('paused', default = 0),
+            'download-dir': self.conf('directory', default = None)
+        }
+
+        torrent_params = {
+            'seedRatioLimit': self.conf('ratio'),
+            'seedRatioMode': (0 if self.conf('ratio') else 1)
+        }
+
+        if not filedata and data.get('type') == 'torrent':
+            log.error('Failed sending torrent, no data')
+            return False
+
+        # Send request to Transmission
+        try:
+            trpc = TransmissionRPC(host[0], port = host[1], username = self.conf('username'), password = self.conf('password'))
+            if data.get('type') == 'torrent_magnet':
+                remote_torrent = trpc.add_torrent_uri(data.get('url'), arguments = params)
+            else:
+                remote_torrent = trpc.add_torrent_file(b64encode(filedata), arguments = params)
+
+            # Change settings of added torrents
+            trpc.set_torrent(remote_torrent['torrent-added']['hashString'], torrent_params)
+
+            return True
+        except Exception, err:
+            log.error('Failed to change settings for transfer: %s', err)
+            return False
 
 
 class TransmissionRPC(object):
 
     """TransmissionRPC lite library"""
 
-    log = CPLog(__name__)
+    def __init__(self, host = 'localhost', port = 9091, username = None, password = None):
-
-    def __init__(
-        self,
-        host='localhost',
-        port=9091,
-        username=None,
-        password=None,
-        ):
 
         super(TransmissionRPC, self).__init__()
-        self.url = 'http://' + host + ':' + str(port) \
+
-            + '/transmission/rpc'
+        self.url = 'http://' + host + ':' + str(port) + '/transmission/rpc'
         self.tag = 0
         self.session_id = 0
         self.session = {}
         if username and password:
             password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
-            password_manager.add_password(realm=None, uri=self.url,
+            password_manager.add_password(realm = None, uri = self.url, user = username, passwd = password)
-                    user=username, passwd=password)
+            opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(password_manager), urllib2.HTTPDigestAuthHandler(password_manager))
-            opener = \
+            opener.addheaders = [('User-agent', 'couchpotato-transmission-client/1.0')]
-                urllib2.build_opener(urllib2.HTTPBasicAuthHandler(password_manager),
-                    urllib2.HTTPDigestAuthHandler(password_manager))
-            opener.addheaders = [('User-agent',
-                                 'couchpotato-transmission-client/1.0')]
             urllib2.install_opener(opener)
         elif username or password:
-            self.log.debug('User or password missing, not using authentication.'
+            log.debug('User or password missing, not using authentication.')
-                      )
         self.session = self.get_session()
 
     def _request(self, ojson):
         self.tag += 1
         headers = {'x-transmission-session-id': str(self.session_id)}
-        request = urllib2.Request(self.url,
+        request = urllib2.Request(self.url, json.dumps(ojson).encode('utf-8'), headers)
-                                  json.dumps(ojson).encode('utf-8'),
-                                  headers)
         try:
             open_request = urllib2.urlopen(request)
             response = json.loads(open_request.read())
-            self.log.debug('response: '
+            log.debug('response: %s', json.dumps(response))
-                      + str(json.dumps(response).encode('utf-8')))
             if response['result'] == 'success':
-                self.log.debug(u'Transmission action successfull')
+                log.debug('Transmission action successfull')
                 return response['arguments']
             else:
-                self.log.debug('Unknown failure sending command to Transmission. Return text is: '
+                log.debug('Unknown failure sending command to Transmission. Return text is: %s', response['result'])
-                           + response['result'])
                 return False
         except httplib.InvalidURL, err:
-            self.log.error(u'Invalid Transmission host, check your config %s'
+            log.error('Invalid Transmission host, check your config %s', err)
-                       % err)
             return False
         except urllib2.HTTPError, err:
             if err.code == 401:
-                self.log.error(u'Invalid Transmission Username or Password, check your config'
+                log.error('Invalid Transmission Username or Password, check your config')
-                          )
                 return False
             elif err.code == 409:
                 msg = str(err.read())
                 try:
                     self.session_id = \
-                        re.search('X-Transmission-Session-Id:\s*(\w+)',
+                        re.search('X-Transmission-Session-Id:\s*(\w+)', msg).group(1)
-                                  msg).group(1)
+                    log.debug('X-Transmission-Session-Id: %s', self.session_id)
-                    self.log.debug('X-Transmission-Session-Id: '
-                              + self.session_id)
 
                     # #resend request with the updated header
 
                     return self._request(ojson)
                 except:
-                    self.log.error(u'Unable to get Transmission Session-Id %s'
+                    log.error('Unable to get Transmission Session-Id %s', err)
-                               % err)
             else:
-                self.log.error(u'TransmissionRPC HTTPError: %s' % err)
+                log.error('TransmissionRPC HTTPError: %s', err)
         except urllib2.URLError, err:
-            self.log.error(u'Unable to connect to Transmission %s' % err)
+            log.error('Unable to connect to Transmission %s', err)
 
     def get_session(self):
         post_data = {'method': 'session-get', 'tag': self.tag}
@@ -100,84 +126,15 @@ class TransmissionRPC(object):
 
     def add_torrent_uri(self, torrent, arguments):
         arguments['filename'] = torrent
-        post_data = {'arguments': arguments, 'method': 'torrent-add',
+        post_data = {'arguments': arguments, 'method': 'torrent-add', 'tag': self.tag}
-                     'tag': self.tag}
         return self._request(post_data)
 
     def add_torrent_file(self, torrent, arguments):
         arguments['metainfo'] = torrent
-        post_data = {'arguments': arguments, 'method': 'torrent-add',
+        post_data = {'arguments': arguments, 'method': 'torrent-add', 'tag': self.tag}
-                     'tag': self.tag}
         return self._request(post_data)
 
     def set_torrent(self, torrent_id, arguments):
         arguments['ids'] = torrent_id
-        post_data = {'arguments': arguments, 'method': 'torrent-set',
+        post_data = {'arguments': arguments, 'method': 'torrent-set', 'tag': self.tag}
-                     'tag': self.tag}
         return self._request(post_data)
-
-
-class Transmission(Downloader):
-
-    type = ['torrent', 'magnet']
-    log = CPLog(__name__)
-
-    def download(
-        self,
-        data,
-        movie,
-        manual=False,
-        filedata=None,
-        ):
-
-        print data
-        if self.isDisabled(manual) \
-            or not self.isCorrectType(data.get('type')):
-            return
-
-        self.log.debug('Sending "%s" to Transmission.', data.get('name'))
-        self.log.debug('Type "%s" to Transmission.', data.get('type'))
-
-        # Load host from config and split out port.
-
-        host = self.conf('host').split(':')
-        if not isInt(host[1]):
-            self.log.error('Config properties are not filled in correctly, port is missing.'
-                      )
-            return False
-
-        # Set parameters for Transmission
-
-        params = {'paused': self.conf('paused', default=0),
-                  'download-dir': self.conf('directory', default=None)}
-
-        torrent_params = {'seedRatioLimit': self.conf('ratio'),
-                          'seedRatioMode': (0 if self.conf('ratio'
-                          ) else 1)}
-
-        if not filedata or data.get('type') != 'magnet':
-            self.log.error('Failed sending torrent, no data')
-
-        # Send request to Transmission
-
-        try:
-            trpc = TransmissionRPC(host[0], port=host[1],
-                                 username=self.conf('username'),
-                                 password=self.conf('password'))
-            if data.get('type') == 'magnet' or data.get('magnet') \
-                != None:
-                remote_torrent = trpc.add_torrent_uri(data.get('magnet'),
-                        arguments=params)
-            else:
-                remote_torrent = \
-                    trpc.add_torrent_file(b64encode(filedata),
-                        arguments=params)
-
-            # Change settings of added torrents
-
-            trpc.set_torrent(remote_torrent['torrent-added']['hashString'
-                           ], torrent_params)
-            return True
-        except Exception, err:
-            self.log.error('Failed to change settings for transfer: %s', err)
-            return False

couchpotato/core/plugins/release/main.py

@@ -143,7 +143,7 @@ class Release(Plugin):
                 item[info.identifier] = info.value
 
             # Get matching provider
-            provider = fireEvent('provider.belongs_to', item['url'], single = True)
+            provider = fireEvent('provider.belongs_to', item['url'], provider = item.get('provider'), single = True)
             item['download'] = provider.download
 
             fireEvent('searcher.download', data = item, movie = rel.movie.to_dict({

couchpotato/core/plugins/score/scores.py

@@ -1,7 +1,6 @@
 from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import simplifyString
 from couchpotato.core.helpers.variable import tryInt
-from couchpotato.core.plugins.scanner.main import Scanner
 from couchpotato.environment import Env
 import re
 

couchpotato/core/plugins/searcher/main.py

@@ -29,6 +29,9 @@ class Searcher(Plugin):
         # Schedule cronjob
         fireEvent('schedule.cron', 'searcher.all', self.all_movies, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute'))
 
+
+        addEvent('app.load', self.all_movies)
+
     def all_movies(self):
 
         if self.in_progress:
@@ -193,7 +196,7 @@ class Searcher(Plugin):
             if filedata is 'try_next':
                 return filedata
 
-        successful = fireEvent('download', data = data, movie = movie, manual = manual, single = True, filedata = filedata)
+        successful = fireEvent('download', data = data, movie = movie, manual = manual, filedata = filedata, single = True)
 
         if successful:
 

couchpotato/core/providers/base.py

@@ -65,8 +65,11 @@ class YarrProvider(Provider):
     def search(self, movie, quality):
         return []
 
-    def belongsTo(self, url, host = None):
+    def belongsTo(self, url, provider = None, host = None):
         try:
+            if provider and provider == self.getName():
+                return self
+
             hostname = urlparse(url).hostname
             if host and hostname in host:
                 return self
@@ -108,24 +111,5 @@ class YarrProvider(Provider):
 
         return [self.cat_backup_id]
 
-    def imdb_match(self, url, imdb_id):
-        """ Searches for imdb_id in url of webpage """
-        log.info('Finding if imbd_id(%s) is found in url: %s' % (imdb_id, url))
-        try:
-            data = self.urlopen(url)
-        except:
-            log.error('Failed to open %s.' % url)
-            return False
-        imdb_id_alt = re.sub('tt[0]*', 'tt', imdb_id)
-        data = unicode(data, errors='ignore')
-        if 'imdb.com/title/' + imdb_id in data or 'imdb.com/title/' \
-            + imdb_id_alt in data:
-            return True
-        return False
-
-    def for_search(self, string):
-        """ Prepare string for search, removing all characters that might confuse search engine"""
-        return quote_plus(simplifyString(string))
-
     def found(self, new):
         log.info('Found: score(%(score)s) on %(provider)s: %(name)s', new)

couchpotato/core/providers/torrent/base.py

@@ -1,4 +1,4 @@
-from couchpotato.core.helpers.variable import getImdb
+from couchpotato.core.helpers.variable import getImdb, md5
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import YarrProvider
 import cookielib
@@ -19,7 +19,8 @@ class TorrentProvider(YarrProvider):
 
         if url[:4] == 'http':
             try:
-                data = self.urlopen(url)
+                cache_key = md5(url)
+                data = self.getCache(cache_key, url)
             except IOError:
                 log.error('Failed to open %s.', url)
                 return False
@@ -34,7 +35,7 @@ class TorrentProvider(YarrProvider):
             cookiejar = cookielib.CookieJar()
             opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
             urllib2.install_opener(opener)
-            f = opener.open(self.urls['login'], self.getLoginParam())
+            f = opener.open(self.urls['login'], self.getLoginParams())
             f.read()
             f.close()
             self.login_opener = opener
@@ -44,12 +45,13 @@ class TorrentProvider(YarrProvider):
 
         return False
 
-    def download(self, url = '', nzb_id = ''):
+    def loginDownload(self, url = '', nzb_id = ''):
-
         try:
             if not self.login_opener and not self.login():
                 log.error('Failed downloading from %s', self.getName())
-
             return self.urlopen(url, opener = self.login_opener)
         except:
             log.error('Failed downloading from %s: %s', (self.getName(), traceback.format_exc()))
+
+    def getLoginParams(self):
+        return ''

couchpotato/core/providers/torrent/publichd/main.py

@@ -1,116 +1,105 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
 from bs4 import BeautifulSoup
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.variable import getTitle
+from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
+from couchpotato.core.helpers.variable import getTitle, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.torrent.base import TorrentProvider
 from urlparse import parse_qs
-
 import re
+import traceback
+
+log = CPLog(__name__)
 
 
 class PublicHD(TorrentProvider):
 
-    log = CPLog(__name__)
     urls = {
         'test': 'http://publichd.eu',
         'download': 'http://publichd.eu/%s',
         'detail': 'http://publichd.eu/index.php?page=torrent-details&id=%s',
-        'search': 'http://publichd.eu/index.php?page=torrents&search=%s&active=1&category=%d',
+        'search': 'http://publichd.eu/index.php',
-        }
+    }
 
-    cat_ids = [([2], ['720p']), ([5], ['1080p']), ([16], ['brrip']),
+    cat_ids = [
-               ([16], ['bd50'])]
+       ([9], ['bd50']),
+       ([5], ['1080p']),
+       ([2], ['720p']),
+       ([15, 16], ['brrip']),
+    ]
 
     cat_backup_id = 0
+    http_time_between_calls = 0
 
     def search(self, movie, quality):
 
         results = []
-        if self.isDisabled() and quality['hd'] != True:
+
+        if self.isDisabled() or quality['hd'] != True:
             return results
 
-        cache_key = 'publichd.%s.%s' % (movie['library']['identifier'],
+        params = tryUrlencode({
-                quality.get('identifier'))
+            'page':'torrents',
-        search_url = self.urls['search'] \
+            'search': getTitle(movie['library']) + ' ' + quality['identifier'],
-            % (self.for_search(getTitle(movie['library'])
+            'active': 1,
-               + ' ' + quality['identifier']),
+            'category': self.getCatId(quality['identifier'])[0]
-               self.getCatId(quality['identifier'])[0])
+        })
-        self.log.info('searchUrl: %s', search_url)
+        url = '%s?%s' % (self.urls['search'], params)
-        data = self.getCache(cache_key, search_url)
-        if not data:
-            self.log.error('Failed to get data from %s.', search_url)
-            return results
 
-        try:
+        cache_key = 'publichd.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
-            soup = BeautifulSoup(data)
+        data = self.getCache(cache_key, url)
 
-            results_table = soup.find('table',
+        if data:
-                    attrs={'id': 'bgtorrlist2'})
-            entries = results_table.find_all('tr')
-            for result in entries[2:len(entries) - 1]:
-                info_url = result.find(href=re.compile('torrent-details'
-                        ))
-                download = result.find(href=re.compile('\.torrent'))
 
-                if info_url and download:
+            try:
-                    new = {
+                soup = BeautifulSoup(data)
-                        'type': 'torrent',
+
-                        'check_nzb': False,
+                results_table = soup.find('table', attrs = {'id': 'bgtorrlist2'})
-                        'description': '',
+                entries = results_table.find_all('tr')
-                        'provider': self.getName(),
+
+                for result in entries[2:len(entries) - 1]:
+                    info_url = result.find(href = re.compile('torrent-details'))
+                    download = result.find(href = re.compile('\.torrent'))
+
+                    if info_url and download:
+
+                        url = parse_qs(info_url['href'])
+
+                        new = {
+                            'id': url['id'][0],
+                            'name': info_url.string,
+                            'type': 'torrent',
+                            'check_nzb': False,
+                            'description': '',
+                            'provider': self.getName(),
+                            'download': self.download,
+                            'url': self.urls['download'] % download['href'],
+                            'detail_url': self.urls['detail'] % url['id'][0],
+                            'size': self.parseSize(result.find_all('td')[7].string),
+                            'seeders': tryInt(result.find_all('td')[4].string),
+                            'leechers': tryInt(result.find_all('td')[5].string),
+                            'get_more_info': self.getMoreInfo
                         }
-                    self.log.debug('Name: %s', result.find_all('td'
-                                   )[1].string)
-                    self.log.debug('Seeders: %s', result.find_all('td'
-                                   )[4].string)
-                    self.log.debug('Leaches: %s', result.find_all('td'
-                                   )[5].string)
-                    self.log.debug('Size: %s', result.find_all('td'
-                                   )[7].string)
 
-                    url = parse_qs(info_url['href'])
+                        new['score'] = fireEvent('score.calculate', new, movie, single = True)
+                        is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
+                                                        imdb_results = False, single_category = False, single = True)
 
-                    new['name'] = info_url.string
+                        if is_correct_movie:
-                    new['id'] = url['id'][0]
+                            results.append(new)
-                    new['url'] = self.urls['download'] % download['href'
+                            self.found(new)
-                            ]
-                    new['size'] = self.parseSize(result.find_all('td'
-                            )[7].string)
-                    new['seeders'] = int(result.find_all('td'
-                            )[4].string)
-                    new['leechers'] = int(result.find_all('td'
-                            )[5].string)
 
-                    new['score'] = fireEvent('score.calculate', new,
+                return results
-                            movie, single=True)
-                    is_imdb = self.imdb_match(self.urls['detail']
-                            % new['id'], movie['library']['identifier'])
-                    is_correct_movie = fireEvent(
-                        'searcher.correct_movie',
-                        nzb=new,
-                        movie=movie,
-                        quality=quality,
-                        imdb_results=is_imdb,
-                        single_category=False,
-                        single=True,
-                        )
 
-                    if is_correct_movie:
+            except:
-                        new['download'] = self.download
+                log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
-                        results.append(new)
-                        self.found(new)
 
-            return results
+        return []
-        except Exception, err:
-            self.log.debug(err)
-            self.log.info('Error occured during parsing! Passing only processed entries'
-                          )
-            return results
 
-    def download(self, url='', nzb_id=''):
+    def getMoreInfo(self, item):
-        self.log.info('Downloading: %s', url)
+        full_description = self.getCache('publichd.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
-        torrent = self.urlopen(url)
+        html = BeautifulSoup(full_description)
-        return torrent
+        nfo_pre = html.find('div', attrs = {'id':'torrmain'})
+        description = toUnicode(nfo_pre.text) if nfo_pre else ''
+
+        item['description'] = description
+        return item

couchpotato/core/providers/torrent/sceneaccess/main.py

@@ -1,12 +1,11 @@
 from bs4 import BeautifulSoup
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode
+from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode, \
+    toUnicode
 from couchpotato.core.helpers.variable import getTitle, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.torrent.base import TorrentProvider
-from urllib import quote_plus
 import traceback
-import urllib
 
 log = CPLog(__name__)
 
@@ -35,15 +34,16 @@ class SceneAccess(TorrentProvider):
         if self.isDisabled():
             return results
 
+        url = self.urls['search'] % (
+           self.getCatId(quality['identifier'])[0],
+           self.getCatId(quality['identifier'])[0]
+        )
+
         q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
         arguments = tryUrlencode({
             'search': q,
         })
-        url = "%s&%s" % (self.urls['search'], arguments)
+        url = "%s&%s" % (url, arguments)
-        url = url % (
-           self.getCatId(quality['identifier'])[0],
-           self.getCatId(quality['identifier'])[0]
-        )
 
         # Do login for the cookies
         if not self.login_opener and not self.login():
@@ -63,26 +63,27 @@ class SceneAccess(TorrentProvider):
                     link = result.find('td', attrs = {'class' : 'ttr_name'}).find('a')
                     url = result.find('td', attrs = {'class' : 'td_dl'}).find('a')
                     leechers = result.find('td', attrs = {'class' : 'ttr_leechers'}).find('a')
+                    id = link['href'].replace('details?id=', '')
 
                     new = {
-                        'id': link['href'].replace('details?id=', ''),
+                        'id': id,
                         'type': 'torrent',
                         'check_nzb': False,
                         'description': '',
                         'provider': self.getName(),
                         'name': link['title'],
                         'url': self.urls['download'] % url['href'],
+                        'detail_url': self.urls['detail'] % id,
                         'size': self.parseSize(result.find('td', attrs = {'class' : 'ttr_size'}).contents[0]),
                         'seeders': tryInt(result.find('td', attrs = {'class' : 'ttr_seeders'}).find('a').string),
                         'leechers': tryInt(leechers.string) if leechers else 0,
-                        'download': self.download,
+                        'download': self.loginDownload,
+                        'get_more_info': self.getMoreInfo,
                     }
 
-                    imdb_results = self.imdbMatch(self.urls['detail'] % new['id'], movie['library']['identifier'])
-
                     new['score'] = fireEvent('score.calculate', new, movie, single = True)
                     is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
-                                                     imdb_results = imdb_results, single_category = False, single = True)
+                                                     imdb_results = False, single_category = False, single = True)
 
                     if is_correct_movie:
                         results.append(new)
@@ -94,9 +95,18 @@ class SceneAccess(TorrentProvider):
 
         return []
 
-    def getLoginParams(self, params):
+    def getLoginParams(self):
         return tryUrlencode({
             'username': self.conf('username'),
             'password': self.conf('password'),
             'submit': 'come on in',
         })
+
+    def getMoreInfo(self, item):
+        full_description = self.getCache('sceneaccess.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
+        html = BeautifulSoup(full_description)
+        nfo_pre = html.find('div', attrs = {'id':'details_table'})
+        description = toUnicode(nfo_pre.text) if nfo_pre else ''
+
+        item['description'] = description
+        return item

couchpotato/core/providers/torrent/scenehd/main.py

@@ -5,7 +5,6 @@ from couchpotato.core.helpers.variable import getTitle, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.torrent.base import TorrentProvider
 import traceback
-import urllib
 
 log = CPLog(__name__)
 
@@ -72,7 +71,7 @@ class SceneHD(TorrentProvider):
                         'seeders': tryInt(all_cells[10].find('a').string),
                         'leechers': tryInt(leechers),
                         'url': self.urls['download'] % id,
-                        'download': self.download,
+                        'download': self.loginDownload,
                     }
 
                     imdb_link = all_cells[1].find('a')

couchpotato/core/providers/torrent/thepiratebay/__init__.py

@@ -1,26 +1,27 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
 from main import ThePirateBay
-from main import TPBProxy
-
 
 def start():
     return ThePirateBay()
 
-
+config = [{
-config = [{'name': 'ThePirateBay', 'groups': [{
+    'name': 'thepiratebay',
-    'tab': 'searcher',
+    'groups': [{
-    'subtab': 'providers',
+        'tab': 'searcher',
-    'name': 'ThePirateBay',
+        'subtab': 'providers',
-    'description': 'The world\'s largest bittorrent tracker.',
+        'name': 'ThePirateBay',
-    'options': [{'name': 'enabled', 'type': 'enabler',
+        'description': 'The world\'s largest bittorrent tracker.',
-                'default': False}, {
+        'options': [
-        'name': 'domain_for_tpb',
+            {
-        'label': 'Proxy server',
+                'name': 'enabled',
-        'default': 'http://thepiratebay.se',
+                'type': 'enabler',
-        'description': 'Default domain for requests',
+                'default': False
-        'type': 'dropdown',
+            },
-        'values': TPBProxy.list,
+            {
-        }],
+                'name': 'domain',
-    }]}]
+                'advanced': True,
+                'label': 'Proxy server',
+                'description': 'Domain for requests, keep empty to let CouchPotato pick.',
+            }
+        ],
+    }]
+}]

couchpotato/core/providers/torrent/thepiratebay/main.py

@@ -1,189 +1,142 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
 from bs4 import BeautifulSoup
 from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.variable import getTitle
+from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.variable import getTitle, tryInt, cleanHost
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.torrent.base import TorrentProvider
-from random import sample as random
+from couchpotato.environment import Env
-from urlparse import urlparse
+from urllib import quote_plus
-
 import re
 import time
+import traceback
 
-
+log = CPLog(__name__)
-class TPBProxy(object):
-
-    """ TPBProxy deals with failed or blocked TPB proxys.
-        It works as round-robin balancer, if user seleced
-        or default domain becomes unavaliable.
-    """
-
-    list = [
-        ('(Sweden) thepiratebay.se', 'http://thepiratebay.se'),
-        ('(Sweden) tpb.ipredator.se (ssl)', 'https://tpb.ipredator.se'),
-        ('(Germany) depiraatbaai.be', 'http://depiraatbaai.be'),
-        ('(UK) piratereverse.info (ssl)', 'https://piratereverse.info'),
-        ('(UK) tpb.pirateparty.org.uk (ssl)', 'https://tpb.pirateparty.org.uk'),
-        ('(Netherlands) argumentomteemigreren.nl', 'http://argumentomteemigreren.nl'),
-        ('(direct) 194.71.107.80', 'http://194.71.107.80'),
-        ('(direct) 194.71.107.81', 'http://194.71.107.81'),
-        ('(direct) 194.71.107.82', 'http://194.71.107.82'),
-        ('(direct) 194.71.107.83', 'http://194.71.107.83'),
-        ]
-
-    @staticmethod
-    def get_proxy(http_failed_disabled=None, current=None):
-
-        # compare lists and user/default value, exclude filter
-
-        unused = [item for item in TPBProxy.list if item not in http_failed_disabled and current
-                  not in item]
-
-        if len(unused) > 0:
-
-            # only return uri
-
-            return random(unused, 1)[0][1]
-        else:
-
-            # this should disable provider for some time
-
-            raise Exception('All ThePirateBay proxies are exhausted')
 
 
 class ThePirateBay(TorrentProvider):
 
-    log = CPLog(__name__)
+    urls = {
-    cat_ids = [([207], ['720p', '1080p']), ([201], [
+         'detail': '%s/torrent/%s',
-        'cam',
+         'search': '%s/search/%s/0/7/%d'
-        'ts',
+    }
-        'dvdrip',
+
-        'tc',
+    cat_ids = [
-        'r5',
+       ([207], ['720p', '1080p']),
-        'scr',
+       ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr', 'brrip']),
-        'brrip',
+       ([202], ['dvdr'])
-        ]), ([202], ['dvdr'])]
+    ]
 
     cat_backup_id = 200
     disable_provider = False
+    http_time_between_calls = 0
+
+    proxy_list = [
+        'https://thepiratebay.se',
+        'https://tpb.ipredator.se',
+        'https://depiraatbaai.be',
+        'https://piratereverse.info',
+        'https://tpb.pirateparty.org.uk',
+        'https://argumentomteemigreren.nl',
+    ]
 
     def __init__(self):
+        self.domain = self.conf('domain')
         super(ThePirateBay, self).__init__()
-        self.urls = {'test': self.api_domain(), 'detail': '%s/torrent/%s',
-                     'search': '%s/search/%s/0/7/%d'}
 
-    def api_domain(self, url=''):
+    def getDomain(self, url = ''):
 
-        # default domain
+        if not self.domain:
+            for proxy in self.proxy_list:
 
-        domain = self.conf('domain_for_tpb', default='http://thepiratebay.se')
+                prop_name = 'tpb_proxy.%s' % proxy
-        self.log.info('Selected domain for this request: %s', domain)
+                last_check = float(Env.prop(prop_name, default = 0))
-        host = urlparse(domain).hostname
+                if last_check > time.time() - 1209600:
-
+                    continue
-        # Clear disabled list for default or user selected host if time expired
-
-        if self.http_failed_disabled.get(host, 0) > 0:
-            if self.http_failed_disabled[host] > time.time() - 900:
-
-                # get new random domain
 
+                data = ''
                 try:
-                    domain = TPBProxy.get_proxy(self.http_failed_disabled, domain)
+                    data = self.urlopen(proxy, timeout = 3)
-                except Exception, err:
+                except:
-                    self.disable_provider = True
+                    log.debug('Failed tpb proxy %s', proxy)
-                    self.log.error(err)
-            else:
 
-                del self.http_failed_request[host]
+                if 'title="Pirate Search"' in data:
-                del self.http_failed_disabled[host]
+                    log.debug('Using proxy: %s', proxy)
+                    self.domain = proxy
+                    break
 
-        return domain + url
+                Env.prop(prop_name, time.time())
+
+        if not self.domain:
+            log.error('No TPB proxies left, please add one in settings, or let us know which one to add on the forum.')
+            return None
+
+        return cleanHost(self.domain).rstrip('/') + url
 
     def search(self, movie, quality):
 
         results = []
-        if self.isDisabled() or self.disable_provider:
+        if self.isDisabled() or not self.getDomain():
             return results
 
-        cache_key = 'thepiratebay.%s.%s' % (movie['library']['identifier'], quality.get('identifier'
+        cache_key = 'thepiratebay.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
-                                            ))
+        search_url = self.urls['search'] % (self.getDomain(), quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
-        search_url = self.urls['search'] % (self.api_domain(),
-                                            self.for_search(getTitle(movie['library']) + ' '
-                                            + quality['identifier']),
-                                            self.getCatId(quality['identifier'])[0])
-        self.log.info('searchUrl: %s', search_url)
         data = self.getCache(cache_key, search_url)
 
-        # print data
+        if data:
+            try:
+                soup = BeautifulSoup(data)
+                results_table = soup.find('table', attrs = {'id': 'searchResult'})
+                entries = results_table.find_all('tr')
+                for result in entries[1:]:
+                    link = result.find(href = re.compile('torrent\/\d+\/'))
+                    download = result.find(href = re.compile('magnet:'))
 
-        if not data:
+                    size = re.search('Size (?P<size>.+),', unicode(result.select('font.detDesc')[0])).group('size')
-            self.log.error('Failed to get data from %s.', search_url)
+                    if link and download:
-            return results
 
-        try:
+                        def extra_score(item):
-            soup = BeautifulSoup(data)
+                            trusted = (0, 10)[result.find('img', alt = re.compile('Trusted')) != None]
-            results_table = soup.find('table', attrs={'id': 'searchResult'})
+                            vip = (0, 20)[result.find('img', alt = re.compile('VIP')) != None]
-            entries = results_table.find_all('tr')
+                            confirmed = (0, 30)[result.find('img', alt = re.compile('Helpers')) != None]
-            for result in entries[1:]:
+                            moderated = (0, 50)[result.find('img', alt = re.compile('Moderator')) != None]
-                link = result.find(href=re.compile('torrent\/\d+\/'))
-                download = result.find(href=re.compile('magnet:'))
 
-                # Uploaded 06-28 02:27, Size 1.37 GiB,
+                            return confirmed + trusted + vip + moderated
 
-                size = re.search('Size (?P<size>.+),', unicode(result.select('font.detDesc'
+                        new = {
-                                 )[0])).group('size')
+                            'id': re.search('/(?P<id>\d+)/', link['href']).group('id'),
-                if link and download:
+                            'type': 'torrent_magnet',
-                    new = {
+                            'name': link.string,
-                        'type': 'magnet',
+                            'check_nzb': False,
-                        'check_nzb': False,
+                            'description': '',
-                        'description': '',
+                            'provider': self.getName(),
-                        'provider': self.getName(),
+                            'url': download['href'],
+                            'detail_url': self.getDomain(link['href']),
+                            'size': self.parseSize(size),
+                            'seeders': tryInt(result.find_all('td')[2].string),
+                            'leechers': tryInt(result.find_all('td')[3].string),
+                            'extra_score': extra_score,
+                            'get_more_info': self.getMoreInfo
                         }
 
-                    trusted = (0, 10)[result.find('img', alt=re.compile('Trusted')) != None]
+                        new['score'] = fireEvent('score.calculate', new, movie, single = True)
-                    vip = (0, 20)[result.find('img', alt=re.compile('VIP')) != None]
+                        is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
-                    confirmed = (0, 30)[result.find('img', alt=re.compile('Helpers')) != None]
+                                                        imdb_results = False, single_category = False, single = True)
-                    moderated = (0, 50)[result.find('img', alt=re.compile('Moderator')) != None]
-                    is_imdb = self.imdb_match(self.api_domain(link['href']), movie['library'
-                                              ]['identifier'])
 
-                    self.log.info('Name: %s', link.string)
+                        if is_correct_movie:
-                    self.log.info('Seeders: %s', result.find_all('td')[2].string)
+                            results.append(new)
-                    self.log.info('Leechers: %s', result.find_all('td')[3].string)
+                            self.found(new)
-                    self.log.info('Size: %s', size)
-                    self.log.info('Score(trusted + vip + moderated): %d', confirmed + trusted + vip
-                                  + moderated)
 
-                    new['name'] = link.string
+                return results
-                    new['id'] = re.search('/(?P<id>\d+)/', link['href']).group('id')
+            except:
-                    new['url'] = self.api_domain(link['href'])
+                log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
-                    new['magnet'] = download['href']
-                    new['size'] = self.parseSize(size)
-                    new['seeders'] = int(result.find_all('td')[2].string)
-                    new['leechers'] = int(result.find_all('td')[3].string)
-                    new['extra_score'] = lambda x: confirmed + trusted + vip + moderated
-                    new['score'] = fireEvent('score.calculate', new, movie, single=True)
 
-                    is_correct_movie = fireEvent(
+        return []
-                        'searcher.correct_movie',
-                        nzb=new,
-                        movie=movie,
-                        quality=quality,
-                        imdb_results=is_imdb,
-                        single_category=False,
-                        single=True,
-                        )
 
-                    if is_correct_movie:
+    def getMoreInfo(self, item):
-                        results.append(new)
+        full_description = self.getCache('tpb.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
-                        self.found(new)
+        html = BeautifulSoup(full_description)
+        nfo_pre = html.find('div', attrs = {'class':'nfo'})
+        description = toUnicode(nfo_pre.text) if nfo_pre else ''
 
-            return results
+        item['description'] = description
-        except Exception, error:
+        return item
-            self.log.debug(error)
-            return results
-
-    def download(self, url='', nzb_id=''):
-        return url

couchpotato/core/providers/torrent/torrentleech/main.py

@@ -67,7 +67,7 @@ class TorrentLeech(TorrentProvider):
                         'description': '',
                         'provider': self.getName(),
                         'url': self.urls['download'] % url['href'],
-                        'download': self.download,
+                        'download': self.loginDownload,
                         'size': self.parseSize(result.find_all('td')[4].string),
                         'seeders': tryInt(result.find('td', attrs = {'class' : 'seeders'}).string),
                         'leechers': tryInt(result.find('td', attrs = {'class' : 'leechers'}).string),