Merge branch 'bwq-master' into develop

couchpotato/core/plugins/base.py

@@ -215,7 +215,19 @@ class Plugin(object):
                     cache_timeout = kwargs.get('cache_timeout')
                     del kwargs['cache_timeout']
 
-                data = self.urlopen(url, **kwargs)
+                opener = None
+                if kwargs.get('opener'):
+                    opener = kwargs.get('opener')
+                    del kwargs['opener']
+                
+                if opener:
+                    log.info('Opening url: %s', url)
+                    f = opener.open(url)
+                    data = f.read()
+                    f.close()
+                else:
+                    data = self.urlopen(url, **kwargs)
+
                 if data:
                     self.setCache(cache_key, data, timeout = cache_timeout)
                 return data

couchpotato/core/providers/torrent/base.py

@@ -1,5 +1,31 @@
 from couchpotato.core.providers.base import YarrProvider
+from couchpotato.core.logger import CPLog
+import urllib2
+import cookielib
+
+log = CPLog(__name__)
 
 
 class TorrentProvider(YarrProvider):
     type = 'torrent'
+
+    def login(self, params):
+        
+        try:
+            cookiejar = cookielib.CookieJar()
+            opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
+            urllib2.install_opener(opener)
+            f = opener.open(self.urls['login'], params)
+            loginData = f.read()
+            f.close()
+        
+        except:    
+            log.error('Failed to login.')
+        
+        return opener  
+        
+    def download(self, url = '', nzb_id = ''):
+        loginParams = self.getLoginParams()
+        self.login(params = loginParams)
+        torrent = self.urlopen(url)
+        return torrent

couchpotato/core/providers/torrent/sceneaccess/main.py

@@ -20,6 +20,7 @@ class SceneAccess(TorrentProvider):
 
     urls = {
         'test': 'https://www.sceneaccess.eu/',
+        'login' : 'https://www.sceneaccess.eu/login',
         'detail': 'https://www.sceneaccess.eu/details?id=%s',
         'search': 'https://www.sceneaccess.eu/browse?search=%s&method=2&c%d=%d',
         'download': 'https://www.sceneaccess.eu/%s',
@@ -32,6 +33,10 @@ class SceneAccess(TorrentProvider):
     ]
 
     http_time_between_calls = 1 #seconds
+    
+    def getLoginParams(self):
+        loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), submit='come on in'))
+        return loginParams
 
     def search(self, movie, quality):
 
@@ -40,30 +45,21 @@ class SceneAccess(TorrentProvider):
             return results
 
         cache_key = 'sceneaccess.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
-        searchUrl =  self.urls['search'] % (quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0], self.getCatId(quality['identifier'])[0])
+        searchUrl =  self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0], self.getCatId(quality['identifier'])[0])
-        data = self.getCache(cache_key, searchUrl)
+        loginParams = self.getLoginParams()
+
+        opener = self.login(params = loginParams)
+        if not opener:
+            log.info("Couldn't login at SceneAccess")
+            return results
+
+        data = self.getCache(cache_key, searchUrl, opener = opener)
 
         if data:
-
+            html = BeautifulSoup(data)
-            cat_ids = self.getCatId(quality['identifier'])
+        
-            
+        else:
-            try:
+            log.info("No results found at SceneAccess")
-                cookiejar = cookielib.CookieJar()
-                opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
-                urllib2.install_opener(opener)
-                params = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), submit='come on in'))
-                f = opener.open('https://www.sceneaccess.eu/login', params)
-                data = f.read()
-                f.close()
-                f = opener.open(searchUrl)
-                data = f.read()
-                f.close()
-            
-            except (IOError, URLError):
-                log.error('Failed to open %s.' % url)
-                return results
-
-        html = BeautifulSoup(data)
 
         try:
             resultsTable = html.find('table', attrs = {'id' : 'torrents-table'})            
@@ -89,11 +85,12 @@ class SceneAccess(TorrentProvider):
                 else:
                     new['leechers'] = 0
             
-                new['imdbid'] = movie['library']['identifier']
+                details = self.urls['detail'] % new['id']
-                new['extra_score'] = self.extra_score
+                imdb_results = self.imdbMatch(details, movie['library']['identifier'])
+
                 new['score'] = fireEvent('score.calculate', new, movie, single = True)
                 is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
-                                                 imdb_results = True, single_category = False, single = True)
+                                                 imdb_results = imdb_results, single_category = False, single = True)
 
                 if is_correct_movie:
                     new['download'] = self.download
@@ -105,10 +102,6 @@ class SceneAccess(TorrentProvider):
             log.info("No results found at SceneAccess")
             return []
 
-    def extra_score(self, nzb):
-        url = self.urls['detail'] % nzb['id']
-        imdbId = nzb['imdbid']
-        return self.imdbMatch(url, imdbId)
 
     def imdbMatch(self, url, imdbId):
         try:
@@ -116,7 +109,7 @@ class SceneAccess(TorrentProvider):
             pass
         except IOError:
             log.error('Failed to open %s.' % url)
-            return ''
+            return False
 
         html = BeautifulSoup(data)
         imdbDiv = html.find('span', attrs = {'class':'i_link'})
@@ -124,11 +117,5 @@ class SceneAccess(TorrentProvider):
         imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId)
 
         if 'imdb.com/title/' + imdbId in imdbDiv or 'imdb.com/title/' + imdbIdAlt in imdbDiv:
-            return 50
+            return True
-        return 0
+        return False
-
-    def download(self, url = '', nzb_id = ''):
-        torrent = self.urlopen(url)
-        return torrent
-
-

couchpotato/core/providers/torrent/scenehd/main.py

@@ -21,6 +21,7 @@ class SceneHD(TorrentProvider):
 
     urls = {
         'test': 'http://scenehd.org/',
+        'login' : 'http://scenehd.org/takelogin.php',
         'detail': 'http://scenehd.org/details.php?id=%s',
         'search': 'http://scenehd.org/browse.php?ajax&search=%s',
         'download': 'http://scenehd.org/download.php?id=%s',
@@ -28,6 +29,10 @@ class SceneHD(TorrentProvider):
    
     http_time_between_calls = 1 #seconds
 
+    def getLoginParams(self):
+        loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), ssl='yes'))
+        return loginParams
+
     def search(self, movie, quality):
 
         results = []
@@ -35,28 +40,21 @@ class SceneHD(TorrentProvider):
             return results
 
         cache_key = 'scenehd.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
-        searchUrl =  self.urls['search'] % (quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']))
+        searchUrl =  self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']))
-        data = self.getCache(cache_key, searchUrl)
+        loginParams = self.getLoginParams()
+
+        opener = self.login(params = loginParams)
+        if not opener:
+            log.error("Couldn't login at SceneHD")
+            return results
+
+        data = self.getCache(cache_key, searchUrl, opener = opener)
 
         if data:
-            
+            html = BeautifulSoup(data)
-            try:
+        
-                cookiejar = cookielib.CookieJar()
+        else:
-                opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
+            log.info("No results found at SceneHD")
-                urllib2.install_opener(opener)
-                params = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), ssl='yes'))
-                f = opener.open('http://scenehd.org/takelogin.php', params)
-                data = f.read()
-                f.close()
-                f = opener.open(searchUrl)
-                data = f.read()
-                f.close()
-            
-            except (IOError, URLError):
-                log.error('Failed to open %s.' % url)
-                return results
-
-        html = BeautifulSoup(data)
         
         try:
             resultsTable = html.findAll('table')[6]
@@ -84,17 +82,16 @@ class SceneHD(TorrentProvider):
                 new['name'] = detailLink['title']
             
                 imdbLink = allCells[1].find('a')
+                imdb_results = False
+
                 if imdbLink:
-                    new['imdbresult'] = imdbLink['href'].replace('http://www.imdb.com/title/','').rstrip('/')
+                    imdbFound = imdbLink['href'].replace('http://www.imdb.com/title/','').rstrip('/')
-                else:
+                    imdb_results = self.imdbMatch(imdbFound, movie['library']['identifier'])
-                    new['imdbresult'] = 'tt00000000'
                
                 new['url'] = self.urls['download'] % new['id']
-                new['imdbid'] = movie['library']['identifier']
-                new['extra_score'] = self.extra_score
                 new['score'] = fireEvent('score.calculate', new, movie, single = True)
                 is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
-                                                 imdb_results = True, single_category = False, single = True)
+                                                 imdb_results = imdb_results, single_category = False, single = True)
 
                 if is_correct_movie:
                     new['download'] = self.download
@@ -106,14 +103,9 @@ class SceneHD(TorrentProvider):
             log.info("No results found at SceneHD")
             return []
 
-    def extra_score(self, nzb):
+    def imdbMatch(self, imdbFound, imdbId):
-        imdbIdAlt = re.sub('tt[0]*', 'tt', nzb['imdbresult'])
+        imdbIdAlt = re.sub('tt[0]*', 'tt', imdbFound)
-        if nzb['imdbresult'] == nzb['imdbid'] or imdbIdAlt == nzb['imdbid']:
+        if imdbFound == imdbId or imdbIdAlt == imdbId:
-            return 50
+            return True
-        return 0
+        return False
-
-    def download(self, url = '', nzb_id = ''):
-        torrent = self.urlopen(url)
-        return torrent
-
 

couchpotato/core/providers/torrent/torrentleech/main.py

@@ -22,6 +22,7 @@ class TorrentLeech(TorrentProvider):
 
     urls = {
         'test' : 'http://torrentleech.org/',
+        'login' : 'http://torrentleech.org/user/account/login/',
         'detail' : 'http://torrentleech.org/torrent/%s',
         'search' : 'http://torrentleech.org/torrents/browse/index/query/%s/categories/%d',
         'download' : 'http://torrentleech.org%s',
@@ -38,6 +39,10 @@ class TorrentLeech(TorrentProvider):
     ]
 
     http_time_between_calls = 1 #seconds
+    
+    def getLoginParams(self):
+        loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), remember_me='on', login='submit'))
+        return loginParams
 
     def search(self, movie, quality):
 
@@ -46,30 +51,21 @@ class TorrentLeech(TorrentProvider):
             return results
 
         cache_key = 'torrentleech.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
-        searchUrl =  self.urls['search'] % (quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
+        searchUrl =  self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
-        data = self.getCache(cache_key, searchUrl)
+        loginParams = self.getLoginParams()
+      
+        opener = self.login(params = loginParams)
+        if not opener:
+            log.info("Couldn't login at Torrentleech")
+            return results
+
+        data = self.getCache(cache_key, searchUrl, opener = opener)
 
         if data:
-
+            html = BeautifulSoup(data)
-            cat_ids = self.getCatId(quality['identifier'])
+        
-            
+        else:
-            try:
+            log.info("No results found at Torrentleech")
-                cookiejar = cookielib.CookieJar()
-                opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
-                urllib2.install_opener(opener)
-                params = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), remember_me='on', login='submit'))
-                f = opener.open('http://torrentleech.org/user/account/login/', params)
-                data = f.read()
-                f.close()
-                f = opener.open(searchUrl)
-                data = f.read()
-                f.close()
-            
-            except (IOError, URLError):
-                log.error('Failed to open %s.' % url)
-                return results
-
-        html = BeautifulSoup(data)
 
         try:
             resultsTable = html.find('table', attrs = {'id' : 'torrenttable'})            
@@ -89,13 +85,14 @@ class TorrentLeech(TorrentProvider):
                 new['url'] = self.urls['download'] % url['href']
                 new['size'] = self.parseSize(result.findAll('td')[4].string)
                 new['seeders'] = int(result.find('td', attrs = {'class' : 'seeders'}).string)
-                new['leechers'] = int(result.find('td', attrs = {'class' : 'leechers'}).string)                
+                new['leechers'] = int(result.find('td', attrs = {'class' : 'leechers'}).string)   
-                new['imdbid'] = movie['library']['identifier']
+                
+                details = self.urls['detail'] % new['id']
+                imdb_results = self.imdbMatch(details, movie['library']['identifier'])
                 
-                new['extra_score'] = self.extra_score
                 new['score'] = fireEvent('score.calculate', new, movie, single = True)
                 is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
-                                                imdb_results = True, single_category = False, single = True)
+                                                imdb_results = imdb_results, single_category = False, single = True)
 
                 if is_correct_movie:
                     new['download'] = self.download
@@ -107,10 +104,6 @@ class TorrentLeech(TorrentProvider):
             log.info("No results found at TorrentLeech")
             return []
 
-    def extra_score(self, nzb):
-        url = self.urls['detail'] % nzb['id']
-        imdbId = nzb['imdbid']
-        return self.imdbMatch(url, imdbId)
 
     def imdbMatch(self, url, imdbId):
         try:
@@ -118,16 +111,10 @@ class TorrentLeech(TorrentProvider):
             pass
         except IOError:
             log.error('Failed to open %s.' % url)
-            return ''
+            return False
 
         imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId)
         data = unicode(data, errors='ignore')
         if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' + imdbIdAlt in data:
-            return 50
+            return True
-        return 0
+        return False
-
-    def download(self, url = '', nzb_id = ''):
-        torrent = self.urlopen(url)
-        return torrent
-
-