Merge branch 'refs/heads/develop' into tv

Conflicts: couchpotato/core/media/movie/_base/main.py couchpotato/core/providers/torrent/bitsoup/main.py couchpotato/core/providers/torrent/iptorrents/main.py couchpotato/core/providers/torrent/sceneaccess/main.py couchpotato/core/providers/torrent/torrentleech/main.py couchpotato/core/providers/torrent/torrentshack/main.py
2014-01-11 13:55:12 +01:00
parent 23967d11dd 516cbd73bd
commit 2a2fe448e7
99 changed files with 7125 additions and 3838 deletions
@@ -1,19 +1,15 @@
-from StringIO import StringIO
 from couchpotato.core.event import fireEvent, addEvent
-from couchpotato.core.helpers.encoding import tryUrlencode, ss, toSafeString, \
+from couchpotato.core.helpers.encoding import ss, toSafeString, \
    toUnicode, sp
 from couchpotato.core.helpers.variable import getExt, md5, isLocalIP
 from couchpotato.core.logger import CPLog
 from couchpotato.environment import Env
-from multipartpost import MultipartPostHandler
+import requests
 from tornado import template
 from tornado.web import StaticFileHandler
 from urlparse import urlparse
-import cookielib
 import glob
-import gzip
 import inspect
-import math
 import os.path
 import re
 import time
@@ -39,6 +35,7 @@ class Plugin(object):
    http_time_between_calls = 0
    http_failed_request = {}
    http_failed_disabled = {}
+    http_opener = requests.Session()

    def __new__(typ, *args, **kwargs):
        new_plugin = super(Plugin, typ).__new__(typ)
@@ -106,7 +103,9 @@ class Plugin(object):
            f.close()
            os.chmod(path, Env.getPermission('file'))
        except Exception, e:
-            log.error('Unable writing to file "%s": %s', (path, e))
+            log.error('Unable writing to file "%s": %s', (path, traceback.format_exc()))
+            if os.path.isfile(path):
+                os.remove(path)

    def makeDir(self, path):
        path = ss(path)
@@ -120,11 +119,11 @@ class Plugin(object):
        return False

    # http request
-    def urlopen(self, url, timeout = 30, params = None, headers = None, opener = None, multipart = False, show_error = True):
+    def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True, return_raw = False):
        url = urllib2.quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]")

        if not headers: headers = {}
-        if not params: params = {}
+        if not data: data = {}

        # Fill in some headers
        parsed_url = urlparse(url)
@@ -137,6 +136,8 @@ class Plugin(object):
        headers['Connection'] = headers.get('Connection', 'keep-alive')
        headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0')

+        r = self.http_opener
+
        # Don't try for failed requests
        if self.http_failed_disabled.get(host, 0) > 0:
            if self.http_failed_disabled[host] > (time.time() - 900):
@@ -152,45 +153,18 @@ class Plugin(object):
        self.wait(host)
        try:

-            # Make sure opener has the correct headers
-            if opener:
-                opener.add_headers = headers
+            kwargs = {
+                'headers': headers,
+                'data': data if len(data) > 0 else None,
+                'timeout': timeout,
+                'files': files,
+            }
+            method = 'post' if len(data) > 0 or files else 'get'

-            if multipart:
-                log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data'))
-                request = urllib2.Request(url, params, headers)
+            log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.iterkeys()] if isinstance(data, dict) else 'with data'))
+            response = r.request(method, url, verify = False, **kwargs)

-                if opener:
-                    opener.add_handler(MultipartPostHandler())
-                else:
-                    cookies = cookielib.CookieJar()
-                    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies), MultipartPostHandler)
-
-                response = opener.open(request, timeout = timeout)
-            else:
-                log.info('Opening url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data'))
-
-                if isinstance(params, (str, unicode)) and len(params) > 0:
-                    data = params
-                else:
-                    data = tryUrlencode(params) if len(params) > 0 else None
-
-                request = urllib2.Request(url, data, headers)
-
-                if opener:
-                    response = opener.open(request, timeout = timeout)
-                else:
-                    response = urllib2.urlopen(request, timeout = timeout)
-
-            # unzip if needed
-            if response.info().get('Content-Encoding') == 'gzip':
-                buf = StringIO(response.read())
-                f = gzip.GzipFile(fileobj = buf)
-                data = f.read()
-                f.close()
-            else:
-                data = response.read()
-            response.close()
+            data = response.content if return_raw else response.text

            self.http_failed_request[host] = 0
        except IOError:
@@ -218,15 +192,19 @@ class Plugin(object):
        return data

    def wait(self, host = ''):
+        if self.http_time_between_calls == 0:
+            return
+
        now = time.time()

        last_use = self.http_last_use.get(host, 0)
+        if last_use > 0:

-        wait = math.ceil(last_use - now + self.http_time_between_calls)
+            wait = (last_use - now) + self.http_time_between_calls

-        if wait > 0:
-            log.debug('Waiting for %s, %d seconds', (self.getName(), wait))
-            time.sleep(last_use - now + self.http_time_between_calls)
+            if wait > 0:
+                log.debug('Waiting for %s, %d seconds', (self.getName(), wait))
+                time.sleep(wait)

    def beforeCall(self, handler):
        self.isRunning('%s.%s' % (self.getName(), handler.__name__))
@@ -269,18 +247,19 @@ class Plugin(object):
            try:

                cache_timeout = 300
-                if kwargs.get('cache_timeout'):
+                if kwargs.has_key('cache_timeout'):
                    cache_timeout = kwargs.get('cache_timeout')
                    del kwargs['cache_timeout']

                data = self.urlopen(url, **kwargs)
-                if data:
+                if data and cache_timeout > 0:
                    self.setCache(cache_key, data, timeout = cache_timeout)
                return data
            except:
                if not kwargs.get('show_error', True):
                    raise

+                log.error('Failed getting cache: %s', (traceback.format_exc()))
                return ''

    def setCache(self, cache_key, value, timeout = 300):
@@ -93,7 +93,7 @@ class FileManager(Plugin):
            return dest

        try:
-            filedata = self.urlopen(url, **urlopen_kwargs)
+            filedata = self.urlopen(url, return_raw = True, **urlopen_kwargs)
        except:
            log.error('Failed downloading file %s: %s', (url, traceback.format_exc()))
            return False
@@ -314,6 +314,7 @@ class QualityPlugin(Plugin):
            'Movie.Name.1999.DVDRip-Group': 'dvdrip',
            'Movie.Name.1999.DVD-Rip-Group': 'dvdrip',
            'Movie.Name.1999.DVD-R-Group': 'dvdr',
+            'Movie.Name.Camelie.1999.720p.BluRay.x264-Group': '720p',
        }

        correct = 0
@@ -247,13 +247,13 @@ class Release(Plugin):
            'files': {}
        }), manual = True)

-        if success:
+        if success == True:
            db.expunge_all()
            rel = db.query(Relea).filter_by(id = id).first() # Get release again @RuudBurger why do we need to get it again??

            fireEvent('notify.frontend', type = 'release.manual_download', data = True, message = 'Successfully snatched "%s"' % item['name'])
        return {
-            'success': success
+            'success': success == True
        }

    def download(self, data, media, manual = False):
@@ -266,20 +266,21 @@ class Release(Plugin):
        # Test to see if any downloaders are enabled for this type
        downloader_enabled = fireEvent('download.enabled', manual, data, single = True)
        if not downloader_enabled:
-            log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', (data.get('protocol')))
+            log.info('Tried to download, but none of the "%s" downloaders are enabled or gave an error', data.get('protocol'))
            return False

        # Download NZB or torrent file
        filedata = None
        if data.get('download') and (ismethod(data.get('download')) or isfunction(data.get('download'))):
            filedata = data.get('download')(url = data.get('url'), nzb_id = data.get('id'))
+            log.info('Tried to download, but the "%s" provider gave an error', data.get('protocol'))
            if filedata == 'try_next':
                return filedata

        # Send NZB or torrent file to downloader
        download_result = fireEvent('download', data = data, media = media, manual = manual, filedata = filedata, single = True)
        if not download_result:
-            log.info('Tried to download, but the "%s" downloader gave an error', (data.get('protocol')))
+            log.info('Tried to download, but the "%s" downloader gave an error', data.get('protocol'))
            return False
        log.debug('Downloader result: %s', download_result)

@@ -3,7 +3,7 @@ from couchpotato.api import addApiView
 from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
 from couchpotato.core.helpers.encoding import toUnicode, ss, sp
 from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \
-    getImdb, link, symlink, tryInt, splitString, fnEscape
+    getImdb, link, symlink, tryInt, splitString, fnEscape, isSubFolder
 from couchpotato.core.logger import CPLog
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.settings.model import Library, File, Profile, Release, \
@@ -125,7 +125,7 @@ class Renamer(Plugin):
            return
        else:
            for item in no_process:
-                if '%s%s' % (base_folder, os.path.sep) in item:
+                if isSubFolder(item, base_folder):
                    log.error('To protect your data, the media libraries can\'t be inside of or the same as the "from" folder.')
                    return

@@ -157,13 +157,13 @@ class Renamer(Plugin):

        if media_folder:
            for item in no_process:
-                if '%s%s' % (media_folder, os.path.sep) in item:
+                if isSubFolder(item, media_folder):
                    log.error('To protect your data, the media libraries can\'t be inside of or the same as the provided media folder.')
                    return

        # Make sure a checkSnatched marked all downloads/seeds as such
        if not release_download and self.conf('run_every') > 0:
-            fireEvent('renamer.check_snatched')
+            self.checkSnatched(fire_scan = False)

        self.renaming_started = True

@@ -508,7 +508,10 @@ class Renamer(Plugin):
                        os.remove(src)

                        parent_dir = os.path.dirname(src)
-                        if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and not '%s%s' % (parent_dir, os.path.sep) in [destination, media_folder] and not '%s%s' % (base_folder, os.path.sep) in parent_dir:
+                        if delete_folders.count(parent_dir) == 0 and os.path.isdir(parent_dir) and \
+                            not isSubFolder(destination, parent_dir) and not isSubFolder(media_folder, parent_dir) and \
+                            not isSubFolder(parent_dir, base_folder):
+
                            delete_folders.append(parent_dir)

                except:
@@ -575,7 +578,7 @@ class Renamer(Plugin):
            # Break if CP wants to shut down
            if self.shuttingDown():
                break
-
+ 
        self.renaming_started = False

    def getRenameExtras(self, extra_type = '', replacements = None, folder_name = '', file_name = '', destination = '', group = None, current_file = '', remove_multiple = False):
@@ -806,7 +809,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
        except:
            loge('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc()))

-    def checkSnatched(self):
+    def checkSnatched(self, fire_scan = True):

        if self.checking_snatched:
            log.debug('Already checking snatched')
@@ -829,20 +832,29 @@ Remove it if you want it to be renamed (again, or at least let it try again)

        # Collect all download information with the download IDs from the releases
        download_ids = []
+        no_status_support = []
        try:
            for rel in rels:
                rel_dict = rel.to_dict({'info': {}})
                if rel_dict['info'].get('download_id') and rel_dict['info'].get('download_downloader'):
                    download_ids.append({'id': rel_dict['info']['download_id'], 'downloader': rel_dict['info']['download_downloader']})
+
+                ds = rel_dict['info'].get('download_status_support')
+                if ds == False or ds == 'False':
+                    no_status_support.append(ss(rel_dict['info'].get('download_downloader')))
        except:
            log.error('Error getting download IDs from database')
            self.checking_snatched = False
            return False

-        release_downloads = fireEvent('download.status', download_ids, merge = True)
+        release_downloads = fireEvent('download.status', download_ids, merge = True) if download_ids else []
+
+        if len(no_status_support) > 0:
+            log.debug('Download status functionality is not implemented for one of the active downloaders: %s', no_status_support)
+
        if not release_downloads:
-            log.debug('Download status functionality is not implemented for any active downloaders.')
-            fireEvent('renamer.scan')
+            if fire_scan:
+                self.scan()

            self.checking_snatched = False
            return True
@@ -982,7 +994,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
            if release_download['scan']:
                if release_download['pause'] and self.conf('file_action') == 'link':
                    fireEvent('download.pause', release_download = release_download, pause = True, single = True)
-                fireEvent('renamer.scan', release_download = release_download)
+                self.scan(release_download = release_download)
                if release_download['pause'] and self.conf('file_action') == 'link':
                    fireEvent('download.pause', release_download = release_download, pause = False, single = True)
            if release_download['process_complete']:
@@ -993,8 +1005,8 @@ Remove it if you want it to be renamed (again, or at least let it try again)
                    # Ask the downloader to process the item
                    fireEvent('download.process_complete', release_download = release_download, single = True)

-        if scan_required:
-            fireEvent('renamer.scan')
+        if fire_scan and (scan_required or len(no_status_support) > 0):
+            self.scan()

        self.checking_snatched = False
        return True
@@ -1044,7 +1056,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
        return release_download['id'] and release_download['downloader'] and release_download['folder']

    def movieInFromFolder(self, media_folder):
-        return media_folder and '%s%s' % (sp(self.conf('from')), os.path.sep) in sp(media_folder) or not media_folder
+        return media_folder and isSubFolder(media_folder, sp(self.conf('from'))) or not media_folder

    def extractFiles(self, folder = None, media_folder = None, files = None, cleanup = False):
        if not files: files = []
@@ -80,7 +80,8 @@ class Scanner(Plugin):
        'hdtv': ['hdtv']
    }

-    clean = '[ _\,\.\(\)\[\]\-](extended.cut|directors.cut|french|swedisch|danish|dutch|swesub|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip|hdtvrip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|cd[1-9]|\[.*\])([ _\,\.\(\)\[\]\-]|$)'
+    clean = '[ _\,\.\(\)\[\]\-]?(extended.cut|directors.cut|french|swedisch|danish|dutch|swesub|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip' \
+            '|hdtvrip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|cd[1-9]|\[.*\])([ _\,\.\(\)\[\]\-]|$)'
    multipart_regex = [
        '[ _\.-]+cd[ _\.-]*([0-9a-d]+)', #*cd1
        '[ _\.-]+dvd[ _\.-]*([0-9a-d]+)', #*dvd1