One up

This is my approach on working with Eden, maybe a little late since Frodo is almost released, but better late then never.

- First detect the JSON-RPC version XBMC is running (once per boot of CouchPotatoServer on the first notification, except for sending test message then the JSON version is always checked).
- Set a variable indicating whether or not to use JSON (or normal http).
- If JSON should be used, proceed as before this commit.
- If normal-http should be used, use 'notifyXBMCnoJSON' func
- 'notifyXBMCnoJSON' just opens a specific XBMC api url, unfortunately importing urllib for this was necessary to escape the message strings.

TODO: support multiple XBMC hosts, right now the last host in the hosts array will set the 'useJSONnotifications' var.

Conflicts:

	couchpotato/core/notifications/xbmc/main.py

Conflicts:
	couchpotato/core/notifications/xbmc/main.py

couchpotato/api.py

@@ -1,6 +1,5 @@
 from flask.blueprints import Blueprint
 from flask.helpers import url_for
-from tornado.ioloop import IOLoop
 from tornado.web import RequestHandler, asynchronous
 from werkzeug.utils import redirect
 
@@ -11,7 +10,11 @@ api_nonblock = {}
 
 
 class NonBlockHandler(RequestHandler):
-    stoppers = []
+
+    def __init__(self, application, request, **kwargs):
+        cls = NonBlockHandler
+        cls.stoppers = []
+        super(NonBlockHandler, self).__init__(application, request, **kwargs)
 
     @asynchronous
     def get(self, route):

couchpotato/core/downloaders/base.py

@@ -1,16 +1,17 @@
 from base64 import b32decode, b16encode
 from couchpotato.core.event import addEvent
 from couchpotato.core.logger import CPLog
-from couchpotato.core.plugins.base import Plugin
+from couchpotato.core.providers.base import Provider
 import random
 import re
 
 log = CPLog(__name__)
 
 
-class Downloader(Plugin):
+class Downloader(Provider):
 
     type = []
+    http_time_between_calls = 0
 
     torrent_sources = [
         'http://torrage.com/torrent/%s.torrent',

couchpotato/core/downloaders/nzbget/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'downloaders',
             'name': 'nzbget',
             'label': 'NZBGet',
-            'description': 'Send NZBs to your NZBGet installation.',
+            'description': 'Use <a href="http://nzbget.sourceforge.net/Main_Page" target="_blank">NZBGet</a> to download NZBs.',
             'options': [
                 {
                     'name': 'enabled',
@@ -25,6 +25,7 @@ config = [{
                 },
                 {
                     'name': 'password',
+                    'type': 'password',
                     'description': 'Default NZBGet password is <i>tegbzn6789</i>',
                 },
                 {

couchpotato/core/downloaders/nzbvortex/__init__.py

@@ -0,0 +1,46 @@
+from .main import NZBVortex
+
+def start():
+    return NZBVortex()
+
+config = [{
+    'name': 'nzbvortex',
+    'groups': [
+        {
+            'tab': 'downloaders',
+            'name': 'nzbvortex',
+            'label': 'NZBVortex',
+            'description': 'Use <a href="http://www.nzbvortex.com/landing/" target="_blank">NZBVortex</a> to download NZBs.',
+            'wizard': True,
+            'options': [
+                {
+                    'name': 'enabled',
+                    'default': 0,
+                    'type': 'enabler',
+                    'radio_group': 'nzb',
+                },
+                {
+                    'name': 'host',
+                    'default': 'https://localhost:4321',
+                },
+                {
+                    'name': 'api_key',
+                    'label': 'Api Key',
+                },
+                {
+                    'name': 'manual',
+                    'default': False,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Disable this downloader for automated searches, but use it when I manually send a release.',
+                },
+                {
+                    'name': 'delete_failed',
+                    'default': True,
+                    'type': 'bool',
+                    'description': 'Delete a release after the download has failed.',
+                },
+            ],
+        }
+    ],
+}]

couchpotato/core/downloaders/nzbvortex/main.py

@@ -0,0 +1,176 @@
+from base64 import b64encode
+from couchpotato.core.downloaders.base import Downloader
+from couchpotato.core.helpers.encoding import tryUrlencode, ss
+from couchpotato.core.helpers.variable import cleanHost
+from couchpotato.core.logger import CPLog
+from urllib2 import URLError
+from uuid import uuid4
+import hashlib
+import httplib
+import json
+import socket
+import ssl
+import sys
+import traceback
+import urllib2
+
+log = CPLog(__name__)
+
+class NZBVortex(Downloader):
+
+    type = ['nzb']
+    api_level = None
+    session_id = None
+
+    def download(self, data = {}, movie = {}, manual = False, filedata = None):
+
+        if self.isDisabled(manual) or not self.isCorrectType(data.get('type')) or not self.getApiLevel():
+            return
+
+        # Send the nzb
+        try:
+            nzb_filename = self.createFileName(data, filedata, movie)
+            self.call('nzb/add', params = {'file': (ss(nzb_filename), filedata)}, multipart = True)
+
+            return True
+        except:
+            log.error('Something went wrong sending the NZB file: %s', traceback.format_exc())
+            return False
+
+    def getAllDownloadStatus(self):
+
+        if self.isDisabled(manual = True):
+            return False
+
+        raw_statuses = self.call('nzb')
+
+        statuses = []
+        for item in raw_statuses.get('nzbs', []):
+
+            # Check status
+            status = 'busy'
+            if item['state'] == 20:
+                status = 'completed'
+            elif item['state'] in [21, 22, 24]:
+                status = 'failed'
+
+            statuses.append({
+                'id': item['id'],
+                'name': item['uiTitle'],
+                'status': status,
+                'original_status': item['state'],
+                'timeleft':-1,
+            })
+
+        return statuses
+
+    def removeFailed(self, item):
+
+        if not self.conf('delete_failed', default = True):
+            return False
+
+        log.info('%s failed downloading, deleting...', item['name'])
+
+        try:
+            self.call('nzb/%s/cancel' % item['id'])
+        except:
+            log.error('Failed deleting: %s', traceback.format_exc(0))
+            return False
+
+        return True
+
+    def login(self):
+
+        nonce = self.call('auth/nonce', auth = False).get('authNonce')
+        cnonce = uuid4().hex
+        hashed = b64encode(hashlib.sha256('%s:%s:%s' % (nonce, cnonce, self.conf('api_key'))).digest())
+
+        params = {
+            'nonce': nonce,
+            'cnonce': cnonce,
+            'hash': hashed
+        }
+
+        login_data = self.call('auth/login', parameters = params, auth = False)
+
+        # Save for later
+        if login_data.get('loginResult') == 'successful':
+            self.session_id = login_data.get('sessionID')
+            return True
+
+        log.error('Login failed, please check you api-key')
+        return False
+
+
+    def call(self, call, parameters = {}, repeat = False, auth = True, *args, **kwargs):
+
+        # Login first
+        if not self.session_id and auth:
+            self.login()
+
+        # Always add session id to request
+        if self.session_id:
+            parameters['sessionid'] = self.session_id
+
+        params = tryUrlencode(parameters)
+
+        url = cleanHost(self.conf('host')) + 'api/' + call
+        url_opener = urllib2.build_opener(HTTPSHandler())
+
+        try:
+            data = self.urlopen('%s?%s' % (url, params), opener = url_opener, *args, **kwargs)
+
+            if data:
+                return json.loads(data)
+        except URLError, e:
+            if hasattr(e, 'code') and e.code == 403:
+                # Try login and do again
+                if not repeat:
+                    self.login()
+                    return self.call(call, parameters = parameters, repeat = True, *args, **kwargs)
+
+            log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
+        except:
+            log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
+
+        return {}
+
+    def getApiLevel(self):
+
+        if not self.api_level:
+
+            url = cleanHost(self.conf('host')) + 'api/app/apilevel'
+            url_opener = urllib2.build_opener(HTTPSHandler())
+
+            try:
+                data = self.urlopen(url, opener = url_opener, show_error = False)
+                self.api_level = float(json.loads(data).get('apilevel'))
+            except URLError, e:
+                if hasattr(e, 'code') and e.code == 403:
+                    log.error('This version of NZBVortex isn\'t supported. Please update to 2.8.6 or higher')
+                else:
+                    log.error('NZBVortex doesn\'t seem to be running or maybe the remote option isn\'t enabled yet: %s', traceback.format_exc(1))
+
+        return self.api_level
+
+
+class HTTPSConnection(httplib.HTTPSConnection):
+    def __init__(self, *args, **kwargs):
+        httplib.HTTPSConnection.__init__(self, *args, **kwargs)
+
+    def connect(self):
+        sock = socket.create_connection((self.host, self.port), self.timeout)
+        if sys.version_info < (2, 6, 7):
+            if hasattr(self, '_tunnel_host'):
+                self.sock = sock
+                self._tunnel()
+        else:
+            if self._tunnel_host:
+                self.sock = sock
+                self._tunnel()
+
+        self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version = ssl.PROTOCOL_TLSv1)
+
+class HTTPSHandler(urllib2.HTTPSHandler):
+    def https_open(self, req):
+        return self.do_open(HTTPSConnection, req)

couchpotato/core/downloaders/pneumatic/__init__.py

@@ -11,7 +11,7 @@ config = [{
             'tab': 'downloaders',
             'name': 'pneumatic',
             'label': 'Pneumatic',
-            'description': 'Download the .strm file to a specific folder.',
+            'description': 'Use <a href="http://forum.xbmc.org/showthread.php?tid=97657" target="_blank">Pneumatic</a> to download .strm files.',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/downloaders/sabnzbd/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'downloaders',
             'name': 'sabnzbd',
             'label': 'Sabnzbd',
-            'description': 'Send NZBs to your Sabnzbd installation.',
+            'description': 'Use <a href="http://sabnzbd.org/" target="_blank">SABnzbd</a> to download NZBs.',
             'wizard': True,
             'options': [
                 {

couchpotato/core/downloaders/sabnzbd/main.py

@@ -1,5 +1,5 @@
 from couchpotato.core.downloaders.base import Downloader
-from couchpotato.core.helpers.encoding import tryUrlencode
+from couchpotato.core.helpers.encoding import tryUrlencode, ss
 from couchpotato.core.helpers.variable import cleanHost, mergeDicts
 from couchpotato.core.logger import CPLog
 from urllib2 import URLError
@@ -41,7 +41,7 @@ class Sabnzbd(Downloader):
 
         try:
             if params.get('mode') is 'addfile':
-                sab = self.urlopen(url, timeout = 60, params = {'nzbfile': (nzb_filename, filedata)}, multipart = True, show_error = False)
+                sab = self.urlopen(url, timeout = 60, params = {'nzbfile': (ss(nzb_filename), filedata)}, multipart = True, show_error = False)
             else:
                 sab = self.urlopen(url, timeout = 60, show_error = False)
         except URLError:
@@ -65,7 +65,7 @@ class Sabnzbd(Downloader):
             return False
 
     def getAllDownloadStatus(self):
-        if self.isDisabled(manual = False):
+        if self.isDisabled(manual = True):
             return False
 
         log.debug('Checking SABnzbd download status.')

couchpotato/core/downloaders/synology/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'downloaders',
             'name': 'synology',
             'label': 'Synology',
-            'description': 'Send torrents to Synology\'s Download Station.',
+            'description': 'Use <a href="http://www.synology.com/dsm/home_home_applications_download_station.php" target="_blank">Synology Download Station</a> to download.',
             'wizard': True,
             'options': [
                 {

couchpotato/core/downloaders/transmission/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'downloaders',
             'name': 'transmission',
             'label': 'Transmission',
-            'description': 'Send torrents to Transmission.',
+            'description': 'Use <a href="http://www.transmissionbt.com/" target="_blank">Transmission</a> to download torrents.',
             'wizard': True,
             'options': [
                 {

couchpotato/core/downloaders/utorrent/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'downloaders',
             'name': 'utorrent',
             'label': 'uTorrent',
-            'description': 'Send torrents to uTorrent.',
+            'description': 'Use <a href="http://www.utorrent.com/" target="_blank">uTorrent</a> to download torrents.',
             'wizard': True,
             'options': [
                 {

couchpotato/core/notifications/growl/main.py

@@ -37,8 +37,11 @@ class Growl(Notification):
             )
             self.growl.register()
             self.registered = True
-        except:
-            log.error('Failed register of growl: %s', traceback.format_exc())
+        except Exception, e:
+            if 'timed out' in str(e):
+                self.registered = True
+            else:
+                log.error('Failed register of growl: %s', traceback.format_exc())
 
     def notify(self, message = '', data = {}, listener = None):
         if self.isDisabled(): return

couchpotato/core/notifications/toasty/__init__.py

@@ -0,0 +1,32 @@
+from .main import Toasty
+
+def start():
+    return Toasty()
+
+config = [{
+    'name': 'toasty',
+    'groups': [
+        {
+            'tab': 'notifications',
+            'name': 'toasty',
+            'options': [
+                {
+                    'name': 'enabled',
+                    'default': 0,
+                    'type': 'enabler',
+                },
+                {
+                    'name': 'api_key',
+                    'label': 'Device ID',
+                },
+                {
+                    'name': 'on_snatch',
+                    'default': 0,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Also send message when movie is snatched.',
+                },
+            ],
+        }
+    ],
+}]

couchpotato/core/notifications/toasty/main.py

@@ -0,0 +1,30 @@
+from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
+from couchpotato.core.logger import CPLog
+from couchpotato.core.notifications.base import Notification
+import traceback
+
+log = CPLog(__name__)
+
+class Toasty(Notification):
+
+    urls = {
+        'api': 'http://api.supertoasty.com/notify/%s?%s'
+    }
+
+    def notify(self, message = '', data = {}, listener = None):
+        if self.isDisabled(): return
+
+        data = {
+            'title': self.default_title,
+            'text': toUnicode(message),
+            'sender': toUnicode("CouchPotato"),
+            'image': 'https://raw.github.com/RuudBurger/CouchPotatoServer/master/couchpotato/static/images/homescreen.png',
+        }
+
+        try:
+            self.urlopen(self.urls['api'] % (self.conf('api_key'), tryUrlencode(data)), show_error = False)
+            return True
+        except:
+            log.error('Toasty failed: %s', traceback.format_exc())
+
+        return False

couchpotato/core/notifications/xbmc/__init__.py

@@ -10,6 +10,7 @@ config = [{
             'tab': 'notifications',
             'name': 'xbmc',
             'label': 'XBMC',
+            'description': 'v11 (Eden) and v12 (Frodo)',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/notifications/xbmc/main.py

@@ -4,6 +4,7 @@ from couchpotato.core.notifications.base import Notification
 from flask.helpers import json
 import base64
 import traceback
+import urllib
 
 log = CPLog(__name__)
 
@@ -11,27 +12,147 @@ log = CPLog(__name__)
 class XBMC(Notification):
 
     listen_to = ['renamer.after']
+    use_json_notifications = {}
 
     def notify(self, message = '', data = {}, listener = None):
         if self.isDisabled(): return
 
         hosts = splitString(self.conf('host'))
+
         successful = 0
         for host in hosts:
-            response = self.request(host, [
-                ('GUI.ShowNotification', {"title":"CouchPotato", "message":message}),
-                ('VideoLibrary.Scan', {}),
-            ])
+
+            if self.use_json_notifications.get(host) is None:
+                self.getXBMCJSONversion(host, message = message)
+
+            if self.use_json_notifications.get(host):
+                response = self.request(host, [
+                    ('GUI.ShowNotification', {'title':self.default_title, 'message':message}),
+                    ('VideoLibrary.Scan', {}),
+                ])
+            else:
+                response = self.notifyXBMCnoJSON(host, {'title':self.default_title, 'message':message})
+                response += self.request(host, [('VideoLibrary.Scan', {})])
 
             try:
                 for result in response:
-                    if result['result'] == "OK":
+                    if (result.get('result') and result['result'] == 'OK'):
                         successful += 1
+                    elif (result.get('error')):
+                        log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code']))
+
             except:
                 log.error('Failed parsing results: %s', traceback.format_exc())
 
         return successful == len(hosts) * 2
 
+    def getXBMCJSONversion(self, host, message = ''):
+
+        success = False
+
+        # XBMC JSON-RPC version request
+        response = self.request(host, [
+            ('JSONRPC.Version', {})
+            ])
+        for result in response:
+            if (result.get('result') and type(result['result']['version']).__name__ == 'int'):
+                # only v2 and v4 return an int object
+                # v6 (as of XBMC v12(Frodo)) is required to send notifications
+                xbmc_rpc_version = str(result['result']['version'])
+
+                log.debug('XBMC JSON-RPC Version: %s ; Notifications by JSON-RPC only supported for v6 [as of XBMC v12(Frodo)]', xbmc_rpc_version)
+
+                # disable JSON use
+                self.use_json_notifications[host] = False
+
+                # send the text message
+                resp = self.notifyXBMCnoJSON(host, {'title':self.default_title, 'message':message})
+                for result in resp:
+                    if (result.get('result') and result['result'] == 'OK'):
+                        log.debug('Message delivered successfully!')
+                        success = True
+                        break
+                    elif (result.get('error')):
+                        log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code']))
+                        break
+
+            elif (result.get('result') and type(result['result']['version']).__name__ == 'dict'):
+                # XBMC JSON-RPC v6 returns an array object containing
+                # major, minor and patch number
+                xbmc_rpc_version = str(result['result']['version']['major'])
+                xbmc_rpc_version += '.' + str(result['result']['version']['minor'])
+                xbmc_rpc_version += '.' + str(result['result']['version']['patch'])
+
+                log.debug('XBMC JSON-RPC Version: %s', xbmc_rpc_version)
+
+                # ok, XBMC version is supported
+                self.use_json_notifications[host] = True
+
+                # send the text message
+                resp = self.request(host, [('GUI.ShowNotification', {'title':self.default_title, 'message':message})])
+                for result in resp:
+                    if (result.get('result') and result['result'] == 'OK'):
+                        log.debug('Message delivered successfully!')
+                        success = True
+                        break
+                    elif (result.get('error')):
+                        log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code']))
+                        break
+
+            # error getting version info (we do have contact with XBMC though)
+            elif (result.get('error')):
+                log.error('XBMC error; %s: %s (%s)', (result['id'], result['error']['message'], result['error']['code']))
+
+        log.debug('Use JSON notifications: %s ', self.use_json_notifications)
+
+        return success
+
+    def notifyXBMCnoJSON(self, host, data):
+
+        server = 'http://%s/xbmcCmds/' % host
+
+        # title, message [, timeout , image #can be added!]
+        cmd = "xbmcHttp?command=ExecBuiltIn(Notification('%s','%s'))" % (urllib.quote(data['title']), urllib.quote(data['message']))
+        server += cmd
+
+        # I have no idea what to set to, just tried text/plain and seems to be working :)
+        headers = {
+            'Content-Type': 'text/plain',
+        }
+
+        # authentication support
+        if self.conf('password'):
+            base64string = base64.encodestring('%s:%s' % (self.conf('username'), self.conf('password'))).replace('\n', '')
+            headers['Authorization'] = 'Basic %s' % base64string
+
+        try:
+            log.debug('Sending non-JSON-type request to %s: %s', (host, data))
+
+            # response wil either be 'OK':
+            # <html>
+            # <li>OK
+            # </html>
+            #
+            # or 'Error':
+            # <html>
+            # <li>Error:<message>
+            # </html>
+            #
+            response = self.urlopen(server, headers = headers)
+
+            if 'OK' in response:
+                log.debug('Returned from non-JSON-type request %s: %s', (host, response))
+                # manually fake expected response array
+                return [{'result': 'OK'}]
+            else:
+                log.error('Returned from non-JSON-type request %s: %s', (host, response))
+                # manually fake expected response array
+                return [{'result': 'Error'}]
+
+        except:
+            log.error('Failed sending non-JSON-type request to XBMC: %s', traceback.format_exc())
+            return [{'result': 'Error'}]
+
     def request(self, host, requests):
         server = 'http://%s/jsonrpc' % host
 

couchpotato/core/plugins/automation/main.py

@@ -12,7 +12,7 @@ class Automation(Plugin):
 
         fireEvent('schedule.interval', 'automation.add_movies', self.addMovies, hours = self.conf('hour', default = 12))
 
-        if not Env.get('dev'):
+        if Env.get('dev'):
             addEvent('app.load', self.addMovies)
 
     def addMovies(self):

couchpotato/core/plugins/base.py

@@ -98,6 +98,7 @@ class Plugin(object):
 
     # http request
     def urlopen(self, url, timeout = 30, params = None, headers = None, opener = None, multipart = False, show_error = True):
+        url = ss(url)
 
         if not headers: headers = {}
         if not params: params = {}
@@ -129,8 +130,11 @@ class Plugin(object):
                 log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data'))
                 request = urllib2.Request(url, params, headers)
 
-                cookies = cookielib.CookieJar()
-                opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies), MultipartPostHandler)
+                if opener:
+                    opener.add_handler(MultipartPostHandler())
+                else:
+                    cookies = cookielib.CookieJar()
+                    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies), MultipartPostHandler)
 
                 response = opener.open(request, timeout = timeout)
             else:

couchpotato/core/plugins/renamer/main.py

@@ -33,6 +33,7 @@ class Renamer(Plugin):
         addEvent('renamer.check_snatched', self.checkSnatched)
 
         addEvent('app.load', self.scan)
+        addEvent('app.load', self.checkSnatched)
 
         if self.conf('run_every') > 0:
             fireEvent('schedule.interval', 'renamer.check_snatched', self.checkSnatched, minutes = self.conf('run_every'))

couchpotato/core/plugins/scanner/main.py

@@ -89,7 +89,7 @@ class Scanner(Plugin):
         '()([ab])(\.....?)$' #*a.mkv
     ]
 
-    cp_imdb = '(\.cp\((?P<id>tt[0-9{7}]+)\))'
+    cp_imdb = '(.cp.(?P<id>tt[0-9{7}]+).)'
 
     def __init__(self):
 
@@ -341,7 +341,7 @@ class Scanner(Plugin):
                 group['files']['movie'] = self.getMediaFiles(group['unsorted_files'])
 
             if len(group['files']['movie']) == 0:
-                log.error('Couldn\t find any movie files for %s', identifier)
+                log.error('Couldn\'t find any movie files for %s', identifier)
                 continue
 
             log.debug('Getting metadata for %s', identifier)

couchpotato/core/providers/automation/base.py

@@ -1,13 +1,13 @@
 from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.logger import CPLog
-from couchpotato.core.plugins.base import Plugin
+from couchpotato.core.providers.base import Provider
 from couchpotato.environment import Env
 import time
 
 log = CPLog(__name__)
 
 
-class Automation(Plugin):
+class Automation(Provider):
 
     enabled_option = 'automation_enabled'
 
@@ -19,6 +19,9 @@ class Automation(Plugin):
 
     def _getMovies(self):
 
+        if self.isDisabled():
+            return
+
         if not self.canCheck():
             log.debug('Just checked, skipping %s', self.getName())
             return []

couchpotato/core/providers/automation/bluray/main.py

@@ -1,8 +1,7 @@
 from couchpotato.core.helpers.rss import RSS
-from couchpotato.core.helpers.variable import md5, tryInt
+from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.automation.base import Automation
-import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
 
@@ -14,32 +13,24 @@ class Bluray(Automation, RSS):
 
     def getIMDBids(self):
 
-        if self.isDisabled():
-            return
-
         movies = []
 
-        cache_key = 'bluray.%s' % md5(self.rss_url)
-        rss_data = self.getCache(cache_key, self.rss_url)
-        data = XMLTree.fromstring(rss_data)
+        rss_movies = self.getRSSData(self.rss_url)
 
-        if data is not None:
-            rss_movies = self.getElements(data, 'channel/item')
+        for movie in rss_movies:
+            name = self.getTextElement(movie, 'title').lower().split('blu-ray')[0].strip('(').rstrip()
+            year = self.getTextElement(movie, 'description').split('|')[1].strip('(').strip()
 
-            for movie in rss_movies:
-                name = self.getTextElement(movie, "title").lower().split("blu-ray")[0].strip("(").rstrip()
-                year = self.getTextElement(movie, "description").split("|")[1].strip("(").strip()
+            if not name.find('/') == -1: # make sure it is not a double movie release
+                continue
 
-                if not name.find("/") == -1: # make sure it is not a double movie release
-                    continue
+            if tryInt(year) < self.getMinimal('year'):
+                continue
 
-                if tryInt(year) < self.getMinimal('year'):
-                    continue
+            imdb = self.search(name, year)
 
-                imdb = self.search(name, year)
-
-                if imdb:
-                    if self.isMinimalMovie(imdb):
-                        movies.append(imdb['imdb'])
+            if imdb:
+                if self.isMinimalMovie(imdb):
+                    movies.append(imdb['imdb'])
 
         return movies

couchpotato/core/providers/automation/cp/main.py

@@ -8,7 +8,4 @@ class CP(Automation):
 
     def getMovies(self):
 
-        if self.isDisabled():
-            return
-
         return []

couchpotato/core/providers/automation/imdb/main.py

@@ -1,5 +1,5 @@
 from couchpotato.core.helpers.rss import RSS
-from couchpotato.core.helpers.variable import md5, getImdb, splitString, tryInt
+from couchpotato.core.helpers.variable import getImdb, splitString, tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.automation.base import Automation
 import traceback
@@ -13,9 +13,6 @@ class IMDB(Automation, RSS):
 
     def getIMDBids(self):
 
-        if self.isDisabled():
-            return
-
         movies = []
 
         enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))]
@@ -29,8 +26,7 @@ class IMDB(Automation, RSS):
                 continue
 
             try:
-                cache_key = 'imdb.rss.%s' % md5(url)
-                rss_data = self.getCache(cache_key, url)
+                rss_data = self.getHTMLData(url)
                 imdbs = getImdb(rss_data, multiple = True)
 
                 for imdb in imdbs:

couchpotato/core/providers/automation/itunes/__init__.py

@@ -0,0 +1,35 @@
+from .main import ITunes
+
+def start():
+    return ITunes()
+
+config = [{
+    'name': 'itunes',
+    'groups': [
+        {
+            'tab': 'automation',
+            'name': 'itunes_automation',
+            'label': 'iTunes',
+            'description': 'From any <a href="http://itunes.apple.com/rss">iTunes</a> Store feed. Url should be the RSS link. (uses minimal requirements)',
+            'options': [
+                {
+                    'name': 'automation_enabled',
+                    'default': False,
+                    'type': 'enabler',
+                },
+                {
+                    'name': 'automation_urls_use',
+                    'label': 'Use',
+                    'default': ',',
+                },
+                {
+                    'name': 'automation_urls',
+                    'label': 'url',
+                    'type': 'combined',
+                    'combine': ['automation_urls_use', 'automation_urls'],
+                    'default': 'https://itunes.apple.com/rss/topmovies/limit=25/xml,',
+                },
+            ],
+        },
+    ],
+}]

couchpotato/core/providers/automation/itunes/main.py

@@ -0,0 +1,63 @@
+from couchpotato.core.helpers.rss import RSS
+from couchpotato.core.helpers.variable import md5, splitString, tryInt
+from couchpotato.core.logger import CPLog
+from couchpotato.core.providers.automation.base import Automation
+from xml.etree.ElementTree import QName
+import datetime
+import traceback
+import xml.etree.ElementTree as XMLTree
+
+log = CPLog(__name__)
+
+
+class ITunes(Automation, RSS):
+
+    interval = 1800
+
+    def getIMDBids(self):
+
+        if self.isDisabled():
+            return
+
+        movies = []
+
+        enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))]
+        urls = splitString(self.conf('automation_urls'))
+
+        namespace = 'http://www.w3.org/2005/Atom'
+        namespaceIM = 'http://itunes.apple.com/rss'
+
+        index = -1
+        for url in urls:
+
+            index += 1
+            if not enablers[index]:
+                continue
+
+            try:
+                cache_key = 'itunes.rss.%s' % md5(url)
+                rss_data = self.getCache(cache_key, url)
+
+                data = XMLTree.fromstring(rss_data)
+
+                if data is not None:
+                    entry_tag = str(QName(namespace, 'entry'))
+                    rss_movies = self.getElements(data, entry_tag)
+
+                    for movie in rss_movies:
+                        name_tag = str(QName(namespaceIM, 'name'))
+                        name = self.getTextElement(movie, name_tag)
+
+                        releaseDate_tag = str(QName(namespaceIM, 'releaseDate'))
+                        releaseDateText = self.getTextElement(movie, releaseDate_tag)
+                        year = datetime.datetime.strptime(releaseDateText, '%Y-%m-%dT00:00:00-07:00').strftime("%Y")
+
+                        imdb = self.search(name, year)
+
+                        if imdb and self.isMinimalMovie(imdb):
+                            movies.append(imdb['imdb'])
+
+            except:
+                log.error('Failed loading iTunes rss feed: %s %s', (url, traceback.format_exc()))
+
+        return movies

couchpotato/core/providers/automation/kinepolis/main.py

@@ -1,9 +1,7 @@
 from couchpotato.core.helpers.rss import RSS
-from couchpotato.core.helpers.variable import md5
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.automation.base import Automation
 import datetime
-import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
 
@@ -15,25 +13,17 @@ class Kinepolis(Automation, RSS):
 
     def getIMDBids(self):
 
-        if self.isDisabled():
-            return
-
         movies = []
 
-        cache_key = 'kinepolis.%s' % md5(self.rss_url)
-        rss_data = self.getCache(cache_key, self.rss_url)
-        data = XMLTree.fromstring(rss_data)
+        rss_movies = self.getRSSData(self.rss_url)
 
-        if data is not None:
-            rss_movies = self.getElements(data, 'channel/item')
+        for movie in rss_movies:
+            name = self.getTextElement(movie, 'title')
+            year = datetime.datetime.now().strftime('%Y')
 
-            for movie in rss_movies:
-                name = self.getTextElement(movie, "title")
-                year = datetime.datetime.now().strftime("%Y")
+            imdb = self.search(name, year)
 
-                imdb = self.search(name, year)
-
-                if imdb and self.isMinimalMovie(imdb):
-                    movies.append(imdb['imdb'])
+            if imdb and self.isMinimalMovie(imdb):
+                movies.append(imdb['imdb'])
 
         return movies

couchpotato/core/providers/automation/moviemeter/__init__.py

@@ -0,0 +1,23 @@
+from .main import Moviemeter
+
+def start():
+    return Moviemeter()
+
+config = [{
+    'name': 'moviemeter',
+    'groups': [
+        {
+            'tab': 'automation',
+            'name': 'moviemeter_automation',
+            'label': 'Moviemeter',
+            'description': 'Imports movies from the current top 10 of moviemeter.nl. (uses minimal requirements)',
+            'options': [
+                {
+                    'name': 'automation_enabled',
+                    'default': False,
+                    'type': 'enabler',
+                },
+            ],
+        },
+    ],
+}]

couchpotato/core/providers/automation/moviemeter/main.py

@@ -0,0 +1,28 @@
+from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.rss import RSS
+from couchpotato.core.logger import CPLog
+from couchpotato.core.providers.automation.base import Automation
+
+log = CPLog(__name__)
+
+
+class Moviemeter(Automation, RSS):
+
+    interval = 1800
+    rss_url = 'http://www.moviemeter.nl/rss/cinema'
+
+    def getIMDBids(self):
+
+        movies = []
+
+        rss_movies = self.getRSSData(self.rss_url)
+
+        for movie in rss_movies:
+
+            name_year = fireEvent('scanner.name_year', self.getTextElement(movie, 'title'), single = True)
+            imdb = self.search(name_year.get('name'), name_year.get('year'))
+
+            if imdb and self.isMinimalMovie(imdb):
+                movies.append(imdb['imdb'])
+
+        return movies

couchpotato/core/providers/automation/movies_io/main.py

@@ -1,11 +1,8 @@
 from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.rss import RSS
-from couchpotato.core.helpers.variable import md5
+from couchpotato.core.helpers.variable import tryInt, splitString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.automation.base import Automation
-from xml.etree.ElementTree import ParseError
-import traceback
-import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
 
@@ -16,39 +13,27 @@ class MoviesIO(Automation, RSS):
 
     def getIMDBids(self):
 
-        if self.isDisabled():
-            return
-
         movies = []
 
-        enablers = self.conf('automation_urls_use').split(',')
+        enablers = [tryInt(x) for x in splitString(self.conf('automation_urls_use'))]
 
         index = -1
-        for rss_url in self.conf('automation_urls').split(','):
+        for rss_url in splitString(self.conf('automation_urls')):
 
             index += 1
             if not enablers[index]:
                 continue
 
-            try:
-                cache_key = 'imdb.rss.%s' % md5(rss_url)
+            rss_movies = self.getRSSData(rss_url, headers = {'Referer': ''})
 
-                rss_data = self.getCache(cache_key, rss_url, headers = {'Referer': ''})
-                data = XMLTree.fromstring(rss_data)
-                rss_movies = self.getElements(data, 'channel/item')
+            for movie in rss_movies:
 
-                for movie in rss_movies:
+                nameyear = fireEvent('scanner.name_year', self.getTextElement(movie, 'title'), single = True)
+                imdb = self.search(nameyear.get('name'), nameyear.get('year'), imdb_only = True)
 
-                    nameyear = fireEvent('scanner.name_year', self.getTextElement(movie, "title"), single = True)
-                    imdb = self.search(nameyear.get('name'), nameyear.get('year'), imdb_only = True)
+                if not imdb:
+                    continue
 
-                    if not imdb:
-                        continue
-
-                    movies.append(imdb)
-            except ParseError:
-                log.debug('Failed loading Movies.io watchlist, probably empty: %s', (rss_url))
-            except:
-                log.error('Failed loading Movies.io watchlist: %s %s', (rss_url, traceback.format_exc()))
+                movies.append(imdb)
 
         return movies

couchpotato/core/providers/automation/trakt/main.py

@@ -1,9 +1,8 @@
 from couchpotato.core.event import addEvent
-from couchpotato.core.helpers.variable import md5, sha1
+from couchpotato.core.helpers.variable import sha1
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.automation.base import Automation
 import base64
-import json
 
 log = CPLog(__name__)
 
@@ -25,9 +24,6 @@ class Trakt(Automation):
 
     def getIMDBids(self):
 
-        if self.isDisabled():
-            return
-
         movies = []
         for movie in self.getWatchlist():
             movies.append(movie.get('imdb_id'))
@@ -38,22 +34,11 @@ class Trakt(Automation):
         method = (self.urls['watchlist'] % self.conf('automation_api_key')) + self.conf('automation_username')
         return self.call(method)
 
-
     def call(self, method_url):
 
-        try:
-            if self.conf('automation_password'):
-                headers = {
-                   'Authorization': 'Basic %s' % base64.encodestring('%s:%s' % (self.conf('automation_username'), self.conf('automation_password')))[:-1]
-                }
-            else:
-                headers = {}
+        headers = {}
+        if self.conf('automation_password'):
+            headers['Authorization'] = 'Basic %s' % base64.encodestring('%s:%s' % (self.conf('automation_username'), self.conf('automation_password')))[:-1]
 
-            cache_key = 'trakt.%s' % md5(method_url)
-            json_string = self.getCache(cache_key, self.urls['base'] + method_url, headers = headers)
-            if json_string:
-                return json.loads(json_string)
-        except:
-            log.error('Failed to get data from trakt, check your login.')
-
-        return []
+        data = self.getJsonData(self.urls['base'] + method_url, headers = headers)
+        return data if data else []

couchpotato/core/providers/base.py

@@ -44,6 +44,34 @@ class Provider(Plugin):
 
         return self.is_available.get(host, False)
 
+    def getJsonData(self, url, **kwargs):
+
+        data = self.getCache(md5(url), url, **kwargs)
+
+        if data:
+            try:
+                return json.loads(data)
+            except:
+                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
+
+        return []
+
+    def getRSSData(self, url, **kwargs):
+
+        data = self.getCache(md5(url), url, **kwargs)
+
+        if data:
+            try:
+                data = XMLTree.fromstring(data)
+                return self.getElements(data, 'channel/item')
+            except:
+                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
+
+        return []
+
+    def getHTMLData(self, url, **kwargs):
+        return self.getCache(md5(url), url, **kwargs)
+
 
 class YarrProvider(Provider):
 
@@ -67,15 +95,20 @@ class YarrProvider(Provider):
             urllib2.install_opener(opener)
             log.info2('Logging into %s', self.urls['login'])
             f = opener.open(self.urls['login'], self.getLoginParams())
-            f.read()
+            output = f.read()
             f.close()
-            self.login_opener = opener
-            return True
+
+            if self.loginSuccess(output):
+                self.login_opener = opener
+                return True
         except:
             log.error('Failed to login %s: %s', (self.getName(), traceback.format_exc()))
 
         return False
 
+    def loginSuccess(self, output):
+        return True
+
     def loginDownload(self, url = '', nzb_id = ''):
         try:
             if not self.login_opener and not self.login():
@@ -106,11 +139,11 @@ class YarrProvider(Provider):
             return []
 
         # Create result container
-        imdb_result = hasattr(self, '_search')
-        results = ResultList(self, movie, quality, imdb_result = imdb_result)
+        imdb_results = hasattr(self, '_search')
+        results = ResultList(self, movie, quality, imdb_results = imdb_results)
 
         # Do search based on imdb id
-        if imdb_result:
+        if imdb_results:
             self._search(movie, quality, results)
         # Search possible titles
         else:
@@ -165,34 +198,6 @@ class YarrProvider(Provider):
 
         return [self.cat_backup_id]
 
-    def getJsonData(self, url, **kwargs):
-
-        data = self.getCache(md5(url), url, **kwargs)
-
-        if data:
-            try:
-                return json.loads(data)
-            except:
-                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
-
-        return []
-
-    def getRSSData(self, url, **kwargs):
-
-        data = self.getCache(md5(url), url, **kwargs)
-
-        if data:
-            try:
-                data = XMLTree.fromstring(data)
-                return self.getElements(data, 'channel/item')
-            except:
-                log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
-
-        return []
-
-    def getHTMLData(self, url, **kwargs):
-        return self.getCache(md5(url), url, **kwargs)
-
 
 class ResultList(list):
 

couchpotato/core/providers/movie/imdbapi/__init__.py

@@ -1,6 +0,0 @@
-from .main import IMDBAPI
-
-def start():
-    return IMDBAPI()
-
-config = []

couchpotato/core/providers/movie/omdbapi/__init__.py

@@ -0,0 +1,6 @@
+from .main import OMDBAPI
+
+def start():
+    return OMDBAPI()
+
+config = []

couchpotato/core/providers/movie/omdbapi/main.py

@@ -10,11 +10,11 @@ import traceback
 log = CPLog(__name__)
 
 
-class IMDBAPI(MovieProvider):
+class OMDBAPI(MovieProvider):
 
     urls = {
-        'search': 'http://www.imdbapi.com/?%s',
-        'info': 'http://www.imdbapi.com/?i=%s',
+        'search': 'http://www.omdbapi.com/?%s',
+        'info': 'http://www.omdbapi.com/?i=%s',
     }
 
     http_time_between_calls = 0
@@ -32,7 +32,7 @@ class IMDBAPI(MovieProvider):
                 'name': q
             }
 
-        cache_key = 'imdbapi.cache.%s' % q
+        cache_key = 'omdbapi.cache.%s' % q
         cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}), timeout = 3)
 
         if cached:
@@ -50,7 +50,7 @@ class IMDBAPI(MovieProvider):
         if not identifier:
             return {}
 
-        cache_key = 'imdbapi.cache.%s' % identifier
+        cache_key = 'omdbapi.cache.%s' % identifier
         cached = self.getCache(cache_key, self.urls['info'] % identifier, timeout = 3)
 
         if cached:

couchpotato/core/providers/nzb/ftdworld/main.py

@@ -1,12 +1,10 @@
-from bs4 import BeautifulSoup
 from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
-import re
-import time
+import traceback
 
 log = CPLog(__name__)
 
@@ -14,7 +12,7 @@ log = CPLog(__name__)
 class FTDWorld(NZBProvider):
 
     urls = {
-        'search': 'http://ftdworld.net/category.php?%s',
+        'search': 'http://ftdworld.net/api/index.php?%s',
         'detail': 'http://ftdworld.net/spotinfo.php?id=%s',
         'download': 'http://ftdworld.net/cgi-bin/nzbdown.pl?fileID=%s',
         'login': 'http://ftdworld.net/index.php',
@@ -25,7 +23,7 @@ class FTDWorld(NZBProvider):
     cat_ids = [
         ([4, 11], ['dvdr']),
         ([1], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr', 'brrip']),
-        ([10, 13, 14], ['bd50', '720p', '1080p']),
+        ([7, 10, 13, 14], ['bd50', '720p', '1080p']),
     ]
     cat_backup_id = 1
 
@@ -43,38 +41,29 @@ class FTDWorld(NZBProvider):
             'ctype': ','.join([str(x) for x in self.getCatId(quality['identifier'])]),
         })
 
-        data = self.getHTMLData(self.urls['search'] % params, opener = self.login_opener)
+        data = self.getJsonData(self.urls['search'] % params, opener = self.login_opener)
 
         if data:
             try:
 
-                html = BeautifulSoup(data)
-                main_table = html.find('table', attrs = {'id':'ftdresult'})
-
-                if not main_table:
+                if data.get('numRes') == 0:
                     return
 
-                items = main_table.find_all('tr', attrs = {'class': re.compile('tcontent')})
-
-                for item in items:
-                    tds = item.find_all('td')
-                    nzb_id = tryInt(item.attrs['data-spot'])
-
-                    up = item.find('img', attrs = {'src': re.compile('up.png')})
-                    down = item.find('img', attrs = {'src': re.compile('down.png')})
+                for item in data.get('data'):
 
+                    nzb_id = tryInt(item.get('id'))
                     results.append({
                         'id': nzb_id,
-                        'name': toUnicode(item.find('a', attrs = {'href': re.compile('./spotinfo')}).text.strip()),
-                        'age': self.calculateAge(int(time.mktime(parse(tds[2].text).timetuple()))),
+                        'name': toUnicode(item.get('Title')),
+                        'age': self.calculateAge(tryInt(item.get('Created'))),
                         'url': self.urls['download'] % nzb_id,
                         'download': self.loginDownload,
                         'detail_url': self.urls['detail'] % nzb_id,
-                        'score': (tryInt(up.attrs['title'].split(' ')[0]) * 3) - (tryInt(down.attrs['title'].split(' ')[0]) * 3) if up else 0,
+                        'score': (tryInt(item.get('webPlus', 0)) - tryInt(item.get('webMin', 0))) * 3,
                     })
 
             except:
-                log.error('Failed to parse HTML response from FTDWorld')
+                log.error('Failed to parse HTML response from FTDWorld: %s', traceback.format_exc())
 
     def getLoginParams(self):
         return tryUrlencode({
@@ -82,3 +71,6 @@ class FTDWorld(NZBProvider):
             'passlogin': self.conf('password'),
             'submit': 'Log In',
         })
+
+    def loginSuccess(self, output):
+        return 'password is incorrect' not in output

couchpotato/core/providers/nzb/newznab/__init__.py

@@ -13,7 +13,7 @@ config = [{
             'order': 10,
             'description': 'Enable <a href="http://newznab.com/" target="_blank">NewzNab providers</a> such as <a href="https://nzb.su" target="_blank">NZB.su</a>, \
                 <a href="https://nzbs.org" target="_blank">NZBs.org</a>, <a href="http://dognzb.cr/" target="_blank">DOGnzb.cr</a>, \
-                <a href="https://github.com/spotweb/spotweb" target="_blank">Spotweb</a>',
+                <a href="https://github.com/spotweb/spotweb" target="_blank">Spotweb</a> or <a href="https://nzbgeek.info/" target="_blank">NZBGeek</a>',
             'wizard': True,
             'options': [
                 {
@@ -22,16 +22,16 @@ config = [{
                 },
                 {
                     'name': 'use',
-                    'default': '0,0,0'
+                    'default': '0,0,0,0'
                 },
                 {
                     'name': 'host',
-                    'default': 'nzb.su,dognzb.cr,nzbs.org',
+                    'default': 'nzb.su,dognzb.cr,nzbs.org,https://index.nzbgeek.info',
                     'description': 'The hostname of your newznab provider',
                 },
                 {
                     'name': 'api_key',
-                    'default': ',,',
+                    'default': ',,,',
                     'label': 'Api Key',
                     'description': 'Can be found on your profile page',
                     'type': 'combined',

couchpotato/core/providers/nzb/newznab/main.py

@@ -29,7 +29,7 @@ class Newznab(NZBProvider, RSS):
     def search(self, movie, quality):
         hosts = self.getHosts()
 
-        results = ResultList(self, movie, quality, imdb_result = True)
+        results = ResultList(self, movie, quality, imdb_results = True)
 
         for host in hosts:
             if self.isDisabled(host):
@@ -73,8 +73,8 @@ class Newznab(NZBProvider, RSS):
                 'name': self.getTextElement(nzb, 'title'),
                 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
                 'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024,
-                'url': (self.getUrl(host['host'], self.urls['download']) % nzb_id) + self.getApiExt(host),
-                'detail_url': '%sdetails/%s' % (cleanHost(host['host']), nzb_id),
+                'url': (self.getUrl(host['host'], self.urls['download']) % tryUrlencode(nzb_id)) + self.getApiExt(host),
+                'detail_url': '%sdetails/%s' % (cleanHost(host['host']), tryUrlencode(nzb_id)),
                 'content': self.getTextElement(nzb, 'description'),
             })
 
@@ -136,6 +136,6 @@ class Newznab(NZBProvider, RSS):
                     self.limits_reached[host] = time.time()
                     return 'try_next'
 
-            log.error('Failed download from %s', (host, traceback.format_exc()))
+            log.error('Failed download from %s: %s', (host, traceback.format_exc()))
 
         return 'try_next'

couchpotato/core/providers/nzb/nzbx/__init__.py

@@ -10,7 +10,7 @@ config = [{
             'tab': 'searcher',
             'subtab': 'nzb_providers',
             'name': 'nzbX',
-            'description': 'Free provider, less accurate. See <a href="https://www.nzbx.co/">nzbX</a>',
+            'description': 'Free provider. See <a href="https://www.nzbx.co/">nzbX</a>',
             'options': [
                 {
                     'name': 'enabled',

couchpotato/core/providers/nzb/nzbx/main.py

@@ -2,6 +2,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.nzb.base import NZBProvider
+from couchpotato.environment import Env
 
 log = CPLog(__name__)
 
@@ -22,7 +23,7 @@ class Nzbx(NZBProvider):
             'q': movie['library']['identifier'].replace('tt', ''),
             'sf': quality.get('size_min'),
         })
-        nzbs = self.getJsonData(self.urls['search'] % arguments)
+        nzbs = self.getJsonData(self.urls['search'] % arguments, headers = {'User-Agent': Env.getIdentifier()})
 
         for nzb in nzbs:
 

couchpotato/core/providers/nzb/omgwtfnzbs/main.py

@@ -14,7 +14,7 @@ log = CPLog(__name__)
 class OMGWTFNZBs(NZBProvider, RSS):
 
     urls = {
-        'search': 'http://rss.omgwtfnzbs.com/rss-search.php?%s',
+        'search': 'http://rss.omgwtfnzbs.org/rss-search.php?%s',
     }
 
     http_time_between_calls = 1 #seconds

couchpotato/runner.py

@@ -4,7 +4,6 @@ from couchpotato.api import api, NonBlockHandler
 from couchpotato.core.event import fireEventAsync, fireEvent
 from couchpotato.core.helpers.variable import getDataDir, tryInt
 from logging import handlers
-from tornado.ioloop import IOLoop
 from tornado.web import Application, FallbackHandler
 from tornado.wsgi import WSGIContainer
 from werkzeug.contrib.cache import FileSystemCache
@@ -231,6 +230,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
     fireEventAsync('app.load')
 
     # Go go go!
+    from tornado.ioloop import IOLoop
     web_container = WSGIContainer(app)
     web_container._log = _log
     loop = IOLoop.instance()

couchpotato/static/scripts/api.js

@@ -13,7 +13,7 @@ var ApiClass = new Class({
 		return new Request[r_type](Object.merge({
 			'callbackKey': 'callback_func',
 			'method': 'get',
-			'url': self.createUrl(type),
+			'url': self.createUrl(type, {'t': randomString()}),
 		}, options)).send()
 	},
 

init/freebsd

@@ -1,89 +1,49 @@
 #!/bin/sh
 #
 # PROVIDE: couchpotato
-# REQUIRE: sabnzbd
+# REQUIRE: DAEMON
 # KEYWORD: shutdown
-#
-# Add the following lines to /etc/rc.conf.local or /etc/rc.conf
-# to enable this service:
-#
-# couchpotato_enable (bool):	Set to NO by default.
-#			Set it to YES to enable it.
-# couchpotato_user:  The user account CouchPotato daemon runs as what
-#			you want it to be. It uses '_sabnzbd' user by
-#			default. Do not sets it as empty or it will run
-#			as root.
-# couchpotato_dir:	Directory where CouchPotato lives.
-#			Default: /usr/local/couchpotato
-# couchpotato_chdir:  Change to this directory before running CouchPotato.
-#     Default is same as couchpotato_dir.
-# couchpotato_pid:  The name of the pidfile to create.
-#     Default is couchpotato.pid in couchpotato_dir.
+
+# Add the following lines to /etc/rc.conf to enable couchpotato:
+# couchpotato_enable:   Set to NO by default. Set it to YES to enable it.
+# couchpotato_user:     The user account CouchPotato daemon runs as what
+#                       you want it to be.
+# couchpotato_dir:      Directory where CouchPotato lives.
+#                           Default: /usr/local/CouchPotatoServer
+# couchpotato_datadir:  Directory where CouchPotato user data lives.
+#                           Default: $couchpotato_dir/data
+# couchpotato_conf:     Directory where CouchPotato user data lives.
+#                           Default: $couchpotato_datadir/settings.conf
+# couchpotato_pid:      Full path to PID file.
+#                           Default: $couchpotato_datadir/couchpotato.pid
+# couchpotato_flags:    Set additonal flags as needed.
 
 . /etc/rc.subr
 
 name="couchpotato"
-rcvar=${name}_enable
+rcvar=couchpotato_enable
 
 load_rc_config ${name}
 
-: ${couchpotato_enable:="NO"}
-: ${couchpotato_user:="_sabnzbd"}
-: ${couchpotato_dir:="/usr/local/couchpotato"}
-: ${couchpotato_chdir:="${couchpotato_dir}"}
-: ${couchpotato_pid:="${couchpotato_dir}/couchpotato.pid"}
-: ${couchpotato_conf:="${couchpotato_dir}/data/settings.conf"}
+: ${couchpotato_enable:=NO}
+: ${couchpotato_user:=} #default is root
+: ${couchpotato_dir:=/usr/local/CouchPotatoServer}
+: ${couchpotato_datadir:=${couchpotato_dir}/data}
+: ${couchpotato_conf:=} #default is datadir/settings.conf
+: ${couchpotato_pid:=} #default is datadir/couchpotato.pid
+: ${couchpotato_flags:=}
 
-WGET="/usr/local/bin/wget" # You need wget for this script to safely shutdown CouchPotato.
-if [ -e "${couchpotato_conf}" ]; then
-        HOST=`grep -A14 "\[core\]" "${couchpotato_conf}"|egrep "^host"|perl -wple 's/^host = (.*)$/$1/'`
-        PORT=`grep -A14 "\[core\]" "${couchpotato_conf}"|egrep "^port"|perl -wple 's/^port = (.*)$/$1/'`
-        CPAPI=`grep -A14 "\[core\]" "${couchpotato_conf}"|egrep "^api_key"|perl -wple 's/^api_key = (.*)$/$1/'`
+command="${couchpotato_dir}/CouchPotato.py"
+command_interpreter="/usr/local/bin/python"
+command_args="--daemon --data_dir ${couchpotato_datadir}"
+
+# append optional flags
+if [ -n "${couchpotato_pid}" ]; then
+    pidfile=${couchpotato_pid}
+    couchpotato_flags="${couchpotato_flags} --pid_file ${couchpotato_pid}"
 fi
-
-status_cmd="${name}_status"
-stop_cmd="${name}_stop"
-
-command="/usr/sbin/daemon"
-command_args="-f -p ${couchpotato_pid} python ${couchpotato_dir}/CouchPotato.py ${couchpotato_flags}"
-
-# Check for wget and refuse to start without it.
-if [ ! -x "${WGET}" ]; then
-  warn "couchpotato not started: You need wget to safely shut down CouchPotato."
-  exit 1
+if [ -n "${couchpotato_conf}" ]; then
+    couchpotato_flags="${couchpotato_flags} --config_file ${couchpotato_conf}"
 fi
 
-# Ensure user is root when running this script.
-if [ `id -u` != "0" ]; then
-  echo "Oops, you should be root before running this!"
-  exit 1
-fi
-
-verify_couchpotato_pid() {
-    # Make sure the pid corresponds to the CouchPotato process.
-    pid=`cat ${couchpotato_pid} 2>/dev/null`
-    ps -p ${pid} | grep -q "python ${couchpotato_dir}/CouchPotato.py"
-    return $?
-}
-
-# Try to stop CouchPotato cleanly by calling shutdown over http.
-couchpotato_stop() {
-    if [ ! -e "${couchpotato_conf}" ]; then
-        echo "CouchPotato's settings file does not exist. Try starting CouchPotato, as this should create the file."
-        exit 1
-    fi
-    echo "Stopping $name"
-    verify_couchpotato_pid
-    ${WGET} -O - -q "http://${HOST}:${PORT}/api/${CPAPI}/app.shutdown/" >/dev/null
-    if [ -n "${pid}" ]; then
-      wait_for_pids ${pid}
-      echo "Stopped"
-    fi
-}
-
-couchpotato_status() {
-    verify_couchpotato_pid && echo "$name is running as ${pid}" || echo "$name is not running"
-}
-
 run_rc_command "$1"
-

installer.iss

@@ -1,5 +1,5 @@
 #define MyAppName "CouchPotato"
-#define MyAppVer "2.0.4"
+#define MyAppVer "2.0.5"
 
 [Setup]
 AppName={#MyAppName}

libs/gntp/__init__.py

@@ -1,8 +1,9 @@
-import hashlib
 import re
+import hashlib
 import time
+import StringIO
 
-__version__ = '0.6'
+__version__ = '0.8'
 
 #GNTP/<version> <messagetype> <encryptionAlgorithmID>[:<ivValue>][ <keyHashAlgorithmID>:<keyHash>.<salt>]
 GNTP_INFO_LINE = re.compile(
@@ -19,7 +20,7 @@ GNTP_INFO_LINE_SHORT = re.compile(
 
 GNTP_HEADER = re.compile('([\w-]+):(.+)')
 
-GNTP_EOL = u'\r\n'
+GNTP_EOL = '\r\n'
 
 
 class BaseError(Exception):
@@ -43,6 +44,14 @@ class UnsupportedError(BaseError):
 	errordesc = 'Currently unsupported by gntp.py'
 
 
+class _GNTPBuffer(StringIO.StringIO):
+	"""GNTP Buffer class"""
+	def writefmt(self, message = "", *args):
+		"""Shortcut function for writing GNTP Headers"""
+		self.write((message % args).encode('utf8', 'replace'))
+		self.write(GNTP_EOL)
+
+
 class _GNTPBase(object):
 	"""Base initilization
 
@@ -206,8 +215,8 @@ class _GNTPBase(object):
 			if not match:
 				continue
 
-			key = match.group(1).strip()
-			val = match.group(2).strip()
+			key = unicode(match.group(1).strip(), 'utf8', 'replace')
+			val = unicode(match.group(2).strip(), 'utf8', 'replace')
 			dict[key] = val
 		return dict
 
@@ -217,6 +226,15 @@ class _GNTPBase(object):
 		else:
 			self.headers[key] = unicode('%s' % value, 'utf8', 'replace')
 
+	def add_resource(self, data):
+		"""Add binary resource
+
+		:param string data: Binary Data
+		"""
+		identifier = hashlib.md5(data).hexdigest()
+		self.resources[identifier] = data
+		return 'x-growl-resource://%s' % identifier
+
 	def decode(self, data, password = None):
 		"""Decode GNTP Message
 
@@ -229,19 +247,30 @@ class _GNTPBase(object):
 		self.headers = self._parse_dict(parts[0])
 
 	def encode(self):
-		"""Encode a GNTP Message
+		"""Encode a generic GNTP Message
 
-		:return string: Encoded GNTP Message ready to be sent
+		:return string: GNTP Message ready to be sent
 		"""
-		self.validate()
 
-		message = self._format_info() + GNTP_EOL
+		buffer = _GNTPBuffer()
+
+		buffer.writefmt(self._format_info())
+
 		#Headers
 		for k, v in self.headers.iteritems():
-			message += u'%s: %s%s' % (k, v, GNTP_EOL)
+			buffer.writefmt('%s: %s', k, v)
+		buffer.writefmt()
 
-		message += GNTP_EOL
-		return message
+		#Resources
+		for resource, data in self.resources.iteritems():
+			buffer.writefmt('Identifier: %s', resource)
+			buffer.writefmt('Length: %d', len(data))
+			buffer.writefmt()
+			buffer.write(data)
+			buffer.writefmt()
+			buffer.writefmt()
+
+		return buffer.getvalue()
 
 
 class GNTPRegister(_GNTPBase):
@@ -290,7 +319,7 @@ class GNTPRegister(_GNTPBase):
 
 		for i, part in enumerate(parts):
 			if i == 0:
-				continue  # Skip Header
+				continue # Skip Header
 			if part.strip() == '':
 				continue
 			notice = self._parse_dict(part)
@@ -319,22 +348,33 @@ class GNTPRegister(_GNTPBase):
 
 		:return string: Encoded GNTP Registration message
 		"""
-		self.validate()
 
-		message = self._format_info() + GNTP_EOL
+		buffer = _GNTPBuffer()
+
+		buffer.writefmt(self._format_info())
+
 		#Headers
 		for k, v in self.headers.iteritems():
-			message += u'%s: %s%s' % (k, v, GNTP_EOL)
+			buffer.writefmt('%s: %s', k, v)
+		buffer.writefmt()
 
 		#Notifications
 		if len(self.notifications) > 0:
 			for notice in self.notifications:
-				message += GNTP_EOL
 				for k, v in notice.iteritems():
-					message += u'%s: %s%s' % (k, v, GNTP_EOL)
+					buffer.writefmt('%s: %s', k, v)
+				buffer.writefmt()
 
-		message += GNTP_EOL
-		return message
+		#Resources
+		for resource, data in self.resources.iteritems():
+			buffer.writefmt('Identifier: %s', resource)
+			buffer.writefmt('Length: %d', len(data))
+			buffer.writefmt()
+			buffer.write(data)
+			buffer.writefmt()
+			buffer.writefmt()
+
+		return buffer.getvalue()
 
 
 class GNTPNotice(_GNTPBase):
@@ -379,7 +419,7 @@ class GNTPNotice(_GNTPBase):
 
 		for i, part in enumerate(parts):
 			if i == 0:
-				continue  # Skip Header
+				continue # Skip Header
 			if part.strip() == '':
 				continue
 			notice = self._parse_dict(part)
@@ -388,21 +428,6 @@ class GNTPNotice(_GNTPBase):
 				#open('notice.png','wblol').write(notice['Data'])
 				self.resources[notice.get('Identifier')] = notice
 
-	def encode(self):
-		"""Encode a GNTP Notification Message
-
-		:return string: GNTP Notification Message ready to be sent
-		"""
-		self.validate()
-
-		message = self._format_info() + GNTP_EOL
-		#Headers
-		for k, v in self.headers.iteritems():
-			message += u'%s: %s%s' % (k, v, GNTP_EOL)
-
-		message += GNTP_EOL
-		return message
-
 
 class GNTPSubscribe(_GNTPBase):
 	"""Represents a GNTP Subscribe Command
@@ -457,7 +482,8 @@ class GNTPError(_GNTPBase):
 			self.add_header('Error-Description', errordesc)
 
 	def error(self):
-		return self.headers['Error-Code'], self.headers['Error-Description']
+		return (self.headers.get('Error-Code', None),
+			self.headers.get('Error-Description', None))
 
 
 def parse_gntp(data, password = None):

libs/gntp/notifier.py

@@ -22,43 +22,6 @@ __all__ = [
 logger = logging.getLogger(__name__)
 
 
-def mini(description, applicationName = 'PythonMini', noteType = "Message",
-			title = "Mini Message", applicationIcon = None, hostname = 'localhost',
-			password = None, port = 23053, sticky = False, priority = None,
-			callback = None):
-	"""Single notification function
-
-	Simple notification function in one line. Has only one required parameter
-	and attempts to use reasonable defaults for everything else
-	:param string description: Notification message
-
-	.. warning::
-			For now, only URL callbacks are supported. In the future, the
-			callback argument will also support a function
-	"""
-	growl = GrowlNotifier(
-		applicationName = applicationName,
-		notifications = [noteType],
-		defaultNotifications = [noteType],
-		hostname = hostname,
-		password = password,
-		port = port,
-	)
-	result = growl.register()
-	if result is not True:
-		return result
-
-	return growl.notify(
-		noteType = noteType,
-		title = title,
-		description = description,
-		icon = applicationIcon,
-		sticky = sticky,
-		priority = priority,
-		callback = callback,
-	)
-
-
 class GrowlNotifier(object):
 	"""Helper class to simplfy sending Growl messages
 
@@ -93,10 +56,12 @@ class GrowlNotifier(object):
 	def _checkIcon(self, data):
 		'''
 		Check the icon to see if it's valid
-		@param data:
-		@todo Consider checking for a valid URL
+
+		If it's a simple URL icon, then we return True. If it's a data icon
+		then we return False
 		'''
-		return data
+		logger.info('Checking icon')
+		return data.startswith('http')
 
 	def register(self):
 		"""Send GNTP Registration
@@ -112,7 +77,11 @@ class GrowlNotifier(object):
 			enabled = notification in self.defaultNotifications
 			register.add_notification(notification, enabled)
 		if self.applicationIcon:
-			register.add_header('Application-Icon', self.applicationIcon)
+			if self._checkIcon(self.applicationIcon):
+				register.add_header('Application-Icon', self.applicationIcon)
+			else:
+				id = register.add_resource(self.applicationIcon)
+				register.add_header('Application-Icon', id)
 		if self.password:
 			register.set_password(self.password, self.passwordHash)
 		self.add_origin_info(register)
@@ -120,7 +89,7 @@ class GrowlNotifier(object):
 		return self._send('register', register)
 
 	def notify(self, noteType, title, description, icon = None, sticky = False,
-			priority = None, callback = None):
+			priority = None, callback = None, identifier = None):
 		"""Send a GNTP notifications
 
 		.. warning::
@@ -151,11 +120,18 @@ class GrowlNotifier(object):
 		if priority:
 			notice.add_header('Notification-Priority', priority)
 		if icon:
-			notice.add_header('Notification-Icon', self._checkIcon(icon))
+			if self._checkIcon(icon):
+				notice.add_header('Notification-Icon', icon)
+			else:
+				id = notice.add_resource(icon)
+				notice.add_header('Notification-Icon', id)
+
 		if description:
 			notice.add_header('Notification-Text', description)
 		if callback:
 			notice.add_header('Notification-Callback-Target', callback)
+		if identifier:
+			notice.add_header('Notification-Coalescing-ID', identifier)
 
 		self.add_origin_info(notice)
 		self.notify_hook(notice)
@@ -193,9 +169,10 @@ class GrowlNotifier(object):
 	def subscribe_hook(self, packet):
 		pass
 
-	def _send(self, type, packet):
+	def _send(self, messagetype, packet):
 		"""Send the GNTP Packet"""
 
+		packet.validate()
 		data = packet.encode()
 
 		logger.debug('To : %s:%s <%s>\n%s', self.hostname, self.port, packet.__class__, data)
@@ -203,7 +180,7 @@ class GrowlNotifier(object):
 		s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 		s.settimeout(self.socketTimeout)
 		s.connect((self.hostname, self.port))
-		s.send(data.encode('utf8', 'replace'))
+		s.send(data)
 		recv_data = s.recv(1024)
 		while not recv_data.endswith("\r\n\r\n"):
 			recv_data += s.recv(1024)
@@ -212,11 +189,51 @@ class GrowlNotifier(object):
 
 		logger.debug('From : %s:%s <%s>\n%s', self.hostname, self.port, response.__class__, response)
 
-		if response.info['messagetype'] == '-OK':
+		if type(response) == gntp.GNTPOK:
 			return True
 		logger.error('Invalid response: %s', response.error())
 		return response.error()
 
+
+def mini(description, applicationName = 'PythonMini', noteType = "Message",
+			title = "Mini Message", applicationIcon = None, hostname = 'localhost',
+			password = None, port = 23053, sticky = False, priority = None,
+			callback = None, notificationIcon = None, identifier = None,
+			notifierFactory = GrowlNotifier):
+	"""Single notification function
+
+	Simple notification function in one line. Has only one required parameter
+	and attempts to use reasonable defaults for everything else
+	:param string description: Notification message
+
+	.. warning::
+			For now, only URL callbacks are supported. In the future, the
+			callback argument will also support a function
+	"""
+	growl = notifierFactory(
+		applicationName = applicationName,
+		notifications = [noteType],
+		defaultNotifications = [noteType],
+		applicationIcon = applicationIcon,
+		hostname = hostname,
+		password = password,
+		port = port,
+	)
+	result = growl.register()
+	if result is not True:
+		return result
+
+	return growl.notify(
+		noteType = noteType,
+		title = title,
+		description = description,
+		icon = notificationIcon,
+		sticky = sticky,
+		priority = priority,
+		callback = callback,
+		identifier = identifier,
+	)
+
 if __name__ == '__main__':
 	# If we're running this module directly we're likely running it as a test
 	# so extra debugging is useful

libs/tornado/curl_httpclient.py

@@ -96,17 +96,18 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
             pycurl.POLL_INOUT: ioloop.IOLoop.READ | ioloop.IOLoop.WRITE
         }
         if event == pycurl.POLL_REMOVE:
-            self.io_loop.remove_handler(fd)
-            del self._fds[fd]
+            if fd in self._fds:
+                self.io_loop.remove_handler(fd)
+                del self._fds[fd]
         else:
             ioloop_event = event_map[event]
             if fd not in self._fds:
-                self._fds[fd] = ioloop_event
                 self.io_loop.add_handler(fd, self._handle_events,
                                          ioloop_event)
-            else:
                 self._fds[fd] = ioloop_event
+            else:
                 self.io_loop.update_handler(fd, ioloop_event)
+                self._fds[fd] = ioloop_event
 
     def _set_timeout(self, msecs):
         """Called by libcurl to schedule a timeout."""

libs/tornado/ioloop.py

@@ -194,7 +194,7 @@ class IOLoop(Configurable):
     def initialize(self):
         pass
 
-    def close(self, all_fds=False):
+    def close(self, all_fds = False):
         """Closes the IOLoop, freeing any resources used.
 
         If ``all_fds`` is true, all file descriptors registered on the
@@ -320,7 +320,7 @@ class IOLoop(Configurable):
         """
         raise NotImplementedError()
 
-    def add_callback(self, callback):
+    def add_callback(self, callback, *args, **kwargs):
         """Calls the given callback on the next I/O loop iteration.
 
         It is safe to call this method from any thread at any time,
@@ -335,7 +335,7 @@ class IOLoop(Configurable):
         """
         raise NotImplementedError()
 
-    def add_callback_from_signal(self, callback):
+    def add_callback_from_signal(self, callback, *args, **kwargs):
         """Calls the given callback on the next I/O loop iteration.
 
         Safe for use from a Python signal handler; should not be used
@@ -359,8 +359,7 @@ class IOLoop(Configurable):
         assert isinstance(future, IOLoop._FUTURE_TYPES)
         callback = stack_context.wrap(callback)
         future.add_done_callback(
-            lambda future: self.add_callback(
-                functools.partial(callback, future)))
+            lambda future: self.add_callback(callback, future))
 
     def _run_callback(self, callback):
         """Runs a callback with error handling.
@@ -382,7 +381,7 @@ class IOLoop(Configurable):
         The exception itself is not passed explicitly, but is available
         in sys.exc_info.
         """
-        app_log.error("Exception in callback %r", callback, exc_info=True)
+        app_log.error("Exception in callback %r", callback, exc_info = True)
 
 
 
@@ -393,7 +392,7 @@ class PollIOLoop(IOLoop):
     (Linux), `tornado.platform.kqueue.KQueueIOLoop` (BSD and Mac), or
     `tornado.platform.select.SelectIOLoop` (all platforms).
     """
-    def initialize(self, impl, time_func=None):
+    def initialize(self, impl, time_func = None):
         super(PollIOLoop, self).initialize()
         self._impl = impl
         if hasattr(self._impl, 'fileno'):
@@ -417,7 +416,7 @@ class PollIOLoop(IOLoop):
                          lambda fd, events: self._waker.consume(),
                          self.READ)
 
-    def close(self, all_fds=False):
+    def close(self, all_fds = False):
         with self._callback_lock:
             self._closing = True
         self.remove_handler(self._waker.fileno())
@@ -426,7 +425,7 @@ class PollIOLoop(IOLoop):
                 try:
                     os.close(fd)
                 except Exception:
-                    gen_log.debug("error closing fd %s", fd, exc_info=True)
+                    gen_log.debug("error closing fd %s", fd, exc_info = True)
         self._waker.close()
         self._impl.close()
 
@@ -442,8 +441,8 @@ class PollIOLoop(IOLoop):
         self._events.pop(fd, None)
         try:
             self._impl.unregister(fd)
-        except (OSError, IOError):
-            gen_log.debug("Error deleting fd from IOLoop", exc_info=True)
+        except Exception:
+            gen_log.debug("Error deleting fd from IOLoop", exc_info = True)
 
     def set_blocking_signal_threshold(self, seconds, action):
         if not hasattr(signal, "setitimer"):
@@ -501,7 +500,7 @@ class PollIOLoop(IOLoop):
                     # IOLoop is just started once at the beginning.
                     signal.set_wakeup_fd(old_wakeup_fd)
                     old_wakeup_fd = None
-            except ValueError:  # non-main thread
+            except ValueError: # non-main thread
                 pass
 
         while True:
@@ -569,17 +568,18 @@ class PollIOLoop(IOLoop):
             while self._events:
                 fd, events = self._events.popitem()
                 try:
-                    self._handlers[fd](fd, events)
+                    hdlr = self._handlers.get(fd)
+                    if hdlr: hdlr(fd, events)
                 except (OSError, IOError), e:
                     if e.args[0] == errno.EPIPE:
                         # Happens when the client closes the connection
                         pass
                     else:
                         app_log.error("Exception in I/O handler for fd %s",
-                                      fd, exc_info=True)
+                                      fd, exc_info = True)
                 except Exception:
                     app_log.error("Exception in I/O handler for fd %s",
-                                  fd, exc_info=True)
+                                  fd, exc_info = True)
         # reset the stopped flag so another start/stop pair can be issued
         self._stopped = False
         if self._blocking_signal_threshold is not None:
@@ -609,12 +609,13 @@ class PollIOLoop(IOLoop):
         # collection pass whenever there are too many dead timeouts.
         timeout.callback = None
 
-    def add_callback(self, callback):
+    def add_callback(self, callback, *args, **kwargs):
         with self._callback_lock:
             if self._closing:
                 raise RuntimeError("IOLoop is closing")
             list_empty = not self._callbacks
-            self._callbacks.append(stack_context.wrap(callback))
+            self._callbacks.append(functools.partial(
+                    stack_context.wrap(callback), *args, **kwargs))
         if list_empty and thread.get_ident() != self._thread_ident:
             # If we're in the IOLoop's thread, we know it's not currently
             # polling.  If we're not, and we added the first callback to an
@@ -624,12 +625,12 @@ class PollIOLoop(IOLoop):
             # avoid it when we can.
             self._waker.wake()
 
-    def add_callback_from_signal(self, callback):
+    def add_callback_from_signal(self, callback, *args, **kwargs):
         with stack_context.NullContext():
             if thread.get_ident() != self._thread_ident:
                 # if the signal is handled on another thread, we can add
                 # it normally (modulo the NullContext)
-                self.add_callback(callback)
+                self.add_callback(callback, *args, **kwargs)
             else:
                 # If we're on the IOLoop's thread, we cannot use
                 # the regular add_callback because it may deadlock on
@@ -639,7 +640,8 @@ class PollIOLoop(IOLoop):
                 # _callback_lock block in IOLoop.start, we may modify
                 # either the old or new version of self._callbacks,
                 # but either way will work.
-                self._callbacks.append(stack_context.wrap(callback))
+                self._callbacks.append(functools.partial(
+                        stack_context.wrap(callback), *args, **kwargs))
 
 
 class _Timeout(object):
@@ -682,7 +684,7 @@ class PeriodicCallback(object):
 
     `start` must be called after the PeriodicCallback is created.
     """
-    def __init__(self, callback, callback_time, io_loop=None):
+    def __init__(self, callback, callback_time, io_loop = None):
         self.callback = callback
         if callback_time <= 0:
             raise ValueError("Periodic callback must have a positive callback_time")
@@ -710,7 +712,7 @@ class PeriodicCallback(object):
         try:
             self.callback()
         except Exception:
-            app_log.error("Error in periodic callback", exc_info=True)
+            app_log.error("Error in periodic callback", exc_info = True)
         self._schedule_next()
 
     def _schedule_next(self):

libs/tornado/iostream.py

@@ -209,11 +209,19 @@ class BaseIOStream(object):
         """Call the given callback when the stream is closed."""
         self._close_callback = stack_context.wrap(callback)
 
-    def close(self):
-        """Close this stream."""
+    def close(self, exc_info=False):
+        """Close this stream.
+
+        If ``exc_info`` is true, set the ``error`` attribute to the current
+        exception from `sys.exc_info()` (or if ``exc_info`` is a tuple,
+        use that instead of `sys.exc_info`).
+        """
         if not self.closed():
-            if any(sys.exc_info()):
-                self.error = sys.exc_info()[1]
+            if exc_info:
+                if not isinstance(exc_info, tuple):
+                    exc_info = sys.exc_info()
+                if any(exc_info):
+                    self.error = exc_info[1]
             if self._read_until_close:
                 callback = self._read_callback
                 self._read_callback = None
@@ -285,7 +293,7 @@ class BaseIOStream(object):
         except Exception:
             gen_log.error("Uncaught exception, closing connection.",
                           exc_info=True)
-            self.close()
+            self.close(exc_info=True)
             raise
 
     def _run_callback(self, callback, *args):
@@ -300,7 +308,7 @@ class BaseIOStream(object):
                 # (It would eventually get closed when the socket object is
                 # gc'd, but we don't want to rely on gc happening before we
                 # run out of file descriptors)
-                self.close()
+                self.close(exc_info=True)
                 # Re-raise the exception so that IOLoop.handle_callback_exception
                 # can see it and log the error
                 raise
@@ -348,7 +356,7 @@ class BaseIOStream(object):
                 self._pending_callbacks -= 1
         except Exception:
             gen_log.warning("error on read", exc_info=True)
-            self.close()
+            self.close(exc_info=True)
             return
         if self._read_from_buffer():
             return
@@ -397,9 +405,9 @@ class BaseIOStream(object):
                 # Treat ECONNRESET as a connection close rather than
                 # an error to minimize log spam  (the exception will
                 # be available on self.error for apps that care).
-                self.close()
+                self.close(exc_info=True)
                 return
-            self.close()
+            self.close(exc_info=True)
             raise
         if chunk is None:
             return 0
@@ -503,7 +511,7 @@ class BaseIOStream(object):
                 else:
                     gen_log.warning("Write error on %d: %s",
                                     self.fileno(), e)
-                    self.close()
+                    self.close(exc_info=True)
                     return
         if not self._write_buffer and self._write_callback:
             callback = self._write_callback
@@ -664,7 +672,7 @@ class IOStream(BaseIOStream):
             if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK):
                 gen_log.warning("Connect error on fd %d: %s",
                                 self.socket.fileno(), e)
-                self.close()
+                self.close(exc_info=True)
                 return
         self._connect_callback = stack_context.wrap(callback)
         self._add_io_state(self.io_loop.WRITE)
@@ -733,7 +741,7 @@ class SSLIOStream(IOStream):
                 return
             elif err.args[0] in (ssl.SSL_ERROR_EOF,
                                  ssl.SSL_ERROR_ZERO_RETURN):
-                return self.close()
+                return self.close(exc_info=True)
             elif err.args[0] == ssl.SSL_ERROR_SSL:
                 try:
                     peer = self.socket.getpeername()
@@ -741,11 +749,11 @@ class SSLIOStream(IOStream):
                     peer = '(not connected)'
                 gen_log.warning("SSL Error on %d %s: %s",
                                 self.socket.fileno(), peer, err)
-                return self.close()
+                return self.close(exc_info=True)
             raise
         except socket.error, err:
             if err.args[0] in (errno.ECONNABORTED, errno.ECONNRESET):
-                return self.close()
+                return self.close(exc_info=True)
         else:
             self._ssl_accepting = False
             if self._ssl_connect_callback is not None:
@@ -842,7 +850,7 @@ class PipeIOStream(BaseIOStream):
             elif e.args[0] == errno.EBADF:
                 # If the writing half of a pipe is closed, select will
                 # report it as readable but reads will fail with EBADF.
-                self.close()
+                self.close(exc_info=True)
                 return None
             else:
                 raise

libs/tornado/platform/twisted.py

@@ -431,6 +431,8 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
                 self.reactor.removeWriter(self.fds[fd])
 
     def remove_handler(self, fd):
+        if fd not in self.fds:
+            return
         self.fds[fd].lost = True
         if self.fds[fd].reading:
             self.reactor.removeReader(self.fds[fd])
@@ -444,6 +446,12 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
     def stop(self):
         self.reactor.crash()
 
+    def _run_callback(self, callback, *args, **kwargs):
+        try:
+            callback(*args, **kwargs)
+        except Exception:
+            self.handle_callback_exception(callback)
+
     def add_timeout(self, deadline, callback):
         if isinstance(deadline, (int, long, float)):
             delay = max(deadline - self.time(), 0)
@@ -451,13 +459,14 @@ class TwistedIOLoop(tornado.ioloop.IOLoop):
             delay = deadline.total_seconds()
         else:
             raise TypeError("Unsupported deadline %r")
-        return self.reactor.callLater(delay, wrap(callback))
+        return self.reactor.callLater(delay, self._run_callback, wrap(callback))
 
     def remove_timeout(self, timeout):
         timeout.cancel()
 
-    def add_callback(self, callback):
-        self.reactor.callFromThread(wrap(callback))
+    def add_callback(self, callback, *args, **kwargs):
+        self.reactor.callFromThread(self._run_callback,
+                                    wrap(callback), *args, **kwargs)
 
-    def add_callback_from_signal(self, callback):
-        self.add_callback(callback)
+    def add_callback_from_signal(self, callback, *args, **kwargs):
+        self.add_callback(callback, *args, **kwargs)

libs/tornado/process.py

@@ -268,7 +268,7 @@ class Subprocess(object):
         assert ret_pid == pid
         subproc = cls._waiting.pop(pid)
         subproc.io_loop.add_callback_from_signal(
-            functools.partial(subproc._set_returncode, status))
+            subproc._set_returncode, status)
 
     def _set_returncode(self, status):
         if os.WIFSIGNALED(status):

libs/tornado/simple_httpclient.py

@@ -12,7 +12,6 @@ from tornado.util import b, GzipDecompressor
 
 import base64
 import collections
-import contextlib
 import copy
 import functools
 import os.path
@@ -134,7 +133,7 @@ class _HTTPConnection(object):
         self._decompressor = None
         # Timeout handle returned by IOLoop.add_timeout
         self._timeout = None
-        with stack_context.StackContext(self.cleanup):
+        with stack_context.ExceptionStackContext(self._handle_exception):
             self.parsed = urlparse.urlsplit(_unicode(self.request.url))
             if ssl is None and self.parsed.scheme == "https":
                 raise ValueError("HTTPS requires either python2.6+ or "
@@ -309,19 +308,24 @@ class _HTTPConnection(object):
         if self.final_callback is not None:
             final_callback = self.final_callback
             self.final_callback = None
-            final_callback(response)
+            self.io_loop.add_callback(final_callback, response)
 
-    @contextlib.contextmanager
-    def cleanup(self):
-        try:
-            yield
-        except Exception, e:
-            gen_log.warning("uncaught exception", exc_info=True)
-            self._run_callback(HTTPResponse(self.request, 599, error=e,
+    def _handle_exception(self, typ, value, tb):
+        if self.final_callback:
+            gen_log.warning("uncaught exception", exc_info=(typ, value, tb))
+            self._run_callback(HTTPResponse(self.request, 599, error=value,
                                 request_time=self.io_loop.time() - self.start_time,
                                 ))
+
             if hasattr(self, "stream"):
                 self.stream.close()
+            return True
+        else:
+            # If our callback has already been called, we are probably
+            # catching an exception that is not caused by us but rather
+            # some child of our callback. Rather than drop it on the floor,
+            # pass it along.
+            return False
 
     def _on_close(self):
         if self.final_callback is not None:

libs/tornado/testing.py

@@ -36,9 +36,8 @@ except ImportError:
     netutil = None
     SimpleAsyncHTTPClient = None
 from tornado.log import gen_log
-from tornado.stack_context import StackContext
+from tornado.stack_context import ExceptionStackContext
 from tornado.util import raise_exc_info
-import contextlib
 import logging
 import os
 import re
@@ -167,13 +166,10 @@ class AsyncTestCase(unittest.TestCase):
         '''
         return IOLoop()
 
-    @contextlib.contextmanager
-    def _stack_context(self):
-        try:
-            yield
-        except Exception:
-            self.__failure = sys.exc_info()
-            self.stop()
+    def _handle_exception(self, typ, value, tb):
+        self.__failure = sys.exc_info()
+        self.stop()
+        return True
 
     def __rethrow(self):
         if self.__failure is not None:
@@ -182,7 +178,7 @@ class AsyncTestCase(unittest.TestCase):
             raise_exc_info(failure)
 
     def run(self, result=None):
-        with StackContext(self._stack_context):
+        with ExceptionStackContext(self._handle_exception):
             super(AsyncTestCase, self).run(result)
         # In case an exception escaped super.run or the StackContext caught
         # an exception when there wasn't a wait() to re-raise it, do so here.

libs/tornado/web.py

@@ -1317,10 +1317,8 @@ class Application(object):
     def add_handlers(self, host_pattern, host_handlers):
         """Appends the given handlers to our handler list.
 
-        Note that host patterns are processed sequentially in the
-        order they were added, and only the first matching pattern is
-        used.  This means that all handlers for a given host must be
-        added in a single add_handlers call.
+        Host patterns are processed sequentially in the order they were
+        added. All matching patterns will be considered.
         """
         if not host_pattern.endswith("$"):
             host_pattern += "$"
@@ -1365,15 +1363,16 @@ class Application(object):
 
     def _get_host_handlers(self, request):
         host = request.host.lower().split(':')[0]
+        matches = []
         for pattern, handlers in self.handlers:
             if pattern.match(host):
-                return handlers
+                matches.extend(handlers)
         # Look for default host if not behind load balancer (for debugging)
-        if "X-Real-Ip" not in request.headers:
+        if not matches and "X-Real-Ip" not in request.headers:
             for pattern, handlers in self.handlers:
                 if pattern.match(self.default_host):
-                    return handlers
-        return None
+                    matches.extend(handlers)
+        return matches or None
 
     def _load_ui_methods(self, methods):
         if type(methods) is types.ModuleType:

version.py

@@ -1,2 +1,2 @@
-VERSION = '2.0.4'
+VERSION = '2.0.5'
 BRANCH = 'desktop'