Use urlopener

couchpotato/core/downloaders/blackhole/main.py

@@ -5,7 +5,6 @@ from couchpotato.core.logger import CPLog
 from inspect import isfunction
 import os
 import traceback
-import urllib
 
 log = CPLog(__name__)
 
@@ -30,12 +29,12 @@ class Blackhole(Downloader):
                     log.info('Downloading %s to %s.' % (data.get('type'), fullPath))
                     if isfunction(data.get('download')):
                         file = data.get('download')()
-                        if not file:
-                            log.debug('Failed download file: %s' % data.get('name'))
-                            return False
                     else:
-                        log.info('Downloading: %s' % data.get('url'))
+                        file = self.urlopen(data.get('url'))
-                        file = urllib.urlopen(data.get('url')).read()
+
+                    if not file or file == '':
+                        log.debug('Failed download file: %s' % data.get('name'))
+                        return False
 
                     with open(fullPath, 'wb') as f:
                         f.write(file)

couchpotato/core/downloaders/sabnzbd/main.py

@@ -6,7 +6,6 @@ from urllib import urlencode
 import base64
 import os
 import re
-import urllib2
 
 log = CPLog(__name__)
 
@@ -52,15 +51,14 @@ class Sabnzbd(Downloader):
             params['script'] = pp_script_fn
 
         url = cleanHost(self.conf('host')) + "api?" + urlencode(params)
-        log.info("URL: " + url)
 
         try:
-            r = urllib2.urlopen(url)
+            data = self.urlopen(url)
         except Exception, e:
             log.error("Unable to connect to SAB: %s" % e)
             return False
 
-        result = r.read().strip()
+        result = data.strip()
         if not result:
             log.error("SABnzbd didn't return anything.")
             return False

couchpotato/core/notifications/nmj/main.py

@@ -6,7 +6,6 @@ from couchpotato.environment import Env
 import re
 import telnetlib
 import urllib
-import urllib2
 
 try:
     import xml.etree.cElementTree as etree
@@ -81,12 +80,10 @@ class NMJ(Notification):
         database = self.conf('database')
 
         if self.mount:
-            try:
+            log.debug('Try to mount network drive via url: %s' % (mount))
-                req = urllib2.Request(mount)
+            data = self.urlopen(mount)
-                log.debug('Try to mount network drive via url: %s' % (mount))
+            if not data:
-                handle = urllib2.urlopen(req)
+                log.error('Warning: Couldn\'t contact popcorn hour on host %s' % host)
-            except IOError, e:
-                log.error('Warning: Couldn\'t contact popcorn hour on host %s: %s' % (host, e))
                 return False
 
         params = {
@@ -99,13 +96,9 @@ class NMJ(Notification):
         UPDATE_URL = 'http://%(host)s:8008/metadata_database?%(params)s'
         updateUrl = UPDATE_URL % {'host': host, 'params': params}
 
-        try:
+        response = self.urlopen(updateUrl)
-            req = urllib2.Request(updateUrl)
+        if not response:
-            log.debug('Sending NMJ scan update command via url: %s' % (updateUrl))
+            log.error('Warning: Couldn\'t contact Popcorn Hour on host %s' % host)
-            handle = urllib2.urlopen(req)
-            response = handle.read()
-        except IOError, e:
-            log.error('Warning: Couldn\'t contact Popcorn Hour on host %s: %s' % (host, e))
             return False
 
         try:

couchpotato/core/notifications/notifo/main.py

@@ -1,13 +1,9 @@
-from couchpotato.api import addApiView
-from couchpotato.core.event import addEvent
 from couchpotato.core.helpers.encoding import toUnicode
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 from couchpotato.environment import Env
 from flask.helpers import json
 import base64
-import urllib
-import urllib2
 
 log = CPLog(__name__)
 
@@ -23,15 +19,15 @@ class Notifo(Notification):
         if self.isDisabled(): return
 
         try:
-            data = urllib.urlencode({
+            params = {
                 'msg': toUnicode(message),
-            })
+            }
 
-            req = urllib2.Request(self.url)
+            headers = {
-            authHeader = "Basic %s" % base64.encodestring('%s:%s' % (self.conf('username'), self.conf('api_key')))[:-1]
+                'Authorization': "Basic %s" % base64.encodestring('%s:%s' % (self.conf('username'), self.conf('api_key')))[:-1]
-            req.add_header("Authorization", authHeader)
+            }
 
-            handle = urllib2.urlopen(req, data)
+            handle = self.urlopen(self.url, params = params, headers = headers)
             result = json.load(handle)
 
             if result['status'] != 'success' or result['response_message'] != 'OK':

couchpotato/core/notifications/plex/main.py

@@ -1,9 +1,6 @@
-from couchpotato.api import addApiView
-from couchpotato.core.event import addEvent
 from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 from xml.dom import minidom
-import urllib
 
 log = CPLog(__name__)
 
@@ -23,12 +20,12 @@ class Plex(Notification):
             refresh_url = '%s/%%s/refresh' % base_url
 
             try:
-                xml_sections = minidom.parse(urllib.urlopen(base_url))
+                xml_sections = minidom.parse(self.urlopen(base_url))
                 sections = xml_sections.getElementsByTagName('Directory')
                 for s in sections:
                     if s.getAttribute('type') in source_type:
                         url = refresh_url % s.getAttribute('key')
-                        x = urllib.urlopen(url)
+                        x = self.urlopen(url)
             except:
                 log.error('Plex library update failed for %s.' % host)
 

couchpotato/core/notifications/xbmc/main.py

@@ -2,7 +2,6 @@ from couchpotato.core.logger import CPLog
 from couchpotato.core.notifications.base import Notification
 import base64
 import urllib
-import urllib2
 
 log = CPLog(__name__)
 
@@ -22,15 +21,14 @@ class XBMC(Notification):
 
         url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, urllib.urlencode(command))
 
-        try:
+        headers = {}
-            req = urllib2.Request(url)
+        if self.password:
-            if self.password:
+            headers = {
-                authHeader = "Basic %s" % base64.encodestring('%s:%s' % (self.conf('username'), self.conf('password')))[:-1]
+               'Authorization': "Basic %s" % base64.encodestring('%s:%s' % (self.conf('username'), self.conf('password')))[:-1]
-                req.add_header("Authorization", authHeader)
+            }
 
-            urllib2.urlopen(req, timeout = 10).read()
+        if not self.urlopen(url, headers = headers):
-        except Exception, e:
+            log.error("Couldn't sent command to XBMC")
-            log.error("Couldn't sent command to XBMC. %s" % e)
             return False
 
         log.info('XBMC notification to %s successful.' % host)

couchpotato/core/plugins/base.py

@@ -5,8 +5,14 @@ from couchpotato.core.logger import CPLog
 from couchpotato.environment import Env
 from flask.helpers import send_from_directory
 import glob
+import math
 import os.path
 import re
+import socket
+import time
+import urllib
+import urllib2
+import urlparse
 
 log = CPLog(__name__)
 
@@ -19,6 +25,9 @@ class Plugin(object):
     needs_shutdown = False
     running = []
 
+    http_last_use = {}
+    http_time_between_calls = 0
+
     def registerPlugin(self):
         addEvent('app.shutdown', self.doShutdown)
         addEvent('plugin.running', self.isRunning)
@@ -67,6 +76,40 @@ class Plugin(object):
         except Exception, e:
             log.error('Unable to create folder "%s": %s' % (path, e))
 
+    # http request
+    def urlopen(self, url, timeout = 10, params = {}, headers = {}):
+
+        socket.setdefaulttimeout(timeout)
+
+        host = urlparse(url).hostname
+        self.wait(host)
+
+        try:
+            log.info('Opening url: %s, params: %s' % (url, params))
+
+            data = urllib.urlencode(params) if len(params) > 0 else None
+            request = urllib2.Request(url, data, headers)
+
+            data = urllib2.urlopen(request).read()
+        except IOError, e:
+            log.error('Failed opening url, %s: %s' % (url, e))
+            data = ''
+
+        self.http_last_use[host] = time.time()
+
+        return data
+
+    def wait(self, host = ''):
+        now = time.time()
+
+        last_use = self.http_last_use.get(host, 0)
+
+        wait = math.ceil(last_use - now + self.http_time_between_calls)
+
+        if wait > 0:
+            log.debug('Waiting for %s, %d seconds' % (self.getName(), wait))
+            time.sleep(last_use - now + self.http_time_between_calls)
+
     def beforeCall(self, handler):
         log.debug('Calling %s.%s' % (self.getName(), handler.__name__))
         self.isRunning('%s.%s' % (self.getName(), handler.__name__))

couchpotato/core/plugins/file/main.py

@@ -34,26 +34,24 @@ class FileManager(Plugin):
 
     def download(self, url = '', dest = None, overwrite = False):
 
-        try:
+        file = self.urlopen(url)
-            file = urllib2.urlopen(url)
+        if not file:
+            log.error('File is empty, don\'t download')
+            return False
 
-            if not dest: # to Cache
+        if not dest: # to Cache
-                dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url)))
+            dest = os.path.join(Env.get('cache_dir'), '%s.%s' % (md5(url), getExt(url)))
 
-            if overwrite or not os.path.exists(dest):
+        if overwrite or not os.path.exists(dest):
-                log.debug('Writing file to: %s' % dest)
+            log.debug('Writing file to: %s' % dest)
-                output = open(dest, 'wb')
+            output = open(dest, 'wb')
-                output.write(file.read())
+            output.write(file)
-                output.close()
+            output.close()
-            else:
+        else:
-                log.debug('File already exists: %s' % dest)
+            log.debug('File already exists: %s' % dest)
 
-            return dest
+        return dest
 
-        except Exception:
-            log.error('Unable to download file "%s": %s' % (url, traceback.format_exc()))
-
-        return False
 
     def add(self, path = '', part = 1, type = (), available = 1, properties = {}):
         db = get_session()

couchpotato/core/providers/base.py

@@ -4,12 +4,8 @@ from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
 from urllib2 import URLError
 from urlparse import urlparse
-import math
 import re
-import socket
 import time
-import urllib
-import urllib2
 
 log = CPLog(__name__)
 
@@ -17,9 +13,8 @@ log = CPLog(__name__)
 class Provider(Plugin):
 
     type = None # movie, nzb, torrent, subtitle, trailer
-    time_between_searches = 10 # Default timeout for url requests
+    http_time_between_calls = 10 # Default timeout for url requests
 
-    last_use = {}
     last_available_check = {}
     is_available = {}
 
@@ -42,45 +37,16 @@ class Provider(Plugin):
 
         if self.last_available_check.get(host) < now - 900:
             self.last_available_check[host] = now
-            try:
+
-                self.urlopen(test_url, 30)
+            data = self.urlopen(test_url, 30)
-                self.is_available[host] = True
+            if not data:
-            except (IOError, URLError):
                 log.error('%s unavailable, trying again in an 15 minutes.' % self.name)
                 self.is_available[host] = False
+            else:
+                self.is_available[host] = True
 
         return self.is_available[host]
 
-    def urlopen(self, url, timeout = 10, params = {}):
-
-        socket.setdefaulttimeout(timeout)
-
-        host = urlparse(url).hostname
-        self.wait(host)
-
-        try:
-            log.info('Opening url: %s, params: %s' % (url, params))
-            request = urllib2.Request(url, urllib.urlencode(params))
-            data = urllib2.urlopen(request).read()
-        except IOError, e:
-            log.error('Failed opening url, %s: %s' % (url, e))
-            data = ''
-
-        self.last_use[host] = time.time()
-
-        return data
-
-    def wait(self, host = ''):
-        now = time.time()
-
-        last_use = self.last_use.get(host, 0)
-
-        wait = math.ceil(last_use - now + self.time_between_searches)
-
-        if wait > 0:
-            log.debug('Waiting for %s, %d seconds' % (self.getName(), wait))
-            time.sleep(last_use - now + self.time_between_searches)
-
 
 class MovieProvider(Provider):
     type = 'movie'
@@ -129,8 +95,6 @@ class YarrProvider(Provider):
 class NZBProvider(YarrProvider):
     type = 'nzb'
 
-    time_between_searches = 10 # Seconds
-
     def __init__(self):
         addEvent('provider.nzb.search', self.search)
         addEvent('provider.yarr.search', self.search)

couchpotato/core/providers/nzb/newzbin/main.py

@@ -1,16 +1,10 @@
-
+from couchpotato.core.event import fireEvent
-from couchpotato.core.event import addEvent, fireEvent
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import NZBProvider
-from couchpotato.environment import Env
 from dateutil.parser import parse
 from urllib import urlencode
-from urllib2 import URLError
-import httplib
 import time
-import traceback
-import urllib
 import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
@@ -66,14 +60,15 @@ class Newzbin(NZBProvider, RSS):
         cache_key = str('newzbin.%s.%s.%s' % (movie['library']['identifier'], str(format_id), str(cat_id)))
         single_cat = True
 
-        try:
+        data = self.getCache(cache_key)
-            data = self.getCache(cache_key)
+        if not data:
+            data = self.urlopen(url, params = {'username': self.conf('username'), 'password': self.conf('password')})
+            self.setCache(cache_key, data)
+
             if not data:
-                data = self.urlopen(url, params = {'username': self.conf('username'), 'password': self.conf('password')})
+                log.error('Failed to get data from %s.' % url)
-                self.setCache(cache_key, data)
+                return results
-        except (IOError, URLError):
+
-            log.error('Failed to open %s.' % url)
-            return results
 
         if data:
             try:

couchpotato/core/providers/nzb/newznab/main.py

@@ -1,12 +1,10 @@
-from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import cleanHost
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import NZBProvider
-from couchpotato.environment import Env
 from dateutil.parser import parse
 from urllib import urlencode
-from urllib2 import URLError
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -102,14 +100,14 @@ class Newznab(NZBProvider, RSS):
     def createItems(self, url, cache_key, host, single_cat = False, movie = None, quality = None, for_feed = False):
         results = []
 
-        try:
+        data = self.getCache(cache_key)
-            data = self.getCache(cache_key)
+        if not data:
+            data = self.urlopen(url)
+            self.setCache(cache_key, data)
+
             if not data:
-                data = self.urlopen(url)
+                log.error('Failed to get data from %s.' % url)
-                self.setCache(cache_key, data)
+                return results
-        except (IOError, URLError):
-            log.error('Failed to open %s.' % url)
-            return results
 
         if data:
             try:

couchpotato/core/providers/nzb/nzbindex/main.py

@@ -1,11 +1,10 @@
-from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import simplifyString
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import NZBProvider
 from dateutil.parser import parse
 from urllib import urlencode
-from urllib2 import URLError
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -38,14 +37,14 @@ class NzbIndex(NZBProvider, RSS):
 
         cache_key = 'nzbindex.%s.%s' % (movie['library'].get('identifier'), quality.get('identifier'))
 
-        try:
+        data = self.getCache(cache_key)
-            data = self.getCache(cache_key)
+        if not data:
+            data = self.urlopen(url)
+            self.setCache(cache_key, data)
+
             if not data:
-                data = self.urlopen(url)
+                log.error('Failed to get data from %s.' % url)
-                self.setCache(cache_key, data)
+                return results
-        except (IOError, URLError):
-            log.error('Failed to open %s.' % url)
-            return results
 
         if data:
             try:

couchpotato/core/providers/nzb/nzbmatrix/main.py

@@ -1,11 +1,10 @@
-from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
 from urllib import urlencode
-from urllib2 import URLError
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -52,14 +51,14 @@ class NZBMatrix(NZBProvider, RSS):
         cache_key = 'nzbmatrix.%s.%s' % (movie['library'].get('identifier'), cat_ids)
         single_cat = True
 
-        try:
+        data = self.getCache(cache_key)
-            data = self.getCache(cache_key)
+        if not data:
+            data = self.urlopen(url)
+            self.setCache(cache_key, data)
+
             if not data:
-                data = self.urlopen(url)
+                log.error('Failed to get data from %s.' % url)
-                self.setCache(cache_key, data)
+                return results
-        except (IOError, URLError):
-            log.error('Failed to open %s.' % url)
-            return results
 
         if data:
             try:

couchpotato/core/providers/nzb/nzbs/main.py

@@ -1,11 +1,10 @@
-from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import simplifyString
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import NZBProvider
 from dateutil.parser import parse
 from urllib import urlencode
-from urllib2 import URLError
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -48,14 +47,14 @@ class Nzbs(NZBProvider, RSS):
 
         cache_key = 'nzbs.%s.%s' % (movie['library'].get('identifier'), str(cat_id))
 
-        try:
+        data = self.getCache(cache_key)
-            data = self.getCache(cache_key)
+        if not data:
+            data = self.urlopen(url)
+            self.setCache(cache_key, data)
+
             if not data:
-                data = self.urlopen(url)
+                log.error('Failed to get data from %s.' % url)
-                self.setCache(cache_key, data)
+                return results
-        except (IOError, URLError):
-            log.error('Failed to open %s.' % url)
-            return results
 
         if data:
             try:

couchpotato/core/providers/torrent/thepiratebay/main.py

@@ -38,10 +38,9 @@ class ThePirateBay(TorrentProvider):
 
         log.info('Searching: %s' % url)
 
-        try:
+        data = self.urlopen(url)
-            data = urllib2.urlopen(url, timeout = self.timeout).read()
+        if not data:
-        except (IOError, URLError):
+            log.error('Failed to get data from %s.' % url)
-            log.error('Failed to open %s.' % url)
             return results
 
         try:
@@ -129,11 +128,10 @@ class ThePirateBay(TorrentProvider):
 
     def getInfo(self, url):
         log.debug('Getting info: %s' % url)
-        try:
+
-            data = urllib2.urlopen(url, timeout = self.timeout).read()
+        data = self.urlopen(url)
-            pass
+        if not data:
-        except IOError:
+            log.error('Failed to get data from %s.' % url)
-            log.error('Failed to open %s.' % url)
             return ''
 
         div = SoupStrainer('div')