Use urlopener

couchpotato/core/providers/base.py

@@ -4,12 +4,8 @@ from couchpotato.core.plugins.base import Plugin
 from couchpotato.environment import Env
 from urllib2 import URLError
 from urlparse import urlparse
-import math
 import re
-import socket
 import time
-import urllib
-import urllib2
 
 log = CPLog(__name__)
 
@@ -17,9 +13,8 @@ log = CPLog(__name__)
 class Provider(Plugin):
 
     type = None # movie, nzb, torrent, subtitle, trailer
-    time_between_searches = 10 # Default timeout for url requests
+    http_time_between_calls = 10 # Default timeout for url requests
 
-    last_use = {}
     last_available_check = {}
     is_available = {}
 
@@ -42,45 +37,16 @@ class Provider(Plugin):
 
         if self.last_available_check.get(host) < now - 900:
             self.last_available_check[host] = now
-            try:
-                self.urlopen(test_url, 30)
-                self.is_available[host] = True
-            except (IOError, URLError):
+
+            data = self.urlopen(test_url, 30)
+            if not data:
                 log.error('%s unavailable, trying again in an 15 minutes.' % self.name)
                 self.is_available[host] = False
+            else:
+                self.is_available[host] = True
 
         return self.is_available[host]
 
-    def urlopen(self, url, timeout = 10, params = {}):
-
-        socket.setdefaulttimeout(timeout)
-
-        host = urlparse(url).hostname
-        self.wait(host)
-
-        try:
-            log.info('Opening url: %s, params: %s' % (url, params))
-            request = urllib2.Request(url, urllib.urlencode(params))
-            data = urllib2.urlopen(request).read()
-        except IOError, e:
-            log.error('Failed opening url, %s: %s' % (url, e))
-            data = ''
-
-        self.last_use[host] = time.time()
-
-        return data
-
-    def wait(self, host = ''):
-        now = time.time()
-
-        last_use = self.last_use.get(host, 0)
-
-        wait = math.ceil(last_use - now + self.time_between_searches)
-
-        if wait > 0:
-            log.debug('Waiting for %s, %d seconds' % (self.getName(), wait))
-            time.sleep(last_use - now + self.time_between_searches)
-
 
 class MovieProvider(Provider):
     type = 'movie'
@@ -129,8 +95,6 @@ class YarrProvider(Provider):
 class NZBProvider(YarrProvider):
     type = 'nzb'
 
-    time_between_searches = 10 # Seconds
-
     def __init__(self):
         addEvent('provider.nzb.search', self.search)
         addEvent('provider.yarr.search', self.search)

couchpotato/core/providers/nzb/newzbin/main.py

@@ -1,16 +1,10 @@
-
-from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import NZBProvider
-from couchpotato.environment import Env
 from dateutil.parser import parse
 from urllib import urlencode
-from urllib2 import URLError
-import httplib
 import time
-import traceback
-import urllib
 import xml.etree.ElementTree as XMLTree
 
 log = CPLog(__name__)
@@ -66,14 +60,15 @@ class Newzbin(NZBProvider, RSS):
         cache_key = str('newzbin.%s.%s.%s' % (movie['library']['identifier'], str(format_id), str(cat_id)))
         single_cat = True
 
-        try:
-            data = self.getCache(cache_key)
+        data = self.getCache(cache_key)
+        if not data:
+            data = self.urlopen(url, params = {'username': self.conf('username'), 'password': self.conf('password')})
+            self.setCache(cache_key, data)
+
             if not data:
-                data = self.urlopen(url, params = {'username': self.conf('username'), 'password': self.conf('password')})
-                self.setCache(cache_key, data)
-        except (IOError, URLError):
-            log.error('Failed to open %s.' % url)
-            return results
+                log.error('Failed to get data from %s.' % url)
+                return results
+
 
         if data:
             try:

couchpotato/core/providers/nzb/newznab/main.py

@@ -1,12 +1,10 @@
-from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.helpers.variable import cleanHost
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import NZBProvider
-from couchpotato.environment import Env
 from dateutil.parser import parse
 from urllib import urlencode
-from urllib2 import URLError
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -102,14 +100,14 @@ class Newznab(NZBProvider, RSS):
     def createItems(self, url, cache_key, host, single_cat = False, movie = None, quality = None, for_feed = False):
         results = []
 
-        try:
-            data = self.getCache(cache_key)
+        data = self.getCache(cache_key)
+        if not data:
+            data = self.urlopen(url)
+            self.setCache(cache_key, data)
+
             if not data:
-                data = self.urlopen(url)
-                self.setCache(cache_key, data)
-        except (IOError, URLError):
-            log.error('Failed to open %s.' % url)
-            return results
+                log.error('Failed to get data from %s.' % url)
+                return results
 
         if data:
             try:

couchpotato/core/providers/nzb/nzbindex/main.py

@@ -1,11 +1,10 @@
-from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import simplifyString
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import NZBProvider
 from dateutil.parser import parse
 from urllib import urlencode
-from urllib2 import URLError
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -38,14 +37,14 @@ class NzbIndex(NZBProvider, RSS):
 
         cache_key = 'nzbindex.%s.%s' % (movie['library'].get('identifier'), quality.get('identifier'))
 
-        try:
-            data = self.getCache(cache_key)
+        data = self.getCache(cache_key)
+        if not data:
+            data = self.urlopen(url)
+            self.setCache(cache_key, data)
+
             if not data:
-                data = self.urlopen(url)
-                self.setCache(cache_key, data)
-        except (IOError, URLError):
-            log.error('Failed to open %s.' % url)
-            return results
+                log.error('Failed to get data from %s.' % url)
+                return results
 
         if data:
             try:

couchpotato/core/providers/nzb/nzbmatrix/main.py

@@ -1,11 +1,10 @@
-from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import NZBProvider
 from couchpotato.environment import Env
 from dateutil.parser import parse
 from urllib import urlencode
-from urllib2 import URLError
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -52,14 +51,14 @@ class NZBMatrix(NZBProvider, RSS):
         cache_key = 'nzbmatrix.%s.%s' % (movie['library'].get('identifier'), cat_ids)
         single_cat = True
 
-        try:
-            data = self.getCache(cache_key)
+        data = self.getCache(cache_key)
+        if not data:
+            data = self.urlopen(url)
+            self.setCache(cache_key, data)
+
             if not data:
-                data = self.urlopen(url)
-                self.setCache(cache_key, data)
-        except (IOError, URLError):
-            log.error('Failed to open %s.' % url)
-            return results
+                log.error('Failed to get data from %s.' % url)
+                return results
 
         if data:
             try:

couchpotato/core/providers/nzb/nzbs/main.py

@@ -1,11 +1,10 @@
-from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.event import fireEvent
 from couchpotato.core.helpers.encoding import simplifyString
 from couchpotato.core.helpers.rss import RSS
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import NZBProvider
 from dateutil.parser import parse
 from urllib import urlencode
-from urllib2 import URLError
 import time
 import xml.etree.ElementTree as XMLTree
 
@@ -48,14 +47,14 @@ class Nzbs(NZBProvider, RSS):
 
         cache_key = 'nzbs.%s.%s' % (movie['library'].get('identifier'), str(cat_id))
 
-        try:
-            data = self.getCache(cache_key)
+        data = self.getCache(cache_key)
+        if not data:
+            data = self.urlopen(url)
+            self.setCache(cache_key, data)
+
             if not data:
-                data = self.urlopen(url)
-                self.setCache(cache_key, data)
-        except (IOError, URLError):
-            log.error('Failed to open %s.' % url)
-            return results
+                log.error('Failed to get data from %s.' % url)
+                return results
 
         if data:
             try:

couchpotato/core/providers/torrent/thepiratebay/main.py

@@ -38,10 +38,9 @@ class ThePirateBay(TorrentProvider):
 
         log.info('Searching: %s' % url)
 
-        try:
-            data = urllib2.urlopen(url, timeout = self.timeout).read()
-        except (IOError, URLError):
-            log.error('Failed to open %s.' % url)
+        data = self.urlopen(url)
+        if not data:
+            log.error('Failed to get data from %s.' % url)
             return results
 
         try:
@@ -129,11 +128,10 @@ class ThePirateBay(TorrentProvider):
 
     def getInfo(self, url):
         log.debug('Getting info: %s' % url)
-        try:
-            data = urllib2.urlopen(url, timeout = self.timeout).read()
-            pass
-        except IOError:
-            log.error('Failed to open %s.' % url)
+
+        data = self.urlopen(url)
+        if not data:
+            log.error('Failed to get data from %s.' % url)
             return ''
 
         div = SoupStrainer('div')