dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use requests lib for openurl

Browse Source
This commit is contained in:
Ruud
2014-01-10 14:04:16 +01:00
parent 91f3cda995
commit c2568432e7
25 changed files with 86 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/core/downloaders/nzbvortex/main.py
+3 -5
View File
@@ -32,7 +32,7 @@ class NZBVortex(Downloader):
# Send the nzb
try:
nzb_filename = self.createFileName(data, filedata, media)
self.call('nzb/add', params = {'file': (nzb_filename, filedata)}, multipart = True)
self.call('nzb/add', files = {'file': (nzb_filename, filedata)})
time.sleep(10)
raw_statuses = self.call('nzb')
@@ -117,10 +117,9 @@ class NZBVortex(Downloader):
params = tryUrlencode(parameters)
url = cleanHost(self.conf('host')) + 'api/' + call
url_opener = urllib2.build_opener(HTTPSHandler())
try:
data = self.urlopen('%s?%s' % (url, params), opener = url_opener, *args, **kwargs)
data = self.urlopen('%s?%s' % (url, params), *args, **kwargs)
if data:
return json.loads(data)
@@ -142,10 +141,9 @@ class NZBVortex(Downloader):
if not self.api_level:
url = cleanHost(self.conf('host')) + 'api/app/apilevel'
url_opener = urllib2.build_opener(HTTPSHandler())
try:
data = self.urlopen(url, opener = url_opener, show_error = False)
data = self.urlopen(url, show_error = False)
self.api_level = float(json.loads(data).get('apilevel'))
except URLError, e:
if hasattr(e, 'code') and e.code == 403:
couchpotato/core/downloaders/sabnzbd/main.py
+1 -1
View File
@@ -43,7 +43,7 @@ class Sabnzbd(Downloader):
try:
if nzb_filename and req_params.get('mode') is 'addfile':
sab_data = self.call(req_params, params = {'nzbfile': (ss(nzb_filename), filedata)}, multipart = True)
sab_data = self.call(req_params, files = {'nzbfile': (ss(nzb_filename), filedata)})
else:
sab_data = self.call(req_params)
except URLError:
couchpotato/core/downloaders/synology/main.py
-1
View File
@@ -11,7 +11,6 @@ log = CPLog(__name__)
class Synology(Downloader):
protocol = ['nzb', 'torrent', 'torrent_magnet']
log = CPLog(__name__)
def download(self, data = None, media = None, filedata = None):
if not media: media = {}
couchpotato/core/notifications/boxcar/main.py
+2 -2
View File
@@ -16,14 +16,14 @@ class Boxcar(Notification):
try:
message = message.strip()
params = {
data = {
'email': self.conf('email'),
'notification[from_screen_name]': self.default_title,
'notification[message]': toUnicode(message),
'notification[from_remote_service_id]': int(time.time()),
}
self.urlopen(self.url, params = params)
self.urlopen(self.url, data = data)
except:
log.error('Check your email and added services on boxcar.io')
return False
couchpotato/core/notifications/prowl/main.py
+1 -1
View File
@@ -26,7 +26,7 @@ class Prowl(Notification):
}
try:
self.urlopen(self.urls['api'], headers = headers, params = data, multipart = True, show_error = False)
self.urlopen(self.urls['api'], headers = headers, data = data, show_error = False)
log.info('Prowl notifications sent.')
return True
except:
couchpotato/core/notifications/pushalot/main.py
+1 -1
View File
@@ -29,7 +29,7 @@ class Pushalot(Notification):
}
try:
self.urlopen(self.urls['api'], headers = headers, params = data, multipart = True, show_error = False)
self.urlopen(self.urls['api'], headers = headers, data = data, show_error = False)
return True
except:
log.error('PushAlot failed: %s', traceback.format_exc())
couchpotato/core/notifications/pushbullet/main.py
+2 -2
View File
@@ -74,9 +74,9 @@ class Pushbullet(Notification):
}
if cache:
return self.getJsonData(self.url % method, headers = headers, params = kwargs)
return self.getJsonData(self.url % method, headers = headers, data = kwargs)
else:
data = self.urlopen(self.url % method, headers = headers, params = kwargs)
data = self.urlopen(self.url % method, headers = headers, data = kwargs)
return json.loads(data)
except Exception, ex:
couchpotato/core/notifications/trakt/main.py
+1 -1
View File
@@ -35,7 +35,7 @@ class Trakt(Notification):
def call(self, method_url, post_data):
try:
response = self.getJsonData(self.urls['base'] % method_url, params = post_data, cache_timeout = 1)
response = self.getJsonData(self.urls['base'] % method_url, data = post_data, cache_timeout = 1)
if response:
if response.get('status') == "success":
log.info('Successfully called Trakt')
couchpotato/core/notifications/xbmc/main.py
+1 -1
View File
@@ -202,7 +202,7 @@ class XBMC(Notification):
try:
log.debug('Sending request to %s: %s', (host, data))
response = self.getJsonData(server, headers = headers, params = data, timeout = 3, show_error = False)
response = self.getJsonData(server, headers = headers, data = data, timeout = 3, show_error = False)
log.debug('Returned from request %s: %s', (host, response))
return response
couchpotato/core/plugins/base.py
+19 -47
View File
@@ -1,19 +1,15 @@
from StringIO import StringIO
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import tryUrlencode, ss, toSafeString, \
from couchpotato.core.helpers.encoding import ss, toSafeString, \
toUnicode, sp
from couchpotato.core.helpers.variable import getExt, md5, isLocalIP
from couchpotato.core.logger import CPLog
from couchpotato.environment import Env
from multipartpost import MultipartPostHandler
import requests
from tornado import template
from tornado.web import StaticFileHandler
from urlparse import urlparse
import cookielib
import glob
import gzip
import inspect
import math
import os.path
import re
import time
@@ -39,7 +35,7 @@ class Plugin(object):
http_time_between_calls = 0
http_failed_request = {}
http_failed_disabled = {}
http_opener = None
http_opener = requests.Session()
def __new__(typ, *args, **kwargs):
new_plugin = super(Plugin, typ).__new__(typ)
@@ -121,11 +117,11 @@ class Plugin(object):
return False
# http request
def urlopen(self, url, timeout = 30, params = None, headers = None, opener = None, multipart = False, show_error = True):
def urlopen(self, url, timeout = 30, data = None, headers = None, files = None, show_error = True):
url = urllib2.quote(ss(url), safe = "%/:=&?~#+!$,;'@()*[]")
if not headers: headers = {}
if not params: params = {}
if not data: data = {}
# Fill in some headers
parsed_url = urlparse(url)
@@ -138,13 +134,7 @@ class Plugin(object):
headers['Connection'] = headers.get('Connection', 'keep-alive')
headers['Cache-Control'] = headers.get('Cache-Control', 'max-age=0')
# Create opener
if not opener:
if not self.http_opener:
cookies = cookielib.CookieJar()
self.http_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies))
opener = self.http_opener
r = self.http_opener
# Don't try for failed requests
if self.http_failed_disabled.get(host, 0) > 0:
@@ -161,37 +151,18 @@ class Plugin(object):
self.wait(host)
try:
# Make sure opener has the correct headers
opener.add_headers = headers
kwargs = {
'headers': headers,
'data': data if len(data) > 0 else None,
'timeout': timeout,
'files': files,
}
method = 'post' if len(data) > 0 or files else 'get'
if multipart:
log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data'))
request = urllib2.Request(url, params, headers)
log.info('Opening url: %s %s, data: %s', (method, url, [x for x in data.iterkeys()] if isinstance(data, dict) else 'with data'))
response = r.request(method, url, **kwargs)
opener.add_handler(MultipartPostHandler())
response = opener.open(request, timeout = timeout)
else:
log.info('Opening url: %s, params: %s', (url, [x for x in params.iterkeys()] if isinstance(params, dict) else 'with data'))
if isinstance(params, (str, unicode)) and len(params) > 0:
data = params
else:
data = tryUrlencode(params) if len(params) > 0 else None
request = urllib2.Request(url, data, headers)
response = opener.open(request, timeout = timeout)
# unzip if needed
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj = buf)
data = f.read()
f.close()
else:
data = response.read()
response.close()
data = response.text
self.http_failed_request[host] = 0
except IOError:
@@ -274,18 +245,19 @@ class Plugin(object):
try:
cache_timeout = 300
if kwargs.get('cache_timeout'):
if kwargs.has_key('cache_timeout'):
cache_timeout = kwargs.get('cache_timeout')
del kwargs['cache_timeout']
data = self.urlopen(url, **kwargs)
if data:
if data and cache_timeout > 0:
self.setCache(cache_key, data, timeout = cache_timeout)
return data
except:
if not kwargs.get('show_error', True):
raise
log.error('Failed getting cache: %s', (traceback.format_exc()))
return ''
def setCache(self, cache_key, value, timeout = 300):
couchpotato/core/providers/base.py
+11 -19
View File
@@ -5,12 +5,10 @@ from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
from urlparse import urlparse
import cookielib
import json
import re
import time
import traceback
import urllib2
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__)
@@ -95,7 +93,7 @@ class Provider(Plugin):
def getHTMLData(self, url, **kwargs):
cache_key = '%s%s' % (md5(url), md5('%s' % kwargs.get('params', {})))
cache_key = '%s%s' % (md5(url), md5('%s' % kwargs.get('data', {})))
return self.getCache(cache_key, url, **kwargs)
@@ -111,8 +109,7 @@ class YarrProvider(Provider):
sizeMb = ['mb', 'mib']
sizeKb = ['kb', 'kib']
login_opener = None
last_login_check = 0
last_login_check = None
def __init__(self):
addEvent('provider.enabled_protocols', self.getEnabledProtocol)
@@ -129,35 +126,30 @@ class YarrProvider(Provider):
# Check if we are still logged in every hour
now = time.time()
if self.login_opener and self.last_login_check < (now - 3600):
if self.last_login_check and self.last_login_check < (now - 3600):
try:
output = self.urlopen(self.urls['login_check'], opener = self.login_opener)
output = self.urlopen(self.urls['login_check'])
if self.loginCheckSuccess(output):
self.last_login_check = now
return True
else:
self.login_opener = None
except:
self.login_opener = None
except: pass
self.last_login_check = None
if self.login_opener:
if self.last_login_check:
return True
try:
cookiejar = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))
output = self.urlopen(self.urls['login'], params = self.getLoginParams(), opener = opener)
output = self.urlopen(self.urls['login'], data = self.getLoginParams())
if self.loginSuccess(output):
self.last_login_check = now
self.login_opener = opener
return True
error = 'unknown'
except:
error = traceback.format_exc()
self.login_opener = None
self.last_login_check = None
log.error('Failed to login %s: %s', (self.getName(), error))
return False
@@ -171,12 +163,12 @@ class YarrProvider(Provider):
try:
if not self.login():
log.error('Failed downloading from %s', self.getName())
return self.urlopen(url, opener = self.login_opener)
return self.urlopen(url)
except:
log.error('Failed downloading from %s: %s', (self.getName(), traceback.format_exc()))
def getLoginParams(self):
return ''
return {}
def download(self, url = '', nzb_id = ''):
try:
couchpotato/core/providers/info/couchpotatoapi/main.py
+1 -1
View File
@@ -97,7 +97,7 @@ class CouchPotatoApi(MovieProvider):
if not ignore: ignore = []
if not movies: movies = []
suggestions = self.getJsonData(self.urls['suggest'], params = {
suggestions = self.getJsonData(self.urls['suggest'], data = {
'movies': ','.join(movies),
'ignore': ','.join(ignore),
}, headers = self.getRequestHeaders())
couchpotato/core/providers/nzb/binsearch/main.py
+2 -2
View File
@@ -90,13 +90,13 @@ class BinSearch(NZBProvider):
def download(self, url = '', nzb_id = ''):
params = {
data = {
'action': 'nzb',
nzb_id: 'on'
}
try:
return self.urlopen(url, params = params, show_error = False)
return self.urlopen(url, data = data, show_error = False)
except:
log.error('Failed getting nzb from %s: %s', (self.getName(), traceback.format_exc()))
couchpotato/core/providers/torrent/bithdtv/main.py
+3 -3
View File
@@ -31,7 +31,7 @@ class BiTHDTV(TorrentProvider):
url = "%s&%s" % (self.urls['search'], arguments)
data = self.getHTMLData(url, opener = self.login_opener)
data = self.getHTMLData(url)
if data:
# Remove BiT-HDTV's output garbage so outdated BS4 versions successfully parse the HTML
@@ -68,10 +68,10 @@ class BiTHDTV(TorrentProvider):
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def getLoginParams(self):
return tryUrlencode({
return {
'username': self.conf('username'),
'password': self.conf('password'),
})
}
def getMoreInfo(self, item):
full_description = self.getCache('bithdtv.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
couchpotato/core/providers/torrent/bitsoup/main.py
+3 -3
View File
@@ -28,7 +28,7 @@ class Bitsoup(TorrentProvider):
})
url = "%s&%s" % (self.urls['search'], arguments)
data = self.getHTMLData(url, opener = self.login_opener)
data = self.getHTMLData(url)
if data:
html = BeautifulSoup(data)
@@ -73,11 +73,11 @@ class Bitsoup(TorrentProvider):
def getLoginParams(self):
return tryUrlencode({
return {
'username': self.conf('username'),
'password': self.conf('password'),
'ssl': 'yes',
})
}
def loginSuccess(self, output):
couchpotato/core/providers/torrent/hdbits/main.py
+6 -5
View File
@@ -1,5 +1,4 @@
from bs4 import BeautifulSoup
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
@@ -23,7 +22,7 @@ class HDBits(TorrentProvider):
def _search(self, movie, quality, results):
data = self.getJsonData(self.urls['search'] % movie['library']['identifier'], opener = self.login_opener)
data = self.getJsonData(self.urls['search'] % movie['library']['identifier'])
if data:
try:
@@ -42,15 +41,17 @@ class HDBits(TorrentProvider):
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def getLoginParams(self):
data = self.getHTMLData('https://hdbits.org/login')
data = self.getHTMLData('https://hdbits.org/login', cache_timeout = 0)
bs = BeautifulSoup(data)
secret = bs.find('input', attrs = {'name': 'lol'})['value']
return tryUrlencode({
return {
'uname': self.conf('username'),
'password': self.conf('password'),
'returnto': '/',
'lol': secret
})
}
def loginSuccess(self, output):
return '/logout.php' in output.lower()
couchpotato/core/providers/torrent/ilovetorrents/main.py
+4 -4
View File
@@ -42,7 +42,7 @@ class ILoveTorrents(TorrentProvider):
search_url = self.urls['search'] % (movieTitle, page, cats[0])
page += 1
data = self.getHTMLData(search_url, opener = self.login_opener)
data = self.getHTMLData(search_url)
if data:
try:
soup = BeautifulSoup(data)
@@ -96,11 +96,11 @@ class ILoveTorrents(TorrentProvider):
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def getLoginParams(self):
return tryUrlencode({
return {
'username': self.conf('username'),
'password': self.conf('password'),
'submit': 'Welcome to ILT',
})
}
def getMoreInfo(self, item):
cache_key = 'ilt.%s' % item['id']
@@ -109,7 +109,7 @@ class ILoveTorrents(TorrentProvider):
if not description:
try:
full_description = self.getHTMLData(item['detail_url'], opener = self.login_opener)
full_description = self.getHTMLData(item['detail_url'])
html = BeautifulSoup(full_description)
nfo_pre = html.find('td', attrs = {'class':'main'}).findAll('table')[1]
description = toUnicode(nfo_pre.text) if nfo_pre else ''
couchpotato/core/providers/torrent/iptorrents/main.py
+3 -3
View File
@@ -37,7 +37,7 @@ class IPTorrents(TorrentProvider):
while current_page <= pages and not self.shuttingDown():
url = self.urls['search'] % (self.getCatId(quality['identifier'])[0], freeleech, tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), current_page)
data = self.getHTMLData(url, opener = self.login_opener)
data = self.getHTMLData(url)
if data:
html = BeautifulSoup(data)
@@ -90,11 +90,11 @@ class IPTorrents(TorrentProvider):
current_page += 1
def getLoginParams(self):
return tryUrlencode({
return {
'username': self.conf('username'),
'password': self.conf('password'),
'login': 'submit',
})
}
def loginSuccess(self, output):
return 'don\'t have an account' not in output.lower()
couchpotato/core/providers/torrent/passthepopcorn/main.py
+3 -3
View File
@@ -65,7 +65,7 @@ class PassThePopcorn(TorrentProvider):
})
url = '%s?json=noredirect&%s' % (self.urls['torrent'], tryUrlencode(params))
res = self.getJsonData(url, opener = self.login_opener)
res = self.getJsonData(url)
try:
if not 'Movies' in res:
@@ -188,13 +188,13 @@ class PassThePopcorn(TorrentProvider):
return self.unicodeToASCII(self.htmlToUnicode(text))
def getLoginParams(self):
return tryUrlencode({
return {
'username': self.conf('username'),
'password': self.conf('password'),
'passkey': self.conf('passkey'),
'keeplogged': '1',
'login': 'Login'
})
}
def loginSuccess(self, output):
try:
couchpotato/core/providers/torrent/sceneaccess/main.py
+3 -3
View File
@@ -45,7 +45,7 @@ class SceneAccess(TorrentProvider):
url = "%s&%s" % (url, arguments)
data = self.getHTMLData(url, opener = self.login_opener)
data = self.getHTMLData(url)
if data:
html = BeautifulSoup(data)
@@ -78,11 +78,11 @@ class SceneAccess(TorrentProvider):
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def getLoginParams(self):
return tryUrlencode({
return {
'username': self.conf('username'),
'password': self.conf('password'),
'submit': 'come on in',
})
}
def getMoreInfo(self, item):
full_description = self.getCache('sceneaccess.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
couchpotato/core/providers/torrent/torrentbytes/main.py
+3 -3
View File
@@ -35,7 +35,7 @@ class TorrentBytes(TorrentProvider):
def _searchOnTitle(self, title, movie, quality, results):
url = self.urls['search'] % (tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), self.getCatId(quality['identifier'])[0])
data = self.getHTMLData(url, opener = self.login_opener)
data = self.getHTMLData(url)
if data:
html = BeautifulSoup(data)
@@ -69,11 +69,11 @@ class TorrentBytes(TorrentProvider):
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
def getLoginParams(self):
return tryUrlencode({
return {
'username': self.conf('username'),
'password': self.conf('password'),
'login': 'submit',
})
}
def loginSuccess(self, output):
return 'logout.php' in output.lower() or 'Welcome' in output.lower()
couchpotato/core/providers/torrent/torrentday/main.py
+6 -5
View File
@@ -1,4 +1,3 @@
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
@@ -30,7 +29,7 @@ class TorrentDay(TorrentProvider):
q = '"%s %s"' % (title, movie['library']['year'])
params = {
data = {
'/browse.php?': None,
'cata': 'yes',
'jxt': 8,
@@ -38,7 +37,7 @@ class TorrentDay(TorrentProvider):
'search': q,
}
data = self.getJsonData(self.urls['search'], params = params, opener = self.login_opener)
data = self.getJsonData(self.urls['search'], data = data)
try: torrents = data.get('Fs', [])[0].get('Cn', {}).get('torrents', [])
except: return
@@ -54,11 +53,13 @@ class TorrentDay(TorrentProvider):
})
def getLoginParams(self):
return tryUrlencode({
return {
'username': self.conf('username'),
'password': self.conf('password'),
'submit.x': 18,
'submit.y': 11,
'submit': 'submit',
})
}
def loginSuccess(self, output):
return 'Password not correct' not in output
couchpotato/core/providers/torrent/torrentleech/main.py
+3 -3
View File
@@ -36,7 +36,7 @@ class TorrentLeech(TorrentProvider):
def _searchOnTitle(self, title, movie, quality, results):
url = self.urls['search'] % (tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), self.getCatId(quality['identifier'])[0])
data = self.getHTMLData(url, opener = self.login_opener)
data = self.getHTMLData(url)
if data:
html = BeautifulSoup(data)
@@ -68,12 +68,12 @@ class TorrentLeech(TorrentProvider):
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
def getLoginParams(self):
return tryUrlencode({
return {
'username': self.conf('username'),
'password': self.conf('password'),
'remember_me': 'on',
'login': 'submit',
})
}
def loginSuccess(self, output):
return '/user/account/logout' in output.lower() or 'welcome back' in output.lower()
couchpotato/core/providers/torrent/torrentshack/main.py
+3 -3
View File
@@ -34,7 +34,7 @@ class TorrentShack(TorrentProvider):
scene_only = '1' if self.conf('scene_only') else ''
url = self.urls['search'] % (tryUrlencode('%s %s' % (title.replace(':', ''), movie['library']['year'])), scene_only, self.getCatId(quality['identifier'])[0])
data = self.getHTMLData(url, opener = self.login_opener)
data = self.getHTMLData(url)
if data:
html = BeautifulSoup(data)
@@ -65,12 +65,12 @@ class TorrentShack(TorrentProvider):
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
def getLoginParams(self):
return tryUrlencode({
return {
'username': self.conf('username'),
'password': self.conf('password'),
'keeplogged': '1',
'login': 'Login',
})
}
def loginSuccess(self, output):
return 'logout.php' in output.lower()
couchpotato/runner.py
+1 -1
View File
@@ -145,7 +145,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
Env.set('dev', development)
# Disable logging for some modules
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado']:
for logger_name in ['enzyme', 'guessit', 'subliminal', 'apscheduler', 'tornado', 'requests']:
logging.getLogger(logger_name).setLevel(logging.ERROR)
for logger_name in ['gntp', 'migrate']: