dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'refs/heads/develop'

Browse Source
This commit is contained in:
Ruud
2012-06-16 18:09:34 +02:00
parent 7367f7da79 79e842951e
commit 1c0b83eb75
187 changed files with 23312 additions and 3314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/__init__.py
+2 -2
View File
@@ -69,10 +69,10 @@ def getApiKey():
@app.errorhandler(404) @app.errorhandler(404)
def page_not_found(error): def page_not_found(error):
index_url = url_for('web.index') index_url = url_for('web.index')
url = getattr(request, 'path')[len(index_url):] url = request.path[len(index_url):]
if url[:3] != 'api': if url[:3] != 'api':
return redirect(index_url + '#' + url) return redirect(request.url.replace(request.path, index_url + '#' + url))
else: else:
time.sleep(0.1) time.sleep(0.1)
return 'Wrong API key used', 404 return 'Wrong API key used', 404
couchpotato/core/_base/_core/main.py
+3 -3
View File
@@ -96,7 +96,7 @@ class Core(Plugin):
while loop: while loop:
log.debug('Asking who is running') log.debug('Asking who is running')
still_running = fireEvent('plugin.running', merge = True) still_running = fireEvent('plugin.running', merge = True)
log.debug('Still running: %s' % still_running) log.debug('Still running: %s', still_running)
if len(still_running) == 0: if len(still_running) == 0:
break break
@@ -105,7 +105,7 @@ class Core(Plugin):
running = list(set(still_running) - set(self.ignore_restart)) running = list(set(still_running) - set(self.ignore_restart))
if len(running) > 0: if len(running) > 0:
log.info('Waiting on plugins to finish: %s' % running) log.info('Waiting on plugins to finish: %s', running)
else: else:
loop = False loop = False
@@ -118,7 +118,7 @@ class Core(Plugin):
except RuntimeError: except RuntimeError:
pass pass
except: except:
log.error('Failed shutting down the server: %s' % traceback.format_exc()) log.error('Failed shutting down the server: %s', traceback.format_exc())
fireEvent('app.after_shutdown', restart = restart) fireEvent('app.after_shutdown', restart = restart)
couchpotato/core/_base/scheduler/main.py
+5 -5
View File
@@ -28,7 +28,7 @@ class Scheduler(Plugin):
for type in ['interval', 'cron']: for type in ['interval', 'cron']:
try: try:
self.sched.unschedule_job(getattr(self, type)[identifier]['job']) self.sched.unschedule_job(getattr(self, type)[identifier]['job'])
log.debug('%s unscheduled %s' % (type.capitalize(), identifier)) log.debug('%s unscheduled %s', (type.capitalize(), identifier))
except: except:
pass pass
@@ -45,7 +45,7 @@ class Scheduler(Plugin):
job = self.sched.add_cron_job(cron['handle'], day = cron['day'], hour = cron['hour'], minute = cron['minute']) job = self.sched.add_cron_job(cron['handle'], day = cron['day'], hour = cron['hour'], minute = cron['minute'])
cron['job'] = job cron['job'] = job
except ValueError, e: except ValueError, e:
log.error("Failed adding cronjob: %s" % e) log.error('Failed adding cronjob: %s', e)
# Intervals # Intervals
for identifier in self.intervals: for identifier in self.intervals:
@@ -55,7 +55,7 @@ class Scheduler(Plugin):
job = self.sched.add_interval_job(interval['handle'], hours = interval['hours'], minutes = interval['minutes'], seconds = interval['seconds']) job = self.sched.add_interval_job(interval['handle'], hours = interval['hours'], minutes = interval['minutes'], seconds = interval['seconds'])
interval['job'] = job interval['job'] = job
except ValueError, e: except ValueError, e:
log.error("Failed adding interval cronjob: %s" % e) log.error('Failed adding interval cronjob: %s', e)
# Start it # Start it
log.debug('Starting scheduler') log.debug('Starting scheduler')
@@ -75,7 +75,7 @@ class Scheduler(Plugin):
self.started = False self.started = False
def cron(self, identifier = '', handle = None, day = '*', hour = '*', minute = '*'): def cron(self, identifier = '', handle = None, day = '*', hour = '*', minute = '*'):
log.info('Scheduling "%s", cron: day = %s, hour = %s, minute = %s' % (identifier, day, hour, minute)) log.info('Scheduling "%s", cron: day = %s, hour = %s, minute = %s', (identifier, day, hour, minute))
self.remove(identifier) self.remove(identifier)
self.crons[identifier] = { self.crons[identifier] = {
@@ -86,7 +86,7 @@ class Scheduler(Plugin):
} }
def interval(self, identifier = '', handle = None, hours = 0, minutes = 0, seconds = 0): def interval(self, identifier = '', handle = None, hours = 0, minutes = 0, seconds = 0):
log.info('Scheduling %s, interval: hours = %s, minutes = %s, seconds = %s' % (identifier, hours, minutes, seconds)) log.info('Scheduling %s, interval: hours = %s, minutes = %s, seconds = %s', (identifier, hours, minutes, seconds))
self.remove(identifier) self.remove(identifier)
self.intervals[identifier] = { self.intervals[identifier] = {
couchpotato/core/_base/updater/main.py
+19 -18
View File
@@ -1,5 +1,6 @@
from couchpotato.api import addApiView from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import ss
from couchpotato.core.helpers.request import jsonified from couchpotato.core.helpers.request import jsonified
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
@@ -120,7 +121,7 @@ class BaseUpdater(Plugin):
def deletePyc(self, only_excess = True): def deletePyc(self, only_excess = True):
for root, dirs, files in os.walk(Env.get('app_dir')): for root, dirs, files in os.walk(ss(Env.get('app_dir'))):
pyc_files = filter(lambda filename: filename.endswith('.pyc'), files) pyc_files = filter(lambda filename: filename.endswith('.pyc'), files)
py_files = set(filter(lambda filename: filename.endswith('.py'), files)) py_files = set(filter(lambda filename: filename.endswith('.py'), files))
@@ -128,11 +129,11 @@ class BaseUpdater(Plugin):
for excess_pyc_file in excess_pyc_files: for excess_pyc_file in excess_pyc_files:
full_path = os.path.join(root, excess_pyc_file) full_path = os.path.join(root, excess_pyc_file)
log.debug('Removing old PYC file: %s' % full_path) log.debug('Removing old PYC file: %s', full_path)
try: try:
os.remove(full_path) os.remove(full_path)
except: except:
log.error('Couldn\'t remove %s: %s' % (full_path, traceback.format_exc())) log.error('Couldn\'t remove %s: %s', (full_path, traceback.format_exc()))
for dir_name in dirs: for dir_name in dirs:
full_path = os.path.join(root, dir_name) full_path = os.path.join(root, dir_name)
@@ -140,7 +141,7 @@ class BaseUpdater(Plugin):
try: try:
os.rmdir(full_path) os.rmdir(full_path)
except: except:
log.error('Couldn\'t remove empty directory %s: %s' % (full_path, traceback.format_exc())) log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
@@ -170,7 +171,7 @@ class GitUpdater(BaseUpdater):
return True return True
except: except:
log.error('Failed updating via GIT: %s' % traceback.format_exc()) log.error('Failed updating via GIT: %s', traceback.format_exc())
self.update_failed = True self.update_failed = True
@@ -181,14 +182,14 @@ class GitUpdater(BaseUpdater):
if not self.version: if not self.version:
try: try:
output = self.repo.getHead() # Yes, please output = self.repo.getHead() # Yes, please
log.debug('Git version output: %s' % output.hash) log.debug('Git version output: %s', output.hash)
self.version = { self.version = {
'hash': output.hash[:8], 'hash': output.hash[:8],
'date': output.getDate(), 'date': output.getDate(),
'type': 'git', 'type': 'git',
} }
except Exception, e: except Exception, e:
log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s' % e) log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s', e)
return 'No GIT' return 'No GIT'
return self.version return self.version
@@ -198,7 +199,7 @@ class GitUpdater(BaseUpdater):
if self.update_version: if self.update_version:
return return
log.info('Checking for new version on github for %s' % self.repo_name) log.info('Checking for new version on github for %s', self.repo_name)
if not Env.get('dev'): if not Env.get('dev'):
self.repo.fetch() self.repo.fetch()
@@ -210,7 +211,7 @@ class GitUpdater(BaseUpdater):
local = self.repo.getHead() local = self.repo.getHead()
remote = branch.getHead() remote = branch.getHead()
log.info('Versions, local:%s, remote:%s' % (local.hash[:8], remote.hash[:8])) log.info('Versions, local:%s, remote:%s', (local.hash[:8], remote.hash[:8]))
if local.getDate() < remote.getDate(): if local.getDate() < remote.getDate():
self.update_version = { self.update_version = {
@@ -263,13 +264,13 @@ class SourceUpdater(BaseUpdater):
return True return True
except: except:
log.error('Failed updating: %s' % traceback.format_exc()) log.error('Failed updating: %s', traceback.format_exc())
self.update_failed = True self.update_failed = True
return False return False
def replaceWith(self, path): def replaceWith(self, path):
app_dir = Env.get('app_dir') app_dir = ss(Env.get('app_dir'))
# Get list of files we want to overwrite # Get list of files we want to overwrite
self.deletePyc(only_excess = False) self.deletePyc(only_excess = False)
@@ -296,7 +297,7 @@ class SourceUpdater(BaseUpdater):
except ValueError: except ValueError:
pass pass
except Exception, e: except Exception, e:
log.error('Failed overwriting file: %s' % e) log.error('Failed overwriting file: %s', e)
def removeDir(self, path): def removeDir(self, path):
@@ -315,11 +316,11 @@ class SourceUpdater(BaseUpdater):
output = json.loads(f.read()) output = json.loads(f.read())
f.close() f.close()
log.debug('Source version output: %s' % output) log.debug('Source version output: %s', output)
self.version = output self.version = output
self.version['type'] = 'source' self.version['type'] = 'source'
except Exception, e: except Exception, e:
log.error('Failed using source updater. %s' % e) log.error('Failed using source updater. %s', e)
return {} return {}
return self.version return self.version
@@ -336,7 +337,7 @@ class SourceUpdater(BaseUpdater):
self.last_check = time.time() self.last_check = time.time()
except: except:
log.error('Failed updating via source: %s' % traceback.format_exc()) log.error('Failed updating via source: %s', traceback.format_exc())
return self.update_version is not None return self.update_version is not None
@@ -351,7 +352,7 @@ class SourceUpdater(BaseUpdater):
'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())), 'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())),
} }
except: except:
log.error('Failed getting latest request from github: %s' % traceback.format_exc()) log.error('Failed getting latest request from github: %s', traceback.format_exc())
return {} return {}
@@ -372,7 +373,7 @@ class DesktopUpdater(BaseUpdater):
if e['status'] == 'done': if e['status'] == 'done':
fireEventAsync('app.restart') fireEventAsync('app.restart')
else: else:
log.error('Failed updating desktop: %s' % e['exception']) log.error('Failed updating desktop: %s', e['exception'])
self.update_failed = True self.update_failed = True
self.desktop._esky.auto_update(callback = do_restart) self.desktop._esky.auto_update(callback = do_restart)
@@ -403,7 +404,7 @@ class DesktopUpdater(BaseUpdater):
self.last_check = time.time() self.last_check = time.time()
except: except:
log.error('Failed updating desktop: %s' % traceback.format_exc()) log.error('Failed updating desktop: %s', traceback.format_exc())
return self.update_version is not None return self.update_version is not None
couchpotato/core/downloaders/blackhole/main.py
+5 -5
View File
@@ -16,7 +16,7 @@ class Blackhole(Downloader):
directory = self.conf('directory') directory = self.conf('directory')
if not directory or not os.path.isdir(directory): if not directory or not os.path.isdir(directory):
log.error('No directory set for blackhole %s download.' % data.get('type')) log.error('No directory set for blackhole %s download.', data.get('type'))
else: else:
try: try:
if not filedata or len(filedata) < 50: if not filedata or len(filedata) < 50:
@@ -27,19 +27,19 @@ class Blackhole(Downloader):
try: try:
if not os.path.isfile(fullPath): if not os.path.isfile(fullPath):
log.info('Downloading %s to %s.' % (data.get('type'), fullPath)) log.info('Downloading %s to %s.', (data.get('type'), fullPath))
with open(fullPath, 'wb') as f: with open(fullPath, 'wb') as f:
f.write(filedata) f.write(filedata)
return True return True
else: else:
log.info('File %s already exists.' % fullPath) log.info('File %s already exists.', fullPath)
return True return True
except: except:
log.error('Failed to download to blackhole %s' % traceback.format_exc()) log.error('Failed to download to blackhole %s', traceback.format_exc())
pass pass
except: except:
log.info('Failed to download file %s: %s' % (data.get('name'), traceback.format_exc())) log.info('Failed to download file %s: %s', (data.get('name'), traceback.format_exc()))
return False return False
return False return False
couchpotato/core/downloaders/nzbget/main.py
+4 -4
View File
@@ -20,10 +20,10 @@ class NZBGet(Downloader):
return return
if not filedata: if not filedata:
log.error('Unable to get NZB file: %s' % traceback.format_exc()) log.error('Unable to get NZB file: %s', traceback.format_exc())
return False return False
log.info('Sending "%s" to NZBGet.' % data.get('name')) log.info('Sending "%s" to NZBGet.', data.get('name'))
url = self.url % {'host': self.conf('host'), 'password': self.conf('password')} url = self.url % {'host': self.conf('host'), 'password': self.conf('password')}
nzb_name = '%s.nzb' % self.createNzbName(data, movie) nzb_name = '%s.nzb' % self.createNzbName(data, movie)
@@ -41,12 +41,12 @@ class NZBGet(Downloader):
if e.errcode == 401: if e.errcode == 401:
log.error('Password is incorrect.') log.error('Password is incorrect.')
else: else:
log.error('Protocol Error: %s' % e) log.error('Protocol Error: %s', e)
return False return False
if rpc.append(nzb_name, self.conf('category'), False, standard_b64encode(filedata.strip())): if rpc.append(nzb_name, self.conf('category'), False, standard_b64encode(filedata.strip())):
log.info('NZB sent successfully to NZBGet') log.info('NZB sent successfully to NZBGet')
return True return True
else: else:
log.error('NZBGet could not add %s to the queue.' % nzb_name) log.error('NZBGet could not add %s to the queue.', nzb_name)
return False return False
couchpotato/core/downloaders/sabnzbd/__init__.py
-6
View File
@@ -33,12 +33,6 @@ config = [{
'label': 'Category', 'label': 'Category',
'description': 'The category CP places the nzb in. Like <strong>movies</strong> or <strong>couchpotato</strong>', 'description': 'The category CP places the nzb in. Like <strong>movies</strong> or <strong>couchpotato</strong>',
}, },
{
'advanced': True,
'name': 'pp_directory',
'type': 'directory',
'description': 'Your Post-Processing Script directory, set in Sabnzbd > Config > Directories.',
},
{ {
'name': 'manual', 'name': 'manual',
'default': 0, 'default': 0,
couchpotato/core/downloaders/sabnzbd/main.py
+1 -67
View File
@@ -2,11 +2,6 @@ from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.helpers.variable import cleanHost
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from inspect import ismethod, isfunction
from tempfile import mkstemp
import base64
import os
import re
import traceback import traceback
log = CPLog(__name__) log = CPLog(__name__)
@@ -20,20 +15,7 @@ class Sabnzbd(Downloader):
if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): if self.isDisabled(manual) or not self.isCorrectType(data.get('type')):
return return
log.info("Sending '%s' to SABnzbd." % data.get('name')) log.info('Sending "%s" to SABnzbd.', data.get('name'))
if self.conf('ppDir') and data.get('imdb_id'):
try:
pp_script_fn = self.buildPp(data.get('imdb_id'))
except:
log.info("Failed to create post-processing script.")
pp_script_fn = False
if not pp_script_fn:
pp = False
else:
pp = True
else:
pp = False
params = { params = {
'apikey': self.conf('api_key'), 'apikey': self.conf('api_key'),
@@ -53,9 +35,6 @@ class Sabnzbd(Downloader):
else: else:
params['name'] = data.get('url') params['name'] = data.get('url')
if pp:
params['script'] = pp_script_fn
url = cleanHost(self.conf('host')) + "api?" + tryUrlencode(params) url = cleanHost(self.conf('host')) + "api?" + tryUrlencode(params)
try: try:
@@ -82,48 +61,3 @@ class Sabnzbd(Downloader):
else: else:
log.error("Unknown error: " + result[:40]) log.error("Unknown error: " + result[:40])
return False return False
def buildPp(self, imdb_id):
pp_script_path = self.getPpFile()
scriptB64 = '''IyEvdXNyL2Jpbi9weXRob24KaW1wb3J0IG9zCmltcG9ydCBzeXMKcHJpbnQgIkNyZWF0aW5nIGNwLmNw
bmZvIGZvciAlcyIgJSBzeXMuYXJndlsxXQppbWRiSWQgPSB7W0lNREJJREhFUkVdfQpwYXRoID0gb3Mu
cGF0aC5qb2luKHN5cy5hcmd2WzFdLCAiY3AuY3BuZm8iKQp0cnk6CiBmID0gb3BlbihwYXRoLCAndycp
CmV4Y2VwdCBJT0Vycm9yOgogcHJpbnQgIlVuYWJsZSB0byBvcGVuICVzIGZvciB3cml0aW5nIiAlIHBh
dGgKIHN5cy5leGl0KDEpCnRyeToKIGYud3JpdGUob3MucGF0aC5iYXNlbmFtZShzeXMuYXJndlswXSkr
IlxuIitpbWRiSWQpCmV4Y2VwdDoKIHByaW50ICJVbmFibGUgdG8gd3JpdGUgdG8gZmlsZTogJXMiICUg
cGF0aAogc3lzLmV4aXQoMikKZi5jbG9zZSgpCnByaW50ICJXcm90ZSBpbWRiIGlkLCAlcywgdG8gZmls
ZTogJXMiICUgKGltZGJJZCwgcGF0aCkK'''
script = re.sub(r"\{\[IMDBIDHERE\]\}", "'%s'" % imdb_id, base64.b64decode(scriptB64))
try:
f = open(pp_script_path, 'wb')
except:
log.info("Unable to open post-processing script for writing. Check permissions: %s" % pp_script_path)
return False
try:
f.write(script)
f.close()
except:
log.info("Unable to write to post-processing script. Check permissions: %s" % pp_script_path)
return False
log.info("Wrote post-processing script to: %s" % pp_script_path)
return os.path.basename(pp_script_path)
def getPpFile(self):
pp_script_handle, pp_script_path = mkstemp(suffix = '.py', dir = self.conf('ppDir'))
pp_sh = os.fdopen(pp_script_handle)
pp_sh.close()
try:
os.chmod(pp_script_path, int('777', 8))
except:
log.info("Unable to set post-processing script permissions to 777 (may still work correctly): %s" % pp_script_path)
return pp_script_path
couchpotato/core/downloaders/transmission/main.py
+3 -3
View File
@@ -16,7 +16,7 @@ class Transmission(Downloader):
if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): if self.isDisabled(manual) or not self.isCorrectType(data.get('type')):
return return
log.info('Sending "%s" to Transmission.' % data.get('name')) log.info('Sending "%s" to Transmission.', data.get('name'))
# Load host from config and split out port. # Load host from config and split out port.
host = self.conf('host').split(':') host = self.conf('host').split(':')
@@ -42,10 +42,10 @@ class Transmission(Downloader):
torrent.seed_ratio_limit = self.conf('ratio') torrent.seed_ratio_limit = self.conf('ratio')
torrent.seed_ratio_mode = 'single' if self.conf('ratio') else 'global' torrent.seed_ratio_mode = 'single' if self.conf('ratio') else 'global'
except transmissionrpc.TransmissionError, e: except transmissionrpc.TransmissionError, e:
log.error('Failed to change settings for transfer in transmission: %s' % e) log.error('Failed to change settings for transfer in transmission: %s', e)
return True return True
except transmissionrpc.TransmissionError, e: except transmissionrpc.TransmissionError, e:
log.error('Failed to send link to transmission: %s' % e) log.error('Failed to send link to transmission: %s', e)
return False return False
couchpotato/core/event.py
+6 -6
View File
@@ -12,7 +12,7 @@ def runHandler(name, handler, *args, **kwargs):
return handler(*args, **kwargs) return handler(*args, **kwargs)
except: except:
from couchpotato.environment import Env from couchpotato.environment import Env
log.error('Error in event "%s", that wasn\'t caught: %s%s' % (name, traceback.format_exc(), Env.all())) log.error('Error in event "%s", that wasn\'t caught: %s%s', (name, traceback.format_exc(), Env.all()))
def addEvent(name, handler, priority = 100): def addEvent(name, handler, priority = 100):
@@ -43,7 +43,7 @@ def removeEvent(name, handler):
def fireEvent(name, *args, **kwargs): def fireEvent(name, *args, **kwargs):
if not events.get(name): return if not events.get(name): return
#log.debug('Firing event %s' % name) #log.debug('Firing event %s', name)
try: try:
# Fire after event # Fire after event
@@ -100,7 +100,7 @@ def fireEvent(name, *args, **kwargs):
elif r[1]: elif r[1]:
errorHandler(r[1]) errorHandler(r[1])
else: else:
log.debug('Assume disabled eventhandler for: %s' % name) log.debug('Assume disabled eventhandler for: %s', name)
else: else:
results = [] results = []
@@ -130,7 +130,7 @@ def fireEvent(name, *args, **kwargs):
modified_results = fireEvent('result.modify.%s' % name, results, single = True) modified_results = fireEvent('result.modify.%s' % name, results, single = True)
if modified_results: if modified_results:
log.debug('Return modified results for %s' % name) log.debug('Return modified results for %s', name)
results = modified_results results = modified_results
if not is_after_event: if not is_after_event:
@@ -143,7 +143,7 @@ def fireEvent(name, *args, **kwargs):
except KeyError, e: except KeyError, e:
pass pass
except Exception: except Exception:
log.error('%s: %s' % (name, traceback.format_exc())) log.error('%s: %s', (name, traceback.format_exc()))
def fireEventAsync(*args, **kwargs): def fireEventAsync(*args, **kwargs):
try: try:
@@ -152,7 +152,7 @@ def fireEventAsync(*args, **kwargs):
my_thread.start() my_thread.start()
return True return True
except Exception, e: except Exception, e:
log.error('%s: %s' % (args[0], e)) log.error('%s: %s', (args[0], e))
def errorHandler(error): def errorHandler(error):
etype, value, tb = error etype, value, tb = error
couchpotato/core/helpers/encoding.py
+5 -1
View File
@@ -31,10 +31,14 @@ def toUnicode(original, *args):
except: except:
raise raise
except UnicodeDecodeError: except UnicodeDecodeError:
log.error('Unable to decode value: %s... ' % repr(original)[:20]) log.error('Unable to decode value: %s... ', repr(original)[:20])
ascii_text = str(original).encode('string_escape') ascii_text = str(original).encode('string_escape')
return toUnicode(ascii_text) return toUnicode(ascii_text)
def ss(original, *args):
from couchpotato.environment import Env
return toUnicode(original, *args).encode(Env.get('encoding'))
def ek(original, *args): def ek(original, *args):
if isinstance(original, (str, unicode)): if isinstance(original, (str, unicode)):
try: try:
couchpotato/core/helpers/rss.py
+1 -1
View File
@@ -47,5 +47,5 @@ class RSS(object):
try: try:
return XMLTree.parse(data).findall(path) return XMLTree.parse(data).findall(path)
except Exception, e: except Exception, e:
log.error('Error parsing RSS. %s' % e) log.error('Error parsing RSS. %s', e)
return [] return []
couchpotato/core/helpers/variable.py
+2 -2
View File
@@ -118,10 +118,10 @@ def getTitle(library_dict):
try: try:
return library_dict['titles'][0]['title'] return library_dict['titles'][0]['title']
except: except:
log.error('Could not get title for %s' % library_dict['identifier']) log.error('Could not get title for %s', library_dict['identifier'])
return None return None
except: except:
log.error('Could not get title for library item: %s' % library_dict) log.error('Could not get title for library item: %s', library_dict)
return None return None
def randomString(size = 8, chars = string.ascii_uppercase + string.digits): def randomString(size = 8, chars = string.ascii_uppercase + string.digits):
couchpotato/core/loader.py
+6 -6
View File
@@ -45,7 +45,7 @@ class Loader(object):
try: try:
m = getattr(self.loadModule(module_name), plugin.get('name')) m = getattr(self.loadModule(module_name), plugin.get('name'))
log.info("Loading %s: %s" % (plugin['type'], plugin['name'])) log.info('Loading %s: %s', (plugin['type'], plugin['name']))
# Save default settings for plugin/provider # Save default settings for plugin/provider
did_save += self.loadSettings(m, module_name, save = False) did_save += self.loadSettings(m, module_name, save = False)
@@ -57,10 +57,10 @@ class Loader(object):
log.error(e.message) log.error(e.message)
pass pass
# todo:: this needs to be more descriptive. # todo:: this needs to be more descriptive.
log.error('Import error, remove the empty folder: %s' % plugin.get('module')) log.error('Import error, remove the empty folder: %s', plugin.get('module'))
log.debug('Can\'t import %s: %s' % (module_name, traceback.format_exc())) log.debug('Can\'t import %s: %s', (module_name, traceback.format_exc()))
except: except:
log.error('Can\'t import %s: %s' % (module_name, traceback.format_exc())) log.error('Can\'t import %s: %s', (module_name, traceback.format_exc()))
if did_save: if did_save:
fireEvent('settings.save') fireEvent('settings.save')
@@ -84,7 +84,7 @@ class Loader(object):
fireEvent('settings.register', section_name = section['name'], options = options, save = save) fireEvent('settings.register', section_name = section['name'], options = options, save = save)
return True return True
except: except:
log.debug("Failed loading settings for '%s': %s" % (name, traceback.format_exc())) log.debug('Failed loading settings for "%s": %s', (name, traceback.format_exc()))
return False return False
def loadPlugins(self, module, name): def loadPlugins(self, module, name):
@@ -97,7 +97,7 @@ class Loader(object):
return True return True
except Exception, e: except Exception, e:
log.error("Failed loading plugin '%s': %s" % (module.__file__, traceback.format_exc())) log.error('Failed loading plugin "%s": %s', (module.__file__, traceback.format_exc()))
return False return False
def addModule(self, priority, plugin_type, module, name): def addModule(self, priority, plugin_type, module, name):
couchpotato/core/logger.py
+29 -17
View File
@@ -1,5 +1,6 @@
import logging import logging
import re import re
import traceback
class CPLog(object): class CPLog(object):
@@ -13,31 +14,42 @@ class CPLog(object):
self.context = context self.context = context
self.logger = logging.getLogger() self.logger = logging.getLogger()
def info(self, msg): def info(self, msg, replace_tuple = ()):
self.logger.info(self.addContext(msg)) self.logger.info(self.addContext(msg, replace_tuple))
def debug(self, msg): def debug(self, msg, replace_tuple = ()):
self.logger.debug(self.addContext(msg)) self.logger.debug(self.addContext(msg, replace_tuple))
def error(self, msg): def error(self, msg, replace_tuple = ()):
self.logger.error(self.addContext(msg)) self.logger.error(self.addContext(msg, replace_tuple))
def warning(self, msg): def warning(self, msg, replace_tuple = ()):
self.logger.warning(self.addContext(msg)) self.logger.warning(self.addContext(msg, replace_tuple))
def critical(self, msg): def critical(self, msg, replace_tuple = ()):
self.logger.critical(self.addContext(msg), exc_info = 1) self.logger.critical(self.addContext(msg, replace_tuple), exc_info = 1)
def addContext(self, msg): def addContext(self, msg, replace_tuple = ()):
return '[%+25.25s] %s' % (self.context[-25:], self.removePrivateData(msg)) return '[%+25.25s] %s' % (self.context[-25:], self.safeMessage(msg, replace_tuple))
def removePrivateData(self, msg): def safeMessage(self, msg, replace_tuple = ()):
try:
msg = unicode(msg)
except:
pass
from couchpotato.environment import Env from couchpotato.environment import Env
from couchpotato.core.helpers.encoding import ss
msg = ss(msg)
try:
msg = msg % replace_tuple
except:
try:
if isinstance(replace_tuple, tuple):
msg = msg % tuple([ss(x) for x in list(replace_tuple)])
else:
msg = msg % ss(replace_tuple)
except:
self.error('Failed encoding stuff to log: %s' % traceback.format_exc())
if not Env.get('dev'): if not Env.get('dev'):
for replace in self.replace_private: for replace in self.replace_private:
couchpotato/core/notifications/base.py
+1 -1
View File
@@ -41,7 +41,7 @@ class Notification(Plugin):
test_type = self.testNotifyName() test_type = self.testNotifyName()
log.info('Sending test to %s' % test_type) log.info('Sending test to %s', test_type)
success = self.notify( success = self.notify(
message = self.test_message, message = self.test_message,
couchpotato/core/notifications/growl/main.py
+1 -1
View File
@@ -38,7 +38,7 @@ class Growl(Notification):
self.growl.register() self.growl.register()
self.registered = True self.registered = True
except: except:
log.error('Failed register of growl: %s' % traceback.format_exc()) log.error('Failed register of growl: %s', traceback.format_exc())
def notify(self, message = '', data = {}, listener = None): def notify(self, message = '', data = {}, listener = None):
if self.isDisabled(): return if self.isDisabled(): return
couchpotato/core/notifications/nmj/main.py
+8 -8
View File
@@ -33,10 +33,10 @@ class NMJ(Notification):
try: try:
terminal = telnetlib.Telnet(host) terminal = telnetlib.Telnet(host)
except Exception: except Exception:
log.error('Warning: unable to get a telnet session to %s' % (host)) log.error('Warning: unable to get a telnet session to %s', (host))
return self.failed() return self.failed()
log.debug('Connected to %s via telnet' % (host)) log.debug('Connected to %s via telnet', (host))
terminal.read_until('sh-3.00# ') terminal.read_until('sh-3.00# ')
terminal.write('cat /tmp/source\n') terminal.write('cat /tmp/source\n')
terminal.write('cat /tmp/netshare\n') terminal.write('cat /tmp/netshare\n')
@@ -48,9 +48,9 @@ class NMJ(Notification):
if match: if match:
database = match.group(1) database = match.group(1)
device = match.group(2) device = match.group(2)
log.info('Found NMJ database %s on device %s' % (database, device)) log.info('Found NMJ database %s on device %s', (database, device))
else: else:
log.error('Could not get current NMJ database on %s, NMJ is probably not running!' % (host)) log.error('Could not get current NMJ database on %s, NMJ is probably not running!', (host))
return self.failed() return self.failed()
if device.startswith('NETWORK_SHARE/'): if device.startswith('NETWORK_SHARE/'):
@@ -58,7 +58,7 @@ class NMJ(Notification):
if match: if match:
mount = match.group().replace('127.0.0.1', host) mount = match.group().replace('127.0.0.1', host)
log.info('Found mounting url on the Popcorn Hour in configuration: %s' % (mount)) log.info('Found mounting url on the Popcorn Hour in configuration: %s', (mount))
else: else:
log.error('Detected a network share on the Popcorn Hour, but could not get the mounting url') log.error('Detected a network share on the Popcorn Hour, but could not get the mounting url')
return self.failed() return self.failed()
@@ -77,7 +77,7 @@ class NMJ(Notification):
database = self.conf('database') database = self.conf('database')
if self.mount: if self.mount:
log.debug('Try to mount network drive via url: %s' % (mount)) log.debug('Try to mount network drive via url: %s', (mount))
try: try:
data = self.urlopen(mount) data = self.urlopen(mount)
except: except:
@@ -102,11 +102,11 @@ class NMJ(Notification):
et = etree.fromstring(response) et = etree.fromstring(response)
result = et.findtext('returnValue') result = et.findtext('returnValue')
except SyntaxError, e: except SyntaxError, e:
log.error('Unable to parse XML returned from the Popcorn Hour: %s' % (e)) log.error('Unable to parse XML returned from the Popcorn Hour: %s', (e))
return False return False
if int(result) > 0: if int(result) > 0:
log.error('Popcorn Hour returned an errorcode: %s' % (result)) log.error('Popcorn Hour returned an errorcode: %s', (result))
return False return False
else: else:
log.info('NMJ started background scan') log.info('NMJ started background scan')
couchpotato/core/notifications/notifo/main.py
+1 -1
View File
@@ -32,7 +32,7 @@ class Notifo(Notification):
raise Exception raise Exception
except: except:
log.error('Notification failed: %s' % traceback.format_exc()) log.error('Notification failed: %s', traceback.format_exc())
return False return False
log.info('Notifo notification successful.') log.info('Notifo notification successful.')
couchpotato/core/notifications/notifymyandroid/main.py
+1 -1
View File
@@ -23,6 +23,6 @@ class NotifyMyAndroid(Notification):
for key in keys: for key in keys:
if not response[str(key)]['code'] == u'200': if not response[str(key)]['code'] == u'200':
log.error('Could not send notification to NotifyMyAndroid (%s). %s' % (key, response[key]['message'])) log.error('Could not send notification to NotifyMyAndroid (%s). %s', (key, response[key]['message']))
return response return response
couchpotato/core/notifications/notifymywp/main.py
+1 -1
View File
@@ -17,7 +17,7 @@ class NotifyMyWP(Notification):
for key in keys: for key in keys:
if not response[key]['Code'] == u'200': if not response[key]['Code'] == u'200':
log.error('Could not send notification to NotifyMyWindowsPhone (%s). %s' % (key, response[key]['message'])) log.error('Could not send notification to NotifyMyWindowsPhone (%s). %s', (key, response[key]['message']))
return False return False
return response return response
couchpotato/core/notifications/plex/__init__.py
+3 -2
View File
@@ -17,8 +17,9 @@ config = [{
}, },
{ {
'name': 'host', 'name': 'host',
'default': 'localhost:32400', 'default': 'localhost',
'description': 'Default should be on localhost:32400', 'description': 'Default should be on localhost',
'advanced': True,
}, },
], ],
} }
couchpotato/core/notifications/plex/main.py
+27 -2
View File
@@ -1,4 +1,5 @@
from couchpotato.core.event import addEvent from couchpotato.core.event import addEvent
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.helpers.variable import cleanHost
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.notifications.base import Notification from couchpotato.core.notifications.base import Notification
@@ -11,13 +12,14 @@ log = CPLog(__name__)
class Plex(Notification): class Plex(Notification):
def __init__(self): def __init__(self):
super(Plex, self).__init__()
addEvent('renamer.after', self.addToLibrary) addEvent('renamer.after', self.addToLibrary)
def addToLibrary(self, group = {}): def addToLibrary(self, group = {}):
if self.isDisabled(): return if self.isDisabled(): return
log.info('Sending notification to Plex') log.info('Sending notification to Plex')
hosts = [cleanHost(x.strip()) for x in self.conf('host').split(",")] hosts = [cleanHost(x.strip() + ':32400') for x in self.conf('host').split(",")]
for host in hosts: for host in hosts:
@@ -36,7 +38,30 @@ class Plex(Notification):
x = self.urlopen(url) x = self.urlopen(url)
except: except:
log.error('Plex library update failed for %s: %s' % (host, traceback.format_exc())) log.error('Plex library update failed for %s: %s', (host, traceback.format_exc()))
return False return False
return True return True
def notify(self, message = '', data = {}, listener = None):
if self.isDisabled(): return
for host in [x.strip() + ':3000' for x in self.conf('host').split(",")]:
self.send({'command': 'ExecBuiltIn', 'parameter': 'Notification(CouchPotato, %s)' % message}, host)
return True
def send(self, command, host):
url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, tryUrlencode(command))
headers = {}
try:
self.urlopen(url, headers = headers, show_error = False)
except:
log.error("Couldn't sent command to Plex")
return False
log.info('Plex notification to %s successful.', host)
return True
couchpotato/core/notifications/prowl/main.py
+1 -1
View File
@@ -32,7 +32,7 @@ class Prowl(Notification):
log.info('Prowl notifications sent.') log.info('Prowl notifications sent.')
return True return True
elif request_status == 401: elif request_status == 401:
log.error('Prowl auth failed: %s' % response.reason) log.error('Prowl auth failed: %s', response.reason)
return False return False
else: else:
log.error('Prowl notification failed.') log.error('Prowl notification failed.')
couchpotato/core/notifications/pushover/main.py
+1 -1
View File
@@ -35,7 +35,7 @@ class Pushover(Notification):
log.info('Pushover notifications sent.') log.info('Pushover notifications sent.')
return True return True
elif request_status == 401: elif request_status == 401:
log.error('Pushover auth failed: %s' % response.reason) log.error('Pushover auth failed: %s', response.reason)
return False return False
else: else:
log.error('Pushover notification failed.') log.error('Pushover notification failed.')
couchpotato/core/notifications/synoindex/main.py
+3 -3
View File
@@ -15,14 +15,14 @@ class Synoindex(Notification):
if self.isDisabled(): return if self.isDisabled(): return
command = ['/usr/syno/bin/synoindex', '-A', group.get('destination_dir')] command = ['/usr/syno/bin/synoindex', '-A', group.get('destination_dir')]
log.info(u'Executing synoindex command: %s ' % command) log.info(u'Executing synoindex command: %s ', command)
try: try:
p = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) p = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
out = p.communicate() out = p.communicate()
log.info('Result from synoindex: %s' % str(out)) log.info('Result from synoindex: %s', str(out))
return True return True
except OSError, e: except OSError, e:
log.error('Unable to run synoindex: %s' % e) log.error('Unable to run synoindex: %s', e)
return False return False
return True return True
couchpotato/core/notifications/twitter/main.py
+5 -5
View File
@@ -55,7 +55,7 @@ class Twitter(Notification):
else: else:
api.PostUpdate('[%s] %s' % (self.default_title, message)) api.PostUpdate('[%s] %s' % (self.default_title, message))
except Exception, e: except Exception, e:
log.error('Error sending tweet: %s' % e) log.error('Error sending tweet: %s', e)
return False return False
return True return True
@@ -71,7 +71,7 @@ class Twitter(Notification):
resp, content = oauth_client.request(self.urls['request'], 'POST', body = tryUrlencode({'oauth_callback': callback_url})) resp, content = oauth_client.request(self.urls['request'], 'POST', body = tryUrlencode({'oauth_callback': callback_url}))
if resp['status'] != '200': if resp['status'] != '200':
log.error('Invalid response from Twitter requesting temp token: %s' % resp['status']) log.error('Invalid response from Twitter requesting temp token: %s', resp['status'])
return jsonified({ return jsonified({
'success': False, 'success': False,
}) })
@@ -80,7 +80,7 @@ class Twitter(Notification):
auth_url = self.urls['authorize'] + ("?oauth_token=%s" % self.request_token['oauth_token']) auth_url = self.urls['authorize'] + ("?oauth_token=%s" % self.request_token['oauth_token'])
log.info('Redirecting to "%s"' % auth_url) log.info('Redirecting to "%s"', auth_url)
return jsonified({ return jsonified({
'success': True, 'success': True,
'url': auth_url, 'url': auth_url,
@@ -100,10 +100,10 @@ class Twitter(Notification):
access_token = dict(parse_qsl(content)) access_token = dict(parse_qsl(content))
if resp['status'] != '200': if resp['status'] != '200':
log.error('The request for an access token did not succeed: %s' % resp['status']) log.error('The request for an access token did not succeed: %s', resp['status'])
return 'Twitter auth failed' return 'Twitter auth failed'
else: else:
log.debug('Tokens: %s, %s' % (access_token['oauth_token'], access_token['oauth_token_secret'])) log.debug('Tokens: %s, %s', (access_token['oauth_token'], access_token['oauth_token_secret']))
self.conf('access_token_key', value = access_token['oauth_token']) self.conf('access_token_key', value = access_token['oauth_token'])
self.conf('access_token_secret', value = access_token['oauth_token_secret']) self.conf('access_token_secret', value = access_token['oauth_token_secret'])
couchpotato/core/notifications/xbmc/main.py
+1 -1
View File
@@ -35,5 +35,5 @@ class XBMC(Notification):
log.error("Couldn't sent command to XBMC") log.error("Couldn't sent command to XBMC")
return False return False
log.info('XBMC notification to %s successful.' % host) log.info('XBMC notification to %s successful.', host)
return True return True
couchpotato/core/plugins/base.py
+15 -12
View File
@@ -1,6 +1,6 @@
from couchpotato import addView from couchpotato import addView
from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import tryUrlencode, simplifyString from couchpotato.core.helpers.encoding import tryUrlencode, simplifyString, ss
from couchpotato.core.helpers.variable import getExt from couchpotato.core.helpers.variable import getExt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.environment import Env from couchpotato.environment import Env
@@ -71,6 +71,7 @@ class Plugin(object):
return send_from_directory(d, filename) return send_from_directory(d, filename)
def createFile(self, path, content, binary = False): def createFile(self, path, content, binary = False):
path = ss(path)
self.makeDir(os.path.dirname(path)) self.makeDir(os.path.dirname(path))
@@ -80,15 +81,16 @@ class Plugin(object):
f.close() f.close()
os.chmod(path, Env.getPermission('file')) os.chmod(path, Env.getPermission('file'))
except Exception, e: except Exception, e:
log.error('Unable writing to file "%s": %s' % (path, e)) log.error('Unable writing to file "%s": %s', (path, e))
def makeDir(self, path): def makeDir(self, path):
path = ss(path)
try: try:
if not os.path.isdir(path): if not os.path.isdir(path):
os.makedirs(path, Env.getPermission('folder')) os.makedirs(path, Env.getPermission('folder'))
return True return True
except Exception, e: except Exception, e:
log.error('Unable to create folder "%s": %s' % (path, e)) log.error('Unable to create folder "%s": %s', (path, e))
return False return False
@@ -106,7 +108,7 @@ class Plugin(object):
# Don't try for failed requests # Don't try for failed requests
if self.http_failed_disabled.get(host, 0) > 0: if self.http_failed_disabled.get(host, 0) > 0:
if self.http_failed_disabled[host] > (time.time() - 900): if self.http_failed_disabled[host] > (time.time() - 900):
log.info('Disabled calls to %s for 15 minutes because so many failed requests.' % host) log.info('Disabled calls to %s for 15 minutes because so many failed requests.', host)
raise Exception raise Exception
else: else:
del self.http_failed_request[host] del self.http_failed_request[host]
@@ -116,7 +118,7 @@ class Plugin(object):
try: try:
if multipart: if multipart:
log.info('Opening multipart url: %s, params: %s' % (url, [x for x in params.iterkeys()])) log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()]))
request = urllib2.Request(url, params, headers) request = urllib2.Request(url, params, headers)
cookies = cookielib.CookieJar() cookies = cookielib.CookieJar()
@@ -124,7 +126,7 @@ class Plugin(object):
data = opener.open(request, timeout = timeout).read() data = opener.open(request, timeout = timeout).read()
else: else:
log.info('Opening url: %s, params: %s' % (url, [x for x in params.iterkeys()])) log.info('Opening url: %s, params: %s', (url, [x for x in params.iterkeys()]))
data = tryUrlencode(params) if len(params) > 0 else None data = tryUrlencode(params) if len(params) > 0 else None
request = urllib2.Request(url, data, headers) request = urllib2.Request(url, data, headers)
@@ -133,7 +135,7 @@ class Plugin(object):
self.http_failed_request[host] = 0 self.http_failed_request[host] = 0
except IOError: except IOError:
if show_error: if show_error:
log.error('Failed opening url in %s: %s %s' % (self.getName(), url, traceback.format_exc(1))) log.error('Failed opening url in %s: %s %s', (self.getName(), url, traceback.format_exc(1)))
# Save failed requests by hosts # Save failed requests by hosts
try: try:
@@ -147,7 +149,7 @@ class Plugin(object):
self.http_failed_disabled[host] = time.time() self.http_failed_disabled[host] = time.time()
except: except:
log.debug('Failed logging failed requests for %s: %s' % (url, traceback.format_exc())) log.debug('Failed logging failed requests for %s: %s', (url, traceback.format_exc()))
raise raise
@@ -163,7 +165,7 @@ class Plugin(object):
wait = math.ceil(last_use - now + self.http_time_between_calls) wait = math.ceil(last_use - now + self.http_time_between_calls)
if wait > 0: if wait > 0:
log.debug('Waiting for %s, %d seconds' % (self.getName(), wait)) log.debug('Waiting for %s, %d seconds', (self.getName(), wait))
time.sleep(last_use - now + self.http_time_between_calls) time.sleep(last_use - now + self.http_time_between_calls)
def beforeCall(self, handler): def beforeCall(self, handler):
@@ -202,7 +204,7 @@ class Plugin(object):
cache_key = simplifyString(cache_key) cache_key = simplifyString(cache_key)
cache = Env.get('cache').get(cache_key) cache = Env.get('cache').get(cache_key)
if cache: if cache:
if not Env.get('dev'): log.debug('Getting cache %s' % cache_key) if not Env.get('dev'): log.debug('Getting cache %s', cache_key)
return cache return cache
if url: if url:
@@ -214,13 +216,14 @@ class Plugin(object):
del kwargs['cache_timeout'] del kwargs['cache_timeout']
data = self.urlopen(url, **kwargs) data = self.urlopen(url, **kwargs)
self.setCache(cache_key, data, timeout = cache_timeout) if data:
self.setCache(cache_key, data, timeout = cache_timeout)
return data return data
except: except:
pass pass
def setCache(self, cache_key, value, timeout = 300): def setCache(self, cache_key, value, timeout = 300):
log.debug('Setting cache %s' % cache_key) log.debug('Setting cache %s', cache_key)
Env.get('cache').set(cache_key, value, timeout) Env.get('cache').set(cache_key, value, timeout)
return value return value
couchpotato/core/plugins/file/main.py
+2
View File
@@ -8,6 +8,7 @@ from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import FileType, File from couchpotato.core.settings.model import FileType, File
from couchpotato.environment import Env from couchpotato.environment import Env
import os.path import os.path
import traceback
log = CPLog(__name__) log = CPLog(__name__)
@@ -46,6 +47,7 @@ class FileManager(Plugin):
try: try:
filedata = self.urlopen(url, **urlopen_kwargs) filedata = self.urlopen(url, **urlopen_kwargs)
except: except:
log.error('Failed downloading file %s: %s', (url, traceback.format_exc()))
return False return False
self.createFile(dest, filedata, binary = True) self.createFile(dest, filedata, binary = True)
couchpotato/core/plugins/library/main.py
+10 -9
View File
@@ -78,7 +78,7 @@ class LibraryPlugin(Plugin):
except: pass except: pass
if not info or len(info) == 0: if not info or len(info) == 0:
log.error('Could not update, no movie info to work with: %s' % identifier) log.error('Could not update, no movie info to work with: %s', identifier)
return False return False
# Main info # Main info
@@ -95,7 +95,7 @@ class LibraryPlugin(Plugin):
db.commit() db.commit()
titles = info.get('titles', []) titles = info.get('titles', [])
log.debug('Adding titles: %s' % titles) log.debug('Adding titles: %s', titles)
for title in titles: for title in titles:
if not title: if not title:
continue continue
@@ -117,13 +117,14 @@ class LibraryPlugin(Plugin):
continue continue
file_path = fireEvent('file.download', url = image, single = True) file_path = fireEvent('file.download', url = image, single = True)
file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', type), single = True) if file_path:
try: file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', type), single = True)
file_obj = db.query(File).filter_by(id = file_obj.get('id')).one() try:
library.files.append(file_obj) file_obj = db.query(File).filter_by(id = file_obj.get('id')).one()
db.commit() library.files.append(file_obj)
except: db.commit()
log.debug('Failed to attach to library: %s' % traceback.format_exc()) except:
log.debug('Failed to attach to library: %s', traceback.format_exc())
library_dict = library.to_dict(self.default_dict) library_dict = library.to_dict(self.default_dict)
couchpotato/core/plugins/log/main.py
+5 -4
View File
@@ -1,5 +1,7 @@
from couchpotato.api import addApiView from couchpotato.api import addApiView
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.helpers.request import jsonified, getParam, getParams from couchpotato.core.helpers.request import jsonified, getParam, getParams
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env from couchpotato.environment import Env
@@ -78,7 +80,7 @@ class Logging(Plugin):
def partial(self): def partial(self):
log_type = getParam('type', 'all') log_type = getParam('type', 'all')
total_lines = getParam('lines', 30) total_lines = tryInt(getParam('lines', 30))
log_lines = [] log_lines = []
@@ -92,13 +94,12 @@ class Logging(Plugin):
reversed_lines = [] reversed_lines = []
f = open(path, 'r') f = open(path, 'r')
reversed_lines = f.read().split('[0m\n') reversed_lines = toUnicode(f.read()).split('[0m\n')
reversed_lines.reverse() reversed_lines.reverse()
brk = False brk = False
for line in reversed_lines: for line in reversed_lines:
#print '%s ' % log_type in line.lower()
if log_type == 'all' or '%s ' % log_type.upper() in line: if log_type == 'all' or '%s ' % log_type.upper() in line:
log_lines.append(line) log_lines.append(line)
@@ -149,7 +150,7 @@ class Logging(Plugin):
except: except:
log.error(log_message) log.error(log_message)
except: except:
log.error('Couldn\'t log via API: %s' % params) log.error('Couldn\'t log via API: %s', params)
return jsonified({ return jsonified({
couchpotato/core/plugins/manage/main.py
+11 -10
View File
@@ -1,6 +1,6 @@
from couchpotato.api import addApiView from couchpotato.api import addApiView
from couchpotato.core.event import fireEvent, addEvent, fireEventAsync from couchpotato.core.event import fireEvent, addEvent, fireEventAsync
from couchpotato.core.helpers.request import jsonified, getParams from couchpotato.core.helpers.request import jsonified, getParam
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env from couchpotato.environment import Env
@@ -25,13 +25,14 @@ class Manage(Plugin):
}) })
if not Env.get('dev'): if not Env.get('dev'):
addEvent('app.load', self.updateLibrary) def updateLibrary():
self.updateLibrary(full = False)
addEvent('app.load', updateLibrary)
def updateLibraryView(self): def updateLibraryView(self):
params = getParams() full = getParam('full', default = 1)
fireEventAsync('manage.update', full = True if full == '1' else False)
fireEventAsync('manage.update', full = params.get('full', True))
return jsonified({ return jsonified({
'success': True 'success': True
@@ -51,11 +52,11 @@ class Manage(Plugin):
if not os.path.isdir(directory): if not os.path.isdir(directory):
if len(directory) > 0: if len(directory) > 0:
log.error('Directory doesn\'t exist: %s' % directory) log.error('Directory doesn\'t exist: %s', directory)
continue continue
log.info('Updating manage library: %s' % directory) log.info('Updating manage library: %s', directory)
identifiers = fireEvent('scanner.folder', folder = directory, newer_than = last_update, single = True) identifiers = fireEvent('scanner.folder', folder = directory, newer_than = last_update if not full else 0, single = True)
if identifiers: if identifiers:
added_identifiers.extend(identifiers) added_identifiers.extend(identifiers)
@@ -67,11 +68,11 @@ class Manage(Plugin):
if self.conf('cleanup') and full and not self.shuttingDown(): if self.conf('cleanup') and full and not self.shuttingDown():
# Get movies with done status # Get movies with done status
done_movies = fireEvent('movie.list', status = 'done', single = True) total_movies, done_movies = fireEvent('movie.list', status = 'done', single = True)
for done_movie in done_movies: for done_movie in done_movies:
if done_movie['library']['identifier'] not in added_identifiers: if done_movie['library']['identifier'] not in added_identifiers:
fireEvent('movie.delete', movie_id = done_movie['id']) fireEvent('movie.delete', movie_id = done_movie['id'], delete_from = 'all')
Env.prop('manage.last_update', time.time()) Env.prop('manage.last_update', time.time())
couchpotato/core/plugins/movie/main.py
+14 -8
View File
@@ -130,6 +130,8 @@ class MoviePlugin(Plugin):
.filter(or_(*[Movie.status.has(identifier = s) for s in status])) \ .filter(or_(*[Movie.status.has(identifier = s) for s in status])) \
.group_by(Movie.id) .group_by(Movie.id)
total_count = q.count()
filter_or = [] filter_or = []
if starts_with: if starts_with:
starts_with = toUnicode(starts_with.lower()) starts_with = toUnicode(starts_with.lower())
@@ -156,8 +158,7 @@ class MoviePlugin(Plugin):
.options(joinedload_all('library.titles')) \ .options(joinedload_all('library.titles')) \
.options(joinedload_all('library.files')) \ .options(joinedload_all('library.files')) \
.options(joinedload_all('status')) \ .options(joinedload_all('status')) \
.options(joinedload_all('files')) \ .options(joinedload_all('files'))
if limit_offset: if limit_offset:
splt = [x.strip() for x in limit_offset.split(',')] splt = [x.strip() for x in limit_offset.split(',')]
@@ -165,7 +166,6 @@ class MoviePlugin(Plugin):
offset = 0 if len(splt) is 1 else splt[1] offset = 0 if len(splt) is 1 else splt[1]
q2 = q2.limit(limit).offset(offset) q2 = q2.limit(limit).offset(offset)
results = q2.all() results = q2.all()
movies = [] movies = []
for movie in results: for movie in results:
@@ -178,7 +178,7 @@ class MoviePlugin(Plugin):
movies.append(temp) movies.append(temp)
#db.close() #db.close()
return movies return (total_count, movies)
def availableChars(self, status = ['active']): def availableChars(self, status = ['active']):
@@ -214,11 +214,12 @@ class MoviePlugin(Plugin):
starts_with = params.get('starts_with', None) starts_with = params.get('starts_with', None)
search = params.get('search', None) search = params.get('search', None)
movies = self.list(status = status, limit_offset = limit_offset, starts_with = starts_with, search = search) total_movies, movies = self.list(status = status, limit_offset = limit_offset, starts_with = starts_with, search = search)
return jsonified({ return jsonified({
'success': True, 'success': True,
'empty': len(movies) == 0, 'empty': len(movies) == 0,
'total': total_movies,
'movies': movies, 'movies': movies,
}) })
@@ -279,6 +280,10 @@ class MoviePlugin(Plugin):
def add(self, params = {}, force_readd = True, search_after = True): def add(self, params = {}, force_readd = True, search_after = True):
if not params.get('identifier'):
log.error('Can\'t add movie without imdb identifier.')
return False
library = fireEvent('library.add', single = True, attrs = params, update_after = False) library = fireEvent('library.add', single = True, attrs = params, update_after = False)
# Status # Status
@@ -314,7 +319,7 @@ class MoviePlugin(Plugin):
m.profile_id = params.get('profile_id', default_profile.get('id')) m.profile_id = params.get('profile_id', default_profile.get('id'))
else: else:
log.debug('Movie already exists, not updating: %s' % params) log.debug('Movie already exists, not updating: %s', params)
added = False added = False
if force_readd: if force_readd:
@@ -351,7 +356,7 @@ class MoviePlugin(Plugin):
return jsonified({ return jsonified({
'success': True, 'success': True,
'added': True, 'added': True if movie_dict else False,
'movie': movie_dict, 'movie': movie_dict,
}) })
@@ -436,6 +441,7 @@ class MoviePlugin(Plugin):
db.commit() db.commit()
elif new_movie_status: elif new_movie_status:
new_status = fireEvent('status.get', new_movie_status, single = True) new_status = fireEvent('status.get', new_movie_status, single = True)
movie.profile_id = None
movie.status_id = new_status.get('id') movie.status_id = new_status.get('id')
db.commit() db.commit()
else: else:
@@ -456,7 +462,7 @@ class MoviePlugin(Plugin):
log.debug('Can\'t restatus movie, doesn\'t seem to exist.') log.debug('Can\'t restatus movie, doesn\'t seem to exist.')
return False return False
log.debug('Changing status for %s' % (m.library.titles[0].title)) log.debug('Changing status for %s', (m.library.titles[0].title))
if not m.profile: if not m.profile:
m.status_id = done_status.get('id') m.status_id = done_status.get('id')
else: else:
couchpotato/core/plugins/movie/static/list.js
+14 -2
View File
@@ -72,7 +72,7 @@ var MovieList = new Class({
self.created = true; self.created = true;
}, },
addMovies: function(movies){ addMovies: function(movies, total){
var self = this; var self = this;
if(!self.created) self.create(); if(!self.created) self.create();
@@ -86,8 +86,19 @@ var MovieList = new Class({
Object.each(movies, function(movie){ Object.each(movies, function(movie){
self.createMovie(movie); self.createMovie(movie);
}); });
self.setCounter(total);
}, },
setCounter: function(count){
var self = this;
if(!self.navigation_counter) return;
self.navigation_counter.set('text', (count || 0));
},
createMovie: function(movie, inject_at){ createMovie: function(movie, inject_at){
var self = this; var self = this;
@@ -118,6 +129,7 @@ var MovieList = new Class({
self.navigation = new Element('div.alph_nav').adopt( self.navigation = new Element('div.alph_nav').adopt(
self.navigation_actions = new Element('ul.inlay.actions.reversed'), self.navigation_actions = new Element('ul.inlay.actions.reversed'),
self.navigation_counter = new Element('span.counter[title=Total]'),
self.navigation_alpha = new Element('ul.numbers', { self.navigation_alpha = new Element('ul.numbers', {
'events': { 'events': {
'click:relay(li)': function(e, el){ 'click:relay(li)': function(e, el){
@@ -443,7 +455,7 @@ var MovieList = new Class({
}, self.filter), }, self.filter),
'onComplete': function(json){ 'onComplete': function(json){
self.store(json.movies); self.store(json.movies);
self.addMovies(json.movies); self.addMovies(json.movies, json.total);
self.load_more.set('text', 'load more movies'); self.load_more.set('text', 'load more movies');
if(self.scrollspy) self.scrollspy.start(); if(self.scrollspy) self.scrollspy.start();
} }
couchpotato/core/plugins/movie/static/movie.css
+51 -6
View File
@@ -13,6 +13,7 @@
overflow: hidden; overflow: hidden;
width: 100%; width: 100%;
transition: all 0.2s linear; transition: all 0.2s linear;
transform: translateZ(0);
} }
.movies .movie.list_view, .movies .movie.mass_edit_view { .movies .movie.list_view, .movies .movie.mass_edit_view {
margin: 1px 0; margin: 1px 0;
@@ -24,6 +25,10 @@
.movies .movie.list_view:hover, .movies .movie.mass_edit_view:hover { .movies .movie.list_view:hover, .movies .movie.mass_edit_view:hover {
background: rgba(255,255,255,0.03); background: rgba(255,255,255,0.03);
} }
.movies .movie_container {
overflow: hidden;
}
.movies .data { .movies .data {
padding: 20px; padding: 20px;
@@ -32,8 +37,8 @@
position: relative; position: relative;
float: right; float: right;
border-radius: 0; border-radius: 0;
overflow: hidden;
transition: all 0.2s linear; transition: all 0.2s linear;
transform: translateZ(0);
} }
.movies .list_view .data, .movies .mass_edit_view .data { .movies .list_view .data, .movies .mass_edit_view .data {
height: 30px; height: 30px;
@@ -62,6 +67,7 @@
height: 180px; height: 180px;
border-radius: 4px 0 0 4px; border-radius: 4px 0 0 4px;
transition: all 0.2s linear; transition: all 0.2s linear;
transform: translateZ(0);
} }
.movies .list_view .poster, .movies .mass_edit_view .poster { .movies .list_view .poster, .movies .mass_edit_view .poster {
@@ -84,6 +90,7 @@
float: left; float: left;
width: 90%; width: 90%;
transition: all 0.2s linear; transition: all 0.2s linear;
transform: translateZ(0);
} }
.movies .list_view .info .title, .movies .mass_edit_view .info .title { .movies .list_view .info .title, .movies .mass_edit_view .info .title {
font-size: 16px; font-size: 16px;
@@ -100,6 +107,7 @@
width: 10%; width: 10%;
text-align: right; text-align: right;
transition: all 0.2s linear; transition: all 0.2s linear;
transform: translateZ(0);
} }
.movies .list_view .info .year, .movies .mass_edit_view .info .year { .movies .list_view .info .year, .movies .mass_edit_view .info .year {
font-size: 16px; font-size: 16px;
@@ -310,6 +318,35 @@
padding-bottom: 4px; padding-bottom: 4px;
height: auto; height: auto;
} }
.movies .movie .trailer_container {
width: 100%;
background: #000;
text-align: center;
transition: all .6s cubic-bezier(0.9,0,0.1,1);
transform: translateZ(0);
overflow: hidden;
}
.movies .movie .trailer_container.hide {
height: 0 !important;
}
.movies .movie .hide_trailer {
position: absolute;
top: 0;
left: 50%;
margin-left: -50px;
width: 100px;
text-align: center;
padding: 3px 10px;
background: #4e5969;
border-radius: 0 0 2px 2px;
transition: all .6s cubic-bezier(0.9,0,0.1,1) .2s;
transform: translateZ(0);
}
.movies .movie .hide_trailer.hide {
top: -30px;
}
.movies .load_more { .movies .load_more {
display: block; display: block;
@@ -323,6 +360,7 @@
.movies .alph_nav { .movies .alph_nav {
transition: box-shadow .4s linear; transition: box-shadow .4s linear;
transform: translateZ(0);
position: fixed; position: fixed;
z-index: 2; z-index: 2;
top: 0; top: 0;
@@ -338,7 +376,9 @@
background: #4e5969; background: #4e5969;
} }
.movies .alph_nav ul.numbers, .movies .alph_nav ul.actions { .movies .alph_nav ul.numbers,
.movies .alph_nav .counter,
.movies .alph_nav ul.actions {
list-style: none; list-style: none;
padding: 0 0 1px; padding: 0 0 1px;
margin: 0; margin: 0;
@@ -346,10 +386,15 @@
user-select: none; user-select: none;
} }
.movies .alph_nav .counter {
width: 60px;
text-align: center;
}
.movies .alph_nav .numbers li, .movies .alph_nav .actions li { .movies .alph_nav .numbers li, .movies .alph_nav .actions li {
display: inline-block; display: inline-block;
vertical-align: top; vertical-align: top;
width: 22px; width: 20px;
height: 24px; height: 24px;
line-height: 26px; line-height: 26px;
text-align: center; text-align: center;
@@ -357,11 +402,11 @@
color: rgba(255,255,255,0.2); color: rgba(255,255,255,0.2);
border: 1px solid transparent; border: 1px solid transparent;
transition: all 0.1s ease-in-out; transition: all 0.1s ease-in-out;
transform: translateZ(0);
text-shadow: none; text-shadow: none;
} }
.movies .alph_nav .numbers li:first-child { .movies .alph_nav .numbers li:first-child {
width: 43px; width: 43px;
margin-left: 7px;
} }
.movies .alph_nav li.available { .movies .alph_nav li.available {
color: rgba(255,255,255,0.8); color: rgba(255,255,255,0.8);
@@ -370,8 +415,8 @@
} }
.movies .alph_nav li.active.available, .movies .alph_nav li.available:hover { .movies .alph_nav li.active.available, .movies .alph_nav li.available:hover {
color: #fff; color: #fff;
font-size: 24px; font-size: 20px;
line-height: 24px; line-height: 20px;
} }
.movies .alph_nav input { .movies .alph_nav input {
couchpotato/core/plugins/movie/static/movie.js
+110 -1
View File
@@ -77,7 +77,7 @@ var Movie = new Class({
self.profile = Quality.getProfile(self.data.profile_id) || {}; self.profile = Quality.getProfile(self.data.profile_id) || {};
self.create(); self.create();
self.busy(false); self.busy(false);
}, },
@@ -246,6 +246,10 @@ var Movie = new Class({
isSelected: function(){ isSelected: function(){
return this.select_checkbox.get('checked'); return this.select_checkbox.get('checked');
},
toElement: function(){
return this.el;
} }
}); });
@@ -425,4 +429,109 @@ var ReleaseAction = new Class({
} }
});
var TrailerAction = new Class({
Extends: MovieAction,
id: null,
create: function(){
var self = this;
self.el = new Element('a.trailer', {
'title': 'Watch the trailer of ' + self.movie.getTitle(),
'events': {
'click': self.watch.bind(self)
}
});
},
watch: function(offset){
var self = this;
var data_url = 'http://gdata.youtube.com/feeds/videos?vq="{title}" {year} trailer&max-results=1&alt=json-in-script&orderby=relevance&sortorder=descending&format=5&fmt=18'
var url = data_url.substitute({
'title': self.movie.getTitle(),
'year': self.movie.get('year'),
'offset': offset || 1
}),
size = $(self.movie).getSize(),
height = (size.x/16)*9,
id = 'trailer-'+randomString();
self.player_container = new Element('div[id='+id+']');
self.container = new Element('div.hide.trailer_container')
.adopt(self.player_container)
.inject(self.movie.container, 'top');
self.container.setStyle('height', 0);
self.container.removeClass('hide');
self.close_button = new Element('a.hide.hide_trailer', {
'text': 'Hide trailer',
'events': {
'click': self.stop.bind(self)
}
}).inject(self.movie);
setTimeout(function(){
$(self.movie).setStyle('max-height', height);
self.container.setStyle('height', height);
}, 100)
new Request.JSONP({
'url': url,
'onComplete': function(json){
var video_url = json.feed.entry[0].id.$t.split('/'),
video_id = video_url[video_url.length-1];
self.player = new YT.Player(id, {
'height': height,
'width': size.x,
'videoId': video_id,
'playerVars': {
'autoplay': 1,
'showsearch': 0,
'wmode': 'transparent',
'iv_load_policy': 3
}
});
self.close_button.removeClass('hide');
var quality_set = false;
var change_quality = function(state){
if(!quality_set && (state.data == 1 || state.data || 2)){
try {
self.player.setPlaybackQuality('hd720');
quality_set = true;
}
catch(e){
}
}
}
self.player.addEventListener('onStateChange', change_quality);
}
}).send()
},
stop: function(){
var self = this;
self.player.stopVideo();
self.container.addClass('hide');
self.close_button.addClass('hide');
setTimeout(function(){
self.container.destroy()
self.close_button.destroy();
}, 1800)
}
}); });
couchpotato/core/plugins/movie/static/search.js
+2 -2
View File
@@ -301,11 +301,11 @@ Block.Search.Item = new Class({
'title': self.title_select.get('value'), 'title': self.title_select.get('value'),
'profile_id': self.profile_select.get('value') 'profile_id': self.profile_select.get('value')
}, },
'onComplete': function(){ 'onComplete': function(json){
self.options.empty(); self.options.empty();
self.options.adopt( self.options.adopt(
new Element('div.message', { new Element('div.message', {
'text': 'Movie succesfully added.' 'text': json.added ? 'Movie succesfully added.' : 'Movie didn\'t add properly. Check logs'
}) })
); );
}, },
couchpotato/core/plugins/profile/main.py
+2 -3
View File
@@ -135,8 +135,7 @@ class ProfilePlugin(Plugin):
success = True success = True
except Exception, e: except Exception, e:
message = 'Failed deleting Profile: %s' % e message = log.error('Failed deleting Profile: %s', e)
log.error(message)
#db.close() #db.close()
@@ -163,7 +162,7 @@ class ProfilePlugin(Plugin):
# Create default quality profile # Create default quality profile
order = -2 order = -2
for profile in profiles: for profile in profiles:
log.info('Creating default profile: %s' % profile.get('label')) log.info('Creating default profile: %s', profile.get('label'))
p = Profile( p = Profile(
label = toUnicode(profile.get('label')), label = toUnicode(profile.get('label')),
order = order order = order
couchpotato/core/plugins/quality/main.py
+11 -11
View File
@@ -17,8 +17,8 @@ class QualityPlugin(Plugin):
qualities = [ qualities = [
{'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]}, {'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]},
{'identifier': '1080p', 'hd': True, 'size': (5000, 20000), 'label': '1080P', 'width': 1920, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts']}, {'identifier': '1080p', 'hd': True, 'size': (5000, 20000), 'label': '1080P', 'width': 1920, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts']},
{'identifier': '720p', 'hd': True, 'size': (3500, 10000), 'label': '720P', 'width': 1280, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts']}, {'identifier': '720p', 'hd': True, 'size': (3500, 10000), 'label': '720P', 'width': 1280, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts']},
{'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p'], 'ext':['avi']}, {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p'], 'ext':['avi']},
{'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': [], 'allow': [], 'ext':['iso', 'img'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts']}, {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': [], 'allow': [], 'ext':['iso', 'img'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts']},
{'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': ['dvdrip'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': ['dvdrip'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']},
@@ -26,7 +26,7 @@ class QualityPlugin(Plugin):
{'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': [], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': [], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']},
{'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']},
{'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, {'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']},
{'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': [], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']} {'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}
] ]
pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr']
@@ -116,7 +116,7 @@ class QualityPlugin(Plugin):
quality = db.query(Quality).filter_by(identifier = q.get('identifier')).first() quality = db.query(Quality).filter_by(identifier = q.get('identifier')).first()
if not quality: if not quality:
log.info('Creating quality: %s' % q.get('label')) log.info('Creating quality: %s', q.get('label'))
quality = Quality() quality = Quality()
db.add(quality) db.add(quality)
@@ -133,7 +133,7 @@ class QualityPlugin(Plugin):
).all() ).all()
if not profile: if not profile:
log.info('Creating profile: %s' % q.get('label')) log.info('Creating profile: %s', q.get('label'))
profile = Profile( profile = Profile(
core = True, core = True,
label = toUnicode(quality.label), label = toUnicode(quality.label),
@@ -170,20 +170,20 @@ class QualityPlugin(Plugin):
# Check tags # Check tags
if quality['identifier'] in words: if quality['identifier'] in words:
log.debug('Found via identifier "%s" in %s' % (quality['identifier'], cur_file)) log.debug('Found via identifier "%s" in %s', (quality['identifier'], cur_file))
return self.setCache(hash, quality) return self.setCache(hash, quality)
if list(set(quality.get('alternative', [])) & set(words)): if list(set(quality.get('alternative', [])) & set(words)):
log.debug('Found %s via alt %s in %s' % (quality['identifier'], quality.get('alternative'), cur_file)) log.debug('Found %s via alt %s in %s', (quality['identifier'], quality.get('alternative'), cur_file))
return self.setCache(hash, quality) return self.setCache(hash, quality)
for tag in quality.get('tags', []): for tag in quality.get('tags', []):
if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words): if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words):
log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file)) log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file))
return self.setCache(hash, quality) return self.setCache(hash, quality)
if list(set(quality.get('tags', [])) & set(words)): if list(set(quality.get('tags', [])) & set(words)):
log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file)) log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file))
return self.setCache(hash, quality) return self.setCache(hash, quality)
# Try again with loose testing # Try again with loose testing
@@ -191,7 +191,7 @@ class QualityPlugin(Plugin):
if quality: if quality:
return self.setCache(hash, quality) return self.setCache(hash, quality)
log.debug('Could not identify quality for: %s' % files) log.debug('Could not identify quality for: %s', files)
return None return None
def guessLoose(self, hash, extra): def guessLoose(self, hash, extra):
@@ -200,7 +200,7 @@ class QualityPlugin(Plugin):
# Last check on resolution only # Last check on resolution only
if quality.get('width', 480) == extra.get('resolution_width', 0): if quality.get('width', 480) == extra.get('resolution_width', 0):
log.debug('Found %s via resolution_width: %s == %s' % (quality['identifier'], quality.get('width', 480), extra.get('resolution_width', 0))) log.debug('Found %s via resolution_width: %s == %s', (quality['identifier'], quality.get('width', 480), extra.get('resolution_width', 0)))
return self.setCache(hash, quality) return self.setCache(hash, quality)
if 480 <= extra.get('resolution_width', 0) <= 720: if 480 <= extra.get('resolution_width', 0) <= 720:
couchpotato/core/plugins/release/main.py
+2 -2
View File
@@ -79,7 +79,7 @@ class Release(Plugin):
rel.files.append(added_file) rel.files.append(added_file)
db.commit() db.commit()
except Exception, e: except Exception, e:
log.debug('Failed to attach "%s" to release: %s' % (cur_file, e)) log.debug('Failed to attach "%s" to release: %s', (cur_file, e))
fireEvent('movie.restatus', movie.id) fireEvent('movie.restatus', movie.id)
@@ -158,7 +158,7 @@ class Release(Plugin):
'success': True 'success': True
}) })
else: else:
log.error('Couldn\'t find release with id: %s' % id) log.error('Couldn\'t find release with id: %s', id)
#db.close() #db.close()
return jsonified({ return jsonified({
couchpotato/core/plugins/renamer/main.py
+20 -20
View File
@@ -100,7 +100,7 @@ class Renamer(Plugin):
else: else:
group['library'] = fireEvent('library.update', identifier = group['library']['identifier'], single = True) group['library'] = fireEvent('library.update', identifier = group['library']['identifier'], single = True)
if not group['library']: if not group['library']:
log.error('Could not rename, no library item to work with: %s' % group_identifier) log.error('Could not rename, no library item to work with: %s', group_identifier)
continue continue
library = group['library'] library = group['library']
@@ -138,7 +138,7 @@ class Renamer(Plugin):
# Move nfo depending on settings # Move nfo depending on settings
if file_type is 'nfo' and not self.conf('rename_nfo'): if file_type is 'nfo' and not self.conf('rename_nfo'):
log.debug('Skipping, renaming of %s disabled' % file_type) log.debug('Skipping, renaming of %s disabled', file_type)
if self.conf('cleanup'): if self.conf('cleanup'):
for current_file in group['files'][file_type]: for current_file in group['files'][file_type]:
remove_files.append(current_file) remove_files.append(current_file)
@@ -197,7 +197,7 @@ class Renamer(Plugin):
break break
if not found: if not found:
log.error('Could not determine dvd structure for: %s' % current_file) log.error('Could not determine dvd structure for: %s', current_file)
# Do rename others # Do rename others
else: else:
@@ -272,7 +272,7 @@ class Renamer(Plugin):
movie.status_id = done_status.get('id') movie.status_id = done_status.get('id')
db.commit() db.commit()
except Exception, e: except Exception, e:
log.error('Failed marking movie finished: %s %s' % (e, traceback.format_exc())) log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc()))
# Go over current movie releases # Go over current movie releases
for release in movie.releases: for release in movie.releases:
@@ -282,20 +282,20 @@ class Renamer(Plugin):
# This is where CP removes older, lesser quality releases # This is where CP removes older, lesser quality releases
if release.quality.order > group['meta_data']['quality']['order']: if release.quality.order > group['meta_data']['quality']['order']:
log.info('Removing lesser quality %s for %s.' % (movie.library.titles[0].title, release.quality.label)) log.info('Removing lesser quality %s for %s.', (movie.library.titles[0].title, release.quality.label))
for current_file in release.files: for current_file in release.files:
remove_files.append(current_file) remove_files.append(current_file)
remove_releases.append(release) remove_releases.append(release)
# Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc # Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc
elif release.quality.order is group['meta_data']['quality']['order']: elif release.quality.order is group['meta_data']['quality']['order']:
log.info('Same quality release already exists for %s, with quality %s. Assuming repack.' % (movie.library.titles[0].title, release.quality.label)) log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (movie.library.titles[0].title, release.quality.label))
for current_file in release.files: for current_file in release.files:
remove_files.append(current_file) remove_files.append(current_file)
remove_releases.append(release) remove_releases.append(release)
# Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan # Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan
else: else:
log.info('Better quality release already exists for %s, with quality %s' % (movie.library.titles[0].title, release.quality.label)) log.info('Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label))
# Add _EXISTS_ to the parent dir # Add _EXISTS_ to the parent dir
if group['dirname']: if group['dirname']:
@@ -333,18 +333,18 @@ class Renamer(Plugin):
if isinstance(src, File): if isinstance(src, File):
src = src.path src = src.path
log.info('Removing "%s"' % src) log.info('Removing "%s"', src)
try: try:
os.remove(src) os.remove(src)
except: except:
log.error('Failed removing %s: %s' % (src, traceback.format_exc())) log.error('Failed removing %s: %s', (src, traceback.format_exc()))
# Rename all files marked # Rename all files marked
group['renamed_files'] = [] group['renamed_files'] = []
for src in rename_files: for src in rename_files:
if rename_files[src]: if rename_files[src]:
dst = rename_files[src] dst = rename_files[src]
log.info('Renaming "%s" to "%s"' % (src, dst)) log.info('Renaming "%s" to "%s"', (src, dst))
# Create dir # Create dir
self.makeDir(os.path.dirname(dst)) self.makeDir(os.path.dirname(dst))
@@ -353,22 +353,22 @@ class Renamer(Plugin):
self.moveFile(src, dst) self.moveFile(src, dst)
group['renamed_files'].append(dst) group['renamed_files'].append(dst)
except: except:
log.error('Failed moving the file "%s" : %s' % (os.path.basename(src), traceback.format_exc())) log.error('Failed moving the file "%s" : %s', (os.path.basename(src), traceback.format_exc()))
# Remove matching releases # Remove matching releases
for release in remove_releases: for release in remove_releases:
log.debug('Removing release %s' % release.identifier) log.debug('Removing release %s', release.identifier)
try: try:
db.delete(release) db.delete(release)
except: except:
log.error('Failed removing %s: %s' % (release.identifier, traceback.format_exc())) log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc()))
if group['dirname'] and group['parentdir']: if group['dirname'] and group['parentdir']:
try: try:
log.info('Deleting folder: %s' % group['parentdir']) log.info('Deleting folder: %s', group['parentdir'])
self.deleteEmptyFolder(group['parentdir']) self.deleteEmptyFolder(group['parentdir'])
except: except:
log.error('Failed removing %s: %s' % (group['parentdir'], traceback.format_exc())) log.error('Failed removing %s: %s', (group['parentdir'], traceback.format_exc()))
if not unknown: if not unknown:
# Search for trailers etc # Search for trailers etc
@@ -406,12 +406,12 @@ class Renamer(Plugin):
shutil.move(old, dest) shutil.move(old, dest)
try: try:
os.chmod(dest, Env.getPermission('folder')) os.chmod(dest, Env.getPermission('file'))
except: except:
log.error('Failed setting permissions for file: %s' % dest) log.error('Failed setting permissions for file: %s, %s', (dest, traceback.format_exc(1)))
except: except:
log.error("Couldn't move file '%s' to '%s': %s" % (old, dest, traceback.format_exc())) log.error('Couldn\'t move file "%s" to "%s": %s', (old, dest, traceback.format_exc()))
raise Exception raise Exception
return True return True
@@ -447,9 +447,9 @@ class Renamer(Plugin):
try: try:
os.rmdir(full_path) os.rmdir(full_path)
except: except:
log.error('Couldn\'t remove empty directory %s: %s' % (full_path, traceback.format_exc())) log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
try: try:
os.rmdir(folder) os.rmdir(folder)
except: except:
log.error('Couldn\'t remove empty directory %s: %s' % (folder, traceback.format_exc())) log.error('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc()))
couchpotato/core/plugins/scanner/main.py
+71 -53
View File
@@ -1,11 +1,10 @@
from couchpotato import get_session from couchpotato import get_session
from couchpotato.core.event import fireEvent, addEvent from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import toUnicode, simplifyString from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss
from couchpotato.core.helpers.variable import getExt, getImdb, tryInt from couchpotato.core.helpers.variable import getExt, getImdb, tryInt
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import File, Movie from couchpotato.core.settings.model import File, Movie
from couchpotato.environment import Env
from enzyme.exceptions import NoParserError, ParseError from enzyme.exceptions import NoParserError, ParseError
from guessit import guess_movie_info from guessit import guess_movie_info
from subliminal.videos import Video from subliminal.videos import Video
@@ -89,9 +88,9 @@ class Scanner(Plugin):
addEvent('scanner.partnumber', self.getPartNumber) addEvent('scanner.partnumber', self.getPartNumber)
def after_rename(group): def after_rename(group):
return self.scanFilesToLibrary(self, folder = group['destination_dir'], files = group['renamed_files']) return self.scanFilesToLibrary(folder = group['destination_dir'], files = group['renamed_files'])
addEvent('rename.after', after_rename) addEvent('renamer.after', after_rename)
def scanFilesToLibrary(self, folder = None, files = None): def scanFilesToLibrary(self, folder = None, files = None):
@@ -103,14 +102,14 @@ class Scanner(Plugin):
if group['library']: if group['library']:
fireEvent('release.add', group = group) fireEvent('release.add', group = group)
def scanFolderToLibrary(self, folder = None, newer_than = None, simple = True): def scanFolderToLibrary(self, folder = None, newer_than = 0, simple = True):
folder = os.path.normpath(folder) folder = os.path.normpath(folder)
if not os.path.isdir(folder): if not os.path.isdir(folder):
return return
groups = self.scan(folder = folder, simple = simple) groups = self.scan(folder = folder, simple = simple, newer_than = newer_than)
added_identifier = [] added_identifier = []
while True and not self.shuttingDown(): while True and not self.shuttingDown():
@@ -131,12 +130,12 @@ class Scanner(Plugin):
return added_identifier return added_identifier
def scan(self, folder = None, files = [], simple = False): def scan(self, folder = None, files = [], simple = False, newer_than = 0):
folder = os.path.normpath(folder) folder = ss(os.path.normpath(folder))
if not folder or not os.path.isdir(folder): if not folder or not os.path.isdir(folder):
log.error('Folder doesn\'t exists: %s' % folder) log.error('Folder doesn\'t exists: %s', folder)
return {} return {}
# Get movie "master" files # Get movie "master" files
@@ -147,19 +146,11 @@ class Scanner(Plugin):
if len(files) == 0: if len(files) == 0:
try: try:
files = [] files = []
for root, dirs, walk_files in os.walk(toUnicode(folder)): for root, dirs, walk_files in os.walk(folder):
for filename in walk_files: for filename in walk_files:
files.append(os.path.join(root, filename)) files.append(os.path.join(root, filename))
except: except:
try: log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
files = []
folder = toUnicode(folder).encode(Env.get('encoding'))
log.info('Trying to convert unicode to str path: %s, %s' % (folder, type(folder)))
for root, dirs, walk_files in os.walk(folder):
for filename in walk_files:
files.append(os.path.join(root, filename))
except:
log.error('Failed getting files from %s: %s' % (folder, traceback.format_exc()))
db = get_session() db = get_session()
@@ -215,7 +206,7 @@ class Scanner(Plugin):
for identifier, group in movie_files.iteritems(): for identifier, group in movie_files.iteritems():
if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier)
log.debug('Grouping files: %s' % identifier) log.debug('Grouping files: %s', identifier)
for file_path in group['unsorted_files']: for file_path in group['unsorted_files']:
wo_ext = file_path[:-(len(getExt(file_path)) + 1)] wo_ext = file_path[:-(len(getExt(file_path)) + 1)]
@@ -241,7 +232,7 @@ class Scanner(Plugin):
# Group the files based on the identifier # Group the files based on the identifier
delete_identifiers = [] delete_identifiers = []
for identifier, found_files in self.path_identifiers.iteritems(): for identifier, found_files in self.path_identifiers.iteritems():
log.debug('Grouping files on identifier: %s' % identifier) log.debug('Grouping files on identifier: %s', identifier)
group = movie_files.get(identifier) group = movie_files.get(identifier)
if group: if group:
@@ -257,13 +248,14 @@ class Scanner(Plugin):
# Cleaning up used # Cleaning up used
for identifier in delete_identifiers: for identifier in delete_identifiers:
del self.path_identifiers[identifier] if self.path_identifiers.get(identifier):
del self.path_identifiers[identifier]
del delete_identifiers del delete_identifiers
# Group based on folder # Group based on folder
delete_identifiers = [] delete_identifiers = []
for identifier, found_files in self.path_identifiers.iteritems(): for identifier, found_files in self.path_identifiers.iteritems():
log.debug('Grouping files on foldername: %s' % identifier) log.debug('Grouping files on foldername: %s', identifier)
for ff in found_files: for ff in found_files:
new_identifier = self.createStringIdentifier(os.path.dirname(ff), folder) new_identifier = self.createStringIdentifier(os.path.dirname(ff), folder)
@@ -282,7 +274,8 @@ class Scanner(Plugin):
# Cleaning up used # Cleaning up used
for identifier in delete_identifiers: for identifier in delete_identifiers:
del self.path_identifiers[identifier] if self.path_identifiers.get(identifier):
del self.path_identifiers[identifier]
del delete_identifiers del delete_identifiers
# Determine file types # Determine file types
@@ -296,13 +289,38 @@ class Scanner(Plugin):
# Check if movie is fresh and maybe still unpacking, ignore files new then 1 minute # Check if movie is fresh and maybe still unpacking, ignore files new then 1 minute
file_too_new = False file_too_new = False
for cur_file in group['unsorted_files']: for cur_file in group['unsorted_files']:
file_time = os.path.getmtime(cur_file) if not os.path.isfile(cur_file):
if file_time > time.time() - 60: file_too_new = time.time()
file_too_new = tryInt(time.time() - file_time) break
file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)]
for t in file_time:
if t > time.time() - 60:
file_too_new = tryInt(time.time() - t)
break
if file_too_new:
break break
if file_too_new: if file_too_new:
log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s' % (time.ctime(file_time), identifier)) log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time.ctime(file_time[0]), identifier))
# Delete the unsorted list
del group['unsorted_files']
continue
# Only process movies newer than x
if newer_than and newer_than > 0:
for cur_file in group['unsorted_files']:
file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)]
if file_time[0] > time.time() or file_time[1] > time.time():
break
log.debug('None of the files have changed since %s for %s, skipping.', (time.ctime(newer_than), identifier))
# Delete the unsorted list
del group['unsorted_files']
continue continue
# Group extra (and easy) files first # Group extra (and easy) files first
@@ -324,10 +342,10 @@ class Scanner(Plugin):
group['files']['movie'] = self.getMediaFiles(group['unsorted_files']) group['files']['movie'] = self.getMediaFiles(group['unsorted_files'])
if len(group['files']['movie']) == 0: if len(group['files']['movie']) == 0:
log.error('Couldn\t find any movie files for %s' % identifier) log.error('Couldn\t find any movie files for %s', identifier)
continue continue
log.debug('Getting metadata for %s' % identifier) log.debug('Getting metadata for %s', identifier)
group['meta_data'] = self.getMetaData(group) group['meta_data'] = self.getMetaData(group)
# Subtitle meta # Subtitle meta
@@ -360,7 +378,7 @@ class Scanner(Plugin):
# Determine movie # Determine movie
group['library'] = self.determineMovie(group) group['library'] = self.determineMovie(group)
if not group['library']: if not group['library']:
log.error('Unable to determine movie: %s' % group['identifiers']) log.error('Unable to determine movie: %s', group['identifiers'])
else: else:
movie = db.query(Movie).filter_by(library_id = group['library']['id']).first() movie = db.query(Movie).filter_by(library_id = group['library']['id']).first()
group['movie_id'] = None if not movie else movie.id group['movie_id'] = None if not movie else movie.id
@@ -373,9 +391,9 @@ class Scanner(Plugin):
self.path_identifiers = {} self.path_identifiers = {}
if len(processed_movies) > 0: if len(processed_movies) > 0:
log.info('Found %s movies in the folder %s' % (len(processed_movies), folder)) log.info('Found %s movies in the folder %s', (len(processed_movies), folder))
else: else:
log.debug('Found no movies in the folder %s' % (folder)) log.debug('Found no movies in the folder %s', (folder))
return processed_movies return processed_movies
def getMetaData(self, group): def getMetaData(self, group):
@@ -395,7 +413,7 @@ class Scanner(Plugin):
data['resolution_height'] = meta.get('resolution_height', 480) data['resolution_height'] = meta.get('resolution_height', 480)
data['aspect'] = meta.get('resolution_width', 720) / meta.get('resolution_height', 480) data['aspect'] = meta.get('resolution_width', 720) / meta.get('resolution_height', 480)
except: except:
log.debug('Error parsing metadata: %s %s' % (cur_file, traceback.format_exc())) log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc()))
pass pass
if data.get('audio'): break if data.get('audio'): break
@@ -423,11 +441,11 @@ class Scanner(Plugin):
'resolution_height': tryInt(p.video[0].height), 'resolution_height': tryInt(p.video[0].height),
} }
except ParseError: except ParseError:
log.debug('Failed to parse meta for %s' % filename) log.debug('Failed to parse meta for %s', filename)
except NoParserError: except NoParserError:
log.debug('No parser found for %s' % filename) log.debug('No parser found for %s', filename)
except: except:
log.debug('Failed parsing %s' % filename) log.debug('Failed parsing %s', filename)
return {} return {}
@@ -449,7 +467,7 @@ class Scanner(Plugin):
if s.language and s.path not in paths: if s.language and s.path not in paths:
detected_languages[s.path] = [s.language] detected_languages[s.path] = [s.language]
except: except:
log.debug('Failed parsing subtitle languages for %s: %s' % (paths, traceback.format_exc())) log.debug('Failed parsing subtitle languages for %s: %s', (paths, traceback.format_exc()))
# IDX # IDX
for extra in group['files']['subtitle_extra']: for extra in group['files']['subtitle_extra']:
@@ -465,7 +483,7 @@ class Scanner(Plugin):
if len(idx_langs) > 0 and os.path.isfile(sub_file): if len(idx_langs) > 0 and os.path.isfile(sub_file):
detected_languages[sub_file] = idx_langs detected_languages[sub_file] = idx_langs
except: except:
log.error('Failed parsing subtitle idx for %s: %s' % (extra, traceback.format_exc())) log.error('Failed parsing subtitle idx for %s: %s', (extra, traceback.format_exc()))
return detected_languages return detected_languages
@@ -478,7 +496,7 @@ class Scanner(Plugin):
for cur_file in files['movie']: for cur_file in files['movie']:
imdb_id = self.getCPImdb(cur_file) imdb_id = self.getCPImdb(cur_file)
if imdb_id: if imdb_id:
log.debug('Found movie via CP tag: %s' % cur_file) log.debug('Found movie via CP tag: %s', cur_file)
break break
# Check and see if nfo contains the imdb-id # Check and see if nfo contains the imdb-id
@@ -487,7 +505,7 @@ class Scanner(Plugin):
for nfo_file in files['nfo']: for nfo_file in files['nfo']:
imdb_id = getImdb(nfo_file) imdb_id = getImdb(nfo_file)
if imdb_id: if imdb_id:
log.debug('Found movie via nfo file: %s' % nfo_file) log.debug('Found movie via nfo file: %s', nfo_file)
break break
except: except:
pass pass
@@ -499,7 +517,7 @@ class Scanner(Plugin):
for filetype_file in files[filetype]: for filetype_file in files[filetype]:
imdb_id = getImdb(filetype_file, check_inside = False) imdb_id = getImdb(filetype_file, check_inside = False)
if imdb_id: if imdb_id:
log.debug('Found movie via imdb in filename: %s' % nfo_file) log.debug('Found movie via imdb in filename: %s', nfo_file)
break break
except: except:
pass pass
@@ -511,7 +529,7 @@ class Scanner(Plugin):
f = db.query(File).filter_by(path = toUnicode(cur_file)).first() f = db.query(File).filter_by(path = toUnicode(cur_file)).first()
try: try:
imdb_id = f.library[0].identifier imdb_id = f.library[0].identifier
log.debug('Found movie via database: %s' % cur_file) log.debug('Found movie via database: %s', cur_file)
break break
except: except:
pass pass
@@ -525,7 +543,7 @@ class Scanner(Plugin):
if len(movie) > 0: if len(movie) > 0:
imdb_id = movie[0]['imdb'] imdb_id = movie[0]['imdb']
if imdb_id: if imdb_id:
log.debug('Found movie via OpenSubtitleHash: %s' % cur_file) log.debug('Found movie via OpenSubtitleHash: %s', cur_file)
break break
# Search based on identifiers # Search based on identifiers
@@ -542,17 +560,17 @@ class Scanner(Plugin):
if len(movie) > 0: if len(movie) > 0:
imdb_id = movie[0]['imdb'] imdb_id = movie[0]['imdb']
log.debug('Found movie via search: %s' % cur_file) log.debug('Found movie via search: %s', cur_file)
if imdb_id: break if imdb_id: break
else: else:
log.debug('Identifier to short to use for search: %s' % identifier) log.debug('Identifier to short to use for search: %s', identifier)
if imdb_id: if imdb_id:
return fireEvent('library.add', attrs = { return fireEvent('library.add', attrs = {
'identifier': imdb_id 'identifier': imdb_id
}, update_after = False, single = True) }, update_after = False, single = True)
log.error('No imdb_id found for %s. Add a NFO file with IMDB id or add the year to the filename.' % group['identifiers']) log.error('No imdb_id found for %s. Add a NFO file with IMDB id or add the year to the filename.', group['identifiers'])
return {} return {}
def getCPImdb(self, string): def getCPImdb(self, string):
@@ -641,17 +659,17 @@ class Scanner(Plugin):
# ignoredpaths # ignoredpaths
for i in self.ignored_in_path: for i in self.ignored_in_path:
if i in filename.lower(): if i in filename.lower():
log.debug('Ignored "%s" contains "%s".' % (filename, i)) log.debug('Ignored "%s" contains "%s".', (filename, i))
return False return False
# Sample file # Sample file
if self.isSampleFile(filename): if self.isSampleFile(filename):
log.debug('Is sample file "%s".' % filename) log.debug('Is sample file "%s".', filename)
return False return False
# Minimal size # Minimal size
if self.filesizeBetween(filename, self.minimal_filesize['media']): if self.filesizeBetween(filename, self.minimal_filesize['media']):
log.debug('File to small: %s' % filename) log.debug('File to small: %s', filename)
return False return False
# All is OK # All is OK
@@ -659,14 +677,14 @@ class Scanner(Plugin):
def isSampleFile(self, filename): def isSampleFile(self, filename):
is_sample = re.search('(^|[\W_])sample\d*[\W_]', filename.lower()) is_sample = re.search('(^|[\W_])sample\d*[\W_]', filename.lower())
if is_sample: log.debug('Is sample file: %s' % filename) if is_sample: log.debug('Is sample file: %s', filename)
return is_sample return is_sample
def filesizeBetween(self, file, min = 0, max = 100000): def filesizeBetween(self, file, min = 0, max = 100000):
try: try:
return (min * 1048576) < os.path.getsize(file) < (max * 1048576) return (min * 1048576) < os.path.getsize(file) < (max * 1048576)
except: except:
log.error('Couldn\'t get filesize of %s.' % file) log.error('Couldn\'t get filesize of %s.', file)
return False return False
@@ -769,7 +787,7 @@ class Scanner(Plugin):
'year': guess.get('year'), 'year': guess.get('year'),
} }
except: except:
log.debug('Could not detect via guessit "%s": %s' % (file_name, traceback.format_exc())) log.debug('Could not detect via guessit "%s": %s', (file_name, traceback.format_exc()))
# Backup to simple # Backup to simple
cleaned = ' '.join(re.split('\W+', simplifyString(release_name))) cleaned = ' '.join(re.split('\W+', simplifyString(release_name)))
couchpotato/core/plugins/score/main.py
+9 -1
View File
@@ -4,7 +4,8 @@ from couchpotato.core.helpers.variable import getTitle
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin from couchpotato.core.plugins.base import Plugin
from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore, \ from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore, \
sizeScore, providerScore, duplicateScore sizeScore, providerScore, duplicateScore, partialIgnoredScore, namePositionScore, \
halfMultipartScore
log = CPLog(__name__) log = CPLog(__name__)
@@ -21,6 +22,7 @@ class Score(Plugin):
for movie_title in movie['library']['titles']: for movie_title in movie['library']['titles']:
score += nameRatioScore(toUnicode(nzb['name']), toUnicode(movie_title['title'])) score += nameRatioScore(toUnicode(nzb['name']), toUnicode(movie_title['title']))
score += namePositionScore(toUnicode(nzb['name']), toUnicode(movie_title['title']))
score += sizeScore(nzb['size']) score += sizeScore(nzb['size'])
@@ -38,6 +40,12 @@ class Score(Plugin):
# Duplicates in name # Duplicates in name
score += duplicateScore(nzb['name'], getTitle(movie['library'])) score += duplicateScore(nzb['name'], getTitle(movie['library']))
# Partial ignored words
score += partialIgnoredScore(nzb['name'], getTitle(movie['library']))
# Ignore single downloads from multipart
score += halfMultipartScore(nzb['name'])
# Extra provider specific check # Extra provider specific check
extra_score = nzb.get('extra_score') extra_score = nzb.get('extra_score')
if extra_score: if extra_score:
couchpotato/core/plugins/score/scores.py
+88 -4
View File
@@ -1,5 +1,7 @@
from couchpotato.core.event import fireEvent from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import simplifyString from couchpotato.core.helpers.encoding import simplifyString
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.plugins.scanner.main import Scanner
from couchpotato.environment import Env from couchpotato.environment import Env
import re import re
@@ -16,11 +18,12 @@ name_scores = [
'german:-10', 'french:-10', 'spanish:-10', 'swesub:-20', 'danish:-10', 'dutch:-10', 'german:-10', 'french:-10', 'spanish:-10', 'swesub:-20', 'danish:-10', 'dutch:-10',
# Release groups # Release groups
'imbt:1', 'cocain:1', 'vomit:1', 'fico:1', 'arrow:1', 'pukka:1', 'prism:1', 'devise:1', 'esir:1', 'ctrlhd:1', 'imbt:1', 'cocain:1', 'vomit:1', 'fico:1', 'arrow:1', 'pukka:1', 'prism:1', 'devise:1', 'esir:1', 'ctrlhd:1',
'metis:1', 'diamond:1', 'wiki:1', 'cbgb:1', 'crossbow:1', 'sinners:1', 'amiable:1', 'refined:1', 'twizted:1', 'felony:1', 'hubris:1', 'machd:1', 'metis:10', 'diamond:10', 'wiki:10', 'cbgb:10', 'crossbow:1', 'sinners:10', 'amiable:10', 'refined:1', 'twizted:1', 'felony:1', 'hubris:1', 'machd:1',
# Extras # Extras
'extras:-40', 'trilogy:-40', 'extras:-40', 'trilogy:-40',
] ]
def nameScore(name, year): def nameScore(name, year):
''' Calculate score for words in the NZB name ''' ''' Calculate score for words in the NZB name '''
@@ -47,8 +50,8 @@ def nameScore(name, year):
return score return score
def nameRatioScore(nzb_name, movie_name):
def nameRatioScore(nzb_name, movie_name):
nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True)) nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True))
movie_words = re.split('\W+', simplifyString(movie_name)) movie_words = re.split('\W+', simplifyString(movie_name))
@@ -56,15 +59,68 @@ def nameRatioScore(nzb_name, movie_name):
return 10 - len(left_over) return 10 - len(left_over)
def namePositionScore(nzb_name, movie_name):
score = 0
nzb_words = re.split('\W+', simplifyString(nzb_name))
qualities = fireEvent('quality.all', single = True)
try:
nzb_name = re.search(r'([\'"])[^\1]*\1', nzb_name).group(0)
except:
pass
name_year = fireEvent('scanner.name_year', nzb_name, single = True)
# Give points for movies beginning with the correct name
name_split = simplifyString(nzb_name).split(simplifyString(movie_name))
if name_split[0].strip() == '':
score += 10
# If year is second in line, give more points
if len(name_split) > 1 and name_year:
after_name = name_split[1].strip()
if tryInt(after_name[:4]) == name_year.get('year', None):
score += 10
after_name = after_name[4:]
# Give -point to crap between year and quality
found_quality = None
for quality in qualities:
# Main in words
if quality['identifier'] in nzb_words:
found_quality = quality['identifier']
# Alt in words
for alt in quality['alternative']:
if alt in nzb_words:
found_quality = alt
break
if not found_quality:
return score - 20
allowed = []
for value in name_scores:
name, sc = value.split(':')
allowed.append(name)
inbetween = re.split('\W+', after_name.split(found_quality)[0].strip())
score -= (10 * len(set(inbetween) - set(allowed)))
return score
def sizeScore(size): def sizeScore(size):
return 0 if size else -20 return 0 if size else -20
def providerScore(provider): def providerScore(provider):
if provider in ['NZBMatrix', 'Nzbs', 'Newzbin']: if provider in ['NZBMatrix', 'Nzbs', 'Newzbin']:
return 30 return 20
if provider in ['Newznab', 'Moovee', 'X264']: if provider in ['Newznab']:
return 10 return 10
return 0 return 0
@@ -79,3 +135,31 @@ def duplicateScore(nzb_name, movie_name):
duplicates = [x for i, x in enumerate(nzb_words) if nzb_words[i:].count(x) > 1] duplicates = [x for i, x in enumerate(nzb_words) if nzb_words[i:].count(x) > 1]
return len(list(set(duplicates) - set(movie_words))) * -4 return len(list(set(duplicates) - set(movie_words))) * -4
def partialIgnoredScore(nzb_name, movie_name):
nzb_name = nzb_name.lower()
movie_name = movie_name.lower()
ignored_words = [x.strip().lower() for x in Env.setting('ignored_words', section = 'searcher').split(',')]
score = 0
for ignored_word in ignored_words:
if ignored_word in nzb_name and ignored_word not in movie_name:
score -= 5
return score
def halfMultipartScore(nzb_name):
wrong_found = 0
for nr in [1, 2, 3, 4, 5, 'i', 'ii', 'iii', 'iv', 'v', 'a', 'b', 'c', 'd', 'e']:
for wrong in ['cd', 'part', 'dis', 'disc', 'dvd']:
if '%s%s' % (wrong, nr) in nzb_name.lower():
wrong_found += 1
if wrong_found == 1:
return -30
return 0
couchpotato/core/plugins/searcher/main.py
+27 -22
View File
@@ -56,7 +56,7 @@ class Searcher(Plugin):
except IndexError: except IndexError:
fireEvent('library.update', movie_dict['library']['identifier'], force = True) fireEvent('library.update', movie_dict['library']['identifier'], force = True)
except: except:
log.error('Search failed for %s: %s' % (movie_dict['library']['identifier'], traceback.format_exc())) log.error('Search failed for %s: %s', (movie_dict['library']['identifier'], traceback.format_exc()))
# Break if CP wants to shut down # Break if CP wants to shut down
if self.shuttingDown(): if self.shuttingDown():
@@ -88,7 +88,7 @@ class Searcher(Plugin):
ret = False ret = False
for quality_type in movie['profile']['types']: for quality_type in movie['profile']['types']:
if not self.couldBeReleased(quality_type['quality']['identifier'], release_dates, pre_releases): if not self.couldBeReleased(quality_type['quality']['identifier'], release_dates, pre_releases):
log.info('To early to search for %s, %s' % (quality_type['quality']['identifier'], default_title)) log.info('To early to search for %s, %s', (quality_type['quality']['identifier'], default_title))
continue continue
has_better_quality = 0 has_better_quality = 0
@@ -101,12 +101,12 @@ class Searcher(Plugin):
# Don't search for quality lower then already available. # Don't search for quality lower then already available.
if has_better_quality is 0: if has_better_quality is 0:
log.info('Search for %s in %s' % (default_title, quality_type['quality']['label'])) log.info('Search for %s in %s', (default_title, quality_type['quality']['label']))
quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True) quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True)
results = fireEvent('yarr.search', movie, quality, merge = True) results = fireEvent('yarr.search', movie, quality, merge = True)
sorted_results = sorted(results, key = lambda k: k['score'], reverse = True) sorted_results = sorted(results, key = lambda k: k['score'], reverse = True)
if len(sorted_results) == 0: if len(sorted_results) == 0:
log.debug('Nothing found for %s in %s' % (default_title, quality_type['quality']['label'])) log.debug('Nothing found for %s in %s', (default_title, quality_type['quality']['label']))
# Check if movie isn't deleted while searching # Check if movie isn't deleted while searching
if not db.query(Movie).filter_by(id = movie.get('id')).first(): if not db.query(Movie).filter_by(id = movie.get('id')).first():
@@ -141,10 +141,13 @@ class Searcher(Plugin):
rls.info.append(rls_info) rls.info.append(rls_info)
db.commit() db.commit()
except InterfaceError: except InterfaceError:
log.debug('Couldn\'t add %s to ReleaseInfo: %s' % (info, traceback.format_exc())) log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc()))
for nzb in sorted_results: for nzb in sorted_results:
if nzb['score'] <= 0:
log.debug('No more releases with score higher than 0')
break
downloaded = self.download(data = nzb, movie = movie) downloaded = self.download(data = nzb, movie = movie)
if downloaded is True: if downloaded is True:
ret = True ret = True
@@ -152,7 +155,7 @@ class Searcher(Plugin):
elif downloaded != 'try_next': elif downloaded != 'try_next':
break break
else: else:
log.info('Better quality (%s) already available or snatched for %s' % (quality_type['quality']['label'], default_title)) log.info('Better quality (%s) already available or snatched for %s', (quality_type['quality']['label'], default_title))
fireEvent('movie.restatus', movie['id']) fireEvent('movie.restatus', movie['id'])
break break
@@ -200,7 +203,7 @@ class Searcher(Plugin):
if movie['status_id'] == active_status.get('id'): if movie['status_id'] == active_status.get('id'):
for profile_type in movie['profile']['types']: for profile_type in movie['profile']['types']:
if profile_type['quality_id'] == rls.quality.id and profile_type['finish']: if profile_type['quality_id'] == rls.quality.id and profile_type['finish']:
log.info('Renamer disabled, marking movie as finished: %s' % log_movie) log.info('Renamer disabled, marking movie as finished: %s', log_movie)
# Mark release done # Mark release done
rls.status_id = done_status.get('id') rls.status_id = done_status.get('id')
@@ -211,7 +214,7 @@ class Searcher(Plugin):
mvie.status_id = done_status.get('id') mvie.status_id = done_status.get('id')
db.commit() db.commit()
except Exception, e: except Exception, e:
log.error('Failed marking movie finished: %s %s' % (e, traceback.format_exc())) log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc()))
#db.close() #db.close()
return True return True
@@ -226,7 +229,7 @@ class Searcher(Plugin):
retention = Env.setting('retention', section = 'nzb') retention = Env.setting('retention', section = 'nzb')
if nzb.get('seeds') is None and retention < nzb.get('age', 0): if nzb.get('seeds') is None and retention < nzb.get('age', 0):
log.info('Wrong: Outside retention, age is %s, needs %s or lower: %s' % (nzb['age'], retention, nzb['name'])) log.info('Wrong: Outside retention, age is %s, needs %s or lower: %s', (nzb['age'], retention, nzb['name']))
return False return False
movie_name = getTitle(movie['library']) movie_name = getTitle(movie['library'])
@@ -245,10 +248,10 @@ class Searcher(Plugin):
log.info("Wrong: '%s' blacklisted words: %s" % (nzb['name'], ", ".join(blacklisted))) log.info("Wrong: '%s' blacklisted words: %s" % (nzb['name'], ", ".join(blacklisted)))
return False return False
pron_tags = ['xxx', 'sex', 'anal', 'tits', 'fuck', 'porn', 'orgy', 'milf', 'boobs'] pron_tags = ['xxx', 'sex', 'anal', 'tits', 'fuck', 'porn', 'orgy', 'milf', 'boobs', 'erotica', 'erotic']
for p_tag in pron_tags: for p_tag in pron_tags:
if p_tag in nzb_words and p_tag not in movie_words: if p_tag in nzb_words and p_tag not in movie_words:
log.info('Wrong: %s, probably pr0n' % (nzb['name'])) log.info('Wrong: %s, probably pr0n', (nzb['name']))
return False return False
#qualities = fireEvent('quality.all', single = True) #qualities = fireEvent('quality.all', single = True)
@@ -256,18 +259,18 @@ class Searcher(Plugin):
# Contains lower quality string # Contains lower quality string
if self.containsOtherQuality(nzb, movie_year = movie['library']['year'], preferred_quality = preferred_quality, single_category = single_category): if self.containsOtherQuality(nzb, movie_year = movie['library']['year'], preferred_quality = preferred_quality, single_category = single_category):
log.info('Wrong: %s, looking for %s' % (nzb['name'], quality['label'])) log.info('Wrong: %s, looking for %s', (nzb['name'], quality['label']))
return False return False
# File to small # File to small
if nzb['size'] and preferred_quality['size_min'] > nzb['size']: if nzb['size'] and preferred_quality['size_min'] > nzb['size']:
log.info('"%s" is too small to be %s. %sMB instead of the minimal of %sMB.' % (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_min'])) log.info('"%s" is too small to be %s. %sMB instead of the minimal of %sMB.', (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_min']))
return False return False
# File to large # File to large
if nzb['size'] and preferred_quality.get('size_max') < nzb['size']: if nzb['size'] and preferred_quality.get('size_max') < nzb['size']:
log.info('"%s" is too large to be %s. %sMB instead of the maximum of %sMB.' % (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_max'])) log.info('"%s" is too large to be %s. %sMB instead of the maximum of %sMB.', (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_max']))
return False return False
@@ -406,17 +409,19 @@ class Searcher(Plugin):
if dates.get('theater') - 604800 < now: if dates.get('theater') - 604800 < now:
return True return True
else: else:
# 6 weeks after theater release # 12 weeks after theater release
if dates.get('theater') + 3628800 < now: if dates.get('theater') > 0 and dates.get('theater') + 7257600 < now:
return True return True
# 6 weeks before dvd release if dates.get('dvd') > 0:
if dates.get('dvd') - 3628800 < now:
return True
# Dvd should be released # 3 weeks before dvd release
if dates.get('dvd') > 0 and dates.get('dvd') < now: if dates.get('dvd') - 1814400 < now:
return True return True
# Dvd should be released
if dates.get('dvd') < now:
return True
return False return False
couchpotato/core/plugins/status/main.py
+1 -1
View File
@@ -92,7 +92,7 @@ class StatusPlugin(Plugin):
for identifier, label in self.statuses.iteritems(): for identifier, label in self.statuses.iteritems():
s = db.query(Status).filter_by(identifier = identifier).first() s = db.query(Status).filter_by(identifier = identifier).first()
if not s: if not s:
log.info('Creating status: %s' % label) log.info('Creating status: %s', label)
s = Status( s = Status(
identifier = identifier, identifier = identifier,
label = toUnicode(label) label = toUnicode(label)
couchpotato/core/plugins/trailer/main.py
+2 -2
View File
@@ -18,7 +18,7 @@ class Trailer(Plugin):
trailers = fireEvent('trailer.search', group = group, merge = True) trailers = fireEvent('trailer.search', group = group, merge = True)
if not trailers or trailers == []: if not trailers or trailers == []:
log.info('No trailers found for: %s' % getTitle(group['library'])) log.info('No trailers found for: %s', getTitle(group['library']))
return return
for trailer in trailers.get(self.conf('quality'), []): for trailer in trailers.get(self.conf('quality'), []):
@@ -26,7 +26,7 @@ class Trailer(Plugin):
if not os.path.isfile(destination): if not os.path.isfile(destination):
fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True) fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True)
else: else:
log.debug('Trailer already exists: %s' % destination) log.debug('Trailer already exists: %s', destination)
# Download first and break # Download first and break
break break
couchpotato/core/plugins/userscript/main.py
+1 -1
View File
@@ -73,7 +73,7 @@ class Userscript(Plugin):
'movie': fireEvent('userscript.get_movie_via_url', url = url, single = True) 'movie': fireEvent('userscript.get_movie_via_url', url = url, single = True)
} }
if not isDict(params['movie']): if not isDict(params['movie']):
log.error('Failed adding movie via url: %s' % url) log.error('Failed adding movie via url: %s', url)
params['error'] = params['movie'] if params['movie'] else 'Failed getting movie info' params['error'] = params['movie'] if params['movie'] else 'Failed getting movie info'
return jsonified(params) return jsonified(params)
couchpotato/core/providers/automation/base.py
+2 -2
View File
@@ -20,7 +20,7 @@ class Automation(Plugin):
def _getMovies(self): def _getMovies(self):
if not self.canCheck(): if not self.canCheck():
log.debug('Just checked, skipping %s' % self.getName()) log.debug('Just checked, skipping %s', self.getName())
return [] return []
self.last_checked = time.time() self.last_checked = time.time()
@@ -43,7 +43,7 @@ class Automation(Plugin):
type_value = movie.get(minimal_type, 0) type_value = movie.get(minimal_type, 0)
type_min = self.getMinimal(minimal_type) type_min = self.getMinimal(minimal_type)
if type_value < type_min: if type_value < type_min:
log.info('%s to low for %s, need %s has %s' % (minimal_type, identifier, type_min, type_value)) log.info('%s to low for %s, need %s has %s', (minimal_type, identifier, type_min, type_value))
return False return False
return True return True
couchpotato/core/providers/automation/imdb/main.py
+8 -4
View File
@@ -31,12 +31,13 @@ class IMDB(Automation, RSS):
if not enablers[index]: if not enablers[index]:
continue continue
elif 'rss.imdb' not in rss_url: elif 'rss.imdb' not in rss_url:
log.error('This isn\'t the correct url.: %s' % rss_url) log.error('This isn\'t the correct url.: %s', rss_url)
continue continue
prop_name = 'automation.imdb.last_update.%s' % md5(rss_url) prop_name = 'automation.imdb.last_update.%s' % md5(rss_url)
last_update = float(Env.prop(prop_name, default = 0)) last_update = float(Env.prop(prop_name, default = 0))
last_movie_added = 0
try: try:
cache_key = 'imdb.rss.%s' % md5(rss_url) cache_key = 'imdb.rss.%s' % md5(rss_url)
@@ -48,14 +49,17 @@ class IMDB(Automation, RSS):
created = int(time.mktime(parse(self.getTextElement(movie, "pubDate")).timetuple())) created = int(time.mktime(parse(self.getTextElement(movie, "pubDate")).timetuple()))
imdb = getImdb(self.getTextElement(movie, "link")) imdb = getImdb(self.getTextElement(movie, "link"))
if not imdb or created < last_update: if created > last_movie_added:
last_movie_added = created
if not imdb or created <= last_update:
continue continue
movies.append(imdb) movies.append(imdb)
except: except:
log.error('Failed loading IMDB watchlist: %s %s' % (rss_url, traceback.format_exc())) log.error('Failed loading IMDB watchlist: %s %s', (rss_url, traceback.format_exc()))
Env.prop(prop_name, time.time()) Env.prop(prop_name, last_movie_added)
return movies return movies
couchpotato/core/providers/automation/trakt/main.py
+15 -9
View File
@@ -41,13 +41,19 @@ class Trakt(Automation):
def call(self, method_url): def call(self, method_url):
if self.conf('automation_password'): try:
headers = { if self.conf('automation_password'):
'Authorization': "Basic %s" % base64.encodestring('%s:%s' % (self.conf('automation_username'), self.conf('automation_password')))[:-1] headers = {
} 'Authorization': 'Basic %s' % base64.encodestring('%s:%s' % (self.conf('automation_username'), self.conf('automation_password')))[:-1]
else: }
headers = {} else:
headers = {}
cache_key = 'trakt.%s' % md5(method_url) cache_key = 'trakt.%s' % md5(method_url)
json_string = self.getCache(cache_key, self.urls['base'] + method_url, headers = headers) json_string = self.getCache(cache_key, self.urls['base'] + method_url, headers = headers)
return json.loads(json_string) if json_string:
return json.loads(json_string)
except:
log.error('Failed to get data from trakt, check your login.')
return []
couchpotato/core/providers/base.py
+3 -3
View File
@@ -31,7 +31,7 @@ class Provider(Plugin):
self.urlopen(test_url, 30) self.urlopen(test_url, 30)
self.is_available[host] = True self.is_available[host] = True
except: except:
log.error('"%s" unavailable, trying again in an 15 minutes.' % host) log.error('"%s" unavailable, trying again in an 15 minutes.', host)
self.is_available[host] = False self.is_available[host] = False
return self.is_available.get(host, False) return self.is_available.get(host, False)
@@ -73,7 +73,7 @@ class YarrProvider(Provider):
if hostname in download_url: if hostname in download_url:
return self return self
except: except:
log.debug('Url % s doesn\'t belong to %s' % (url, self.getName())) log.debug('Url % s doesn\'t belong to %s', (url, self.getName()))
return return
@@ -106,4 +106,4 @@ class YarrProvider(Provider):
return [self.cat_backup_id] return [self.cat_backup_id]
def found(self, new): def found(self, new):
log.info('Found: score(%(score)s) on %(provider)s: %(name)s' % new) log.info('Found: score(%(score)s) on %(provider)s: %(name)s', new)
couchpotato/core/providers/metadata/base.py
+4 -4
View File
@@ -19,14 +19,14 @@ class MetaDataBase(Plugin):
def create(self, release): def create(self, release):
if self.isDisabled(): return if self.isDisabled(): return
log.info('Creating %s metadata.' % self.getName()) log.info('Creating %s metadata.', self.getName())
# Update library to get latest info # Update library to get latest info
try: try:
updated_library = fireEvent('library.update', release['library']['identifier'], force = True, single = True) updated_library = fireEvent('library.update', release['library']['identifier'], force = True, single = True)
release['library'] = mergeDicts(release['library'], updated_library) release['library'] = mergeDicts(release['library'], updated_library)
except: except:
log.error('Failed to update movie, before creating metadata: %s' % traceback.format_exc()) log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc())
root_name = self.getRootName(release) root_name = self.getRootName(release)
meta_name = os.path.basename(root_name) meta_name = os.path.basename(root_name)
@@ -44,14 +44,14 @@ class MetaDataBase(Plugin):
# Get file content # Get file content
content = getattr(self, 'get' + file_type.capitalize())(movie_info = movie_info, data = release) content = getattr(self, 'get' + file_type.capitalize())(movie_info = movie_info, data = release)
if content: if content:
log.debug('Creating %s file: %s' % (file_type, name)) log.debug('Creating %s file: %s', (file_type, name))
if os.path.isfile(content): if os.path.isfile(content):
shutil.copy2(content, name) shutil.copy2(content, name)
else: else:
self.createFile(name, content) self.createFile(name, content)
except: except:
log.error('Unable to create %s file: %s' % (file_type, traceback.format_exc())) log.error('Unable to create %s file: %s', (file_type, traceback.format_exc()))
def getRootName(self, data): def getRootName(self, data):
return return
couchpotato/core/providers/metadata/xbmc/main.py
+1 -1
View File
@@ -77,7 +77,7 @@ class XBMC(MetaDataBase):
votes.text = str(v) votes.text = str(v)
break break
except: except:
log.debug('Failed adding rating info from %s: %s' % (rating_type, traceback.format_exc())) log.debug('Failed adding rating info from %s: %s', (rating_type, traceback.format_exc()))
# Genre # Genre
for genre in movie_info.get('genres', []): for genre in movie_info.get('genres', []):
couchpotato/core/providers/movie/_modifier/main.py
+1 -1
View File
@@ -68,7 +68,7 @@ class MovieResultModifier(Plugin):
if release.status_id == done_status['id']: if release.status_id == done_status['id']:
temp['in_library'] = fireEvent('movie.get', movie.id, single = True) temp['in_library'] = fireEvent('movie.get', movie.id, single = True)
except: except:
log.error('Tried getting more info on searched movies: %s' % traceback.format_exc()) log.error('Tried getting more info on searched movies: %s', traceback.format_exc())
#db.close() #db.close()
return temp return temp
couchpotato/core/providers/movie/couchpotatoapi/main.py
+4 -4
View File
@@ -32,10 +32,10 @@ class CouchPotatoApi(MovieProvider):
headers = {'X-CP-Version': fireEvent('app.version', single = True)} headers = {'X-CP-Version': fireEvent('app.version', single = True)}
data = self.urlopen((self.api_url % ('eta')) + (identifier + '/'), headers = headers) data = self.urlopen((self.api_url % ('eta')) + (identifier + '/'), headers = headers)
dates = json.loads(data) dates = json.loads(data)
log.debug('Found ETA for %s: %s' % (identifier, dates)) log.debug('Found ETA for %s: %s', (identifier, dates))
return dates return dates
except Exception, e: except Exception, e:
log.error('Error getting ETA for %s: %s' % (identifier, e)) log.error('Error getting ETA for %s: %s', (identifier, e))
return {} return {}
@@ -43,9 +43,9 @@ class CouchPotatoApi(MovieProvider):
try: try:
data = self.urlopen((self.api_url % ('suggest')) + ','.join(movies) + '/' + ','.join(ignore) + '/') data = self.urlopen((self.api_url % ('suggest')) + ','.join(movies) + '/' + ','.join(ignore) + '/')
suggestions = json.loads(data) suggestions = json.loads(data)
log.info('Found Suggestions for %s' % (suggestions)) log.info('Found Suggestions for %s', (suggestions))
except Exception, e: except Exception, e:
log.error('Error getting suggestions for %s: %s' % (movies, e)) log.error('Error getting suggestions for %s: %s', (movies, e))
return suggestions return suggestions
couchpotato/core/providers/movie/imdbapi/main.py
+3 -3
View File
@@ -36,7 +36,7 @@ class IMDBAPI(MovieProvider):
if cached: if cached:
result = self.parseMovie(cached) result = self.parseMovie(cached)
if result.get('titles') and len(result.get('titles')) > 0: if result.get('titles') and len(result.get('titles')) > 0:
log.info('Found: %s' % result['titles'][0] + ' (' + str(result['year']) + ')') log.info('Found: %s', result['titles'][0] + ' (' + str(result['year']) + ')')
return [result] return [result]
return [] return []
@@ -54,7 +54,7 @@ class IMDBAPI(MovieProvider):
if cached: if cached:
result = self.parseMovie(cached) result = self.parseMovie(cached)
if result.get('titles') and len(result.get('titles')) > 0: if result.get('titles') and len(result.get('titles')) > 0:
log.info('Found: %s' % result['titles'][0] + ' (' + str(result['year']) + ')') log.info('Found: %s', result['titles'][0] + ' (' + str(result['year']) + ')')
return result return result
return {} return {}
@@ -103,7 +103,7 @@ class IMDBAPI(MovieProvider):
'actors': movie.get('Actors', '').split(','), 'actors': movie.get('Actors', '').split(','),
} }
except: except:
log.error('Failed parsing IMDB API json: %s' % traceback.format_exc()) log.error('Failed parsing IMDB API json: %s', traceback.format_exc())
return movie_data return movie_data
couchpotato/core/providers/movie/themoviedb/main.py
+9 -9
View File
@@ -28,7 +28,7 @@ class TheMovieDb(MovieProvider):
results = self.getCache(cache_key) results = self.getCache(cache_key)
if not results: if not results:
log.debug('Searching for movie by hash: %s' % file) log.debug('Searching for movie by hash: %s', file)
try: try:
raw = tmdb.searchByHashingFile(file) raw = tmdb.searchByHashingFile(file)
@@ -36,15 +36,15 @@ class TheMovieDb(MovieProvider):
if raw: if raw:
try: try:
results = self.parseMovie(raw) results = self.parseMovie(raw)
log.info('Found: %s' % results['titles'][0] + ' (' + str(results['year']) + ')') log.info('Found: %s', results['titles'][0] + ' (' + str(results['year']) + ')')
self.setCache(cache_key, results) self.setCache(cache_key, results)
return results return results
except SyntaxError, e: except SyntaxError, e:
log.error('Failed to parse XML response: %s' % e) log.error('Failed to parse XML response: %s', e)
return False return False
except: except:
log.debug('No movies known by hash for: %s' % file) log.debug('No movies known by hash for: %s', file)
pass pass
return results return results
@@ -60,7 +60,7 @@ class TheMovieDb(MovieProvider):
results = self.getCache(cache_key) results = self.getCache(cache_key)
if not results: if not results:
log.debug('Searching for movie: %s' % q) log.debug('Searching for movie: %s', q)
raw = tmdb.search(search_string) raw = tmdb.search(search_string)
results = [] results = []
@@ -75,12 +75,12 @@ class TheMovieDb(MovieProvider):
if nr == limit: if nr == limit:
break break
log.info('Found: %s' % [result['titles'][0] + ' (' + str(result['year']) + ')' for result in results]) log.info('Found: %s', [result['titles'][0] + ' (' + str(result['year']) + ')' for result in results])
self.setCache(cache_key, results) self.setCache(cache_key, results)
return results return results
except SyntaxError, e: except SyntaxError, e:
log.error('Failed to parse XML response: %s' % e) log.error('Failed to parse XML response: %s', e)
return False return False
return results return results
@@ -98,7 +98,7 @@ class TheMovieDb(MovieProvider):
movie = None movie = None
try: try:
log.debug('Getting info: %s' % cache_key) log.debug('Getting info: %s', cache_key)
movie = tmdb.imdbLookup(id = identifier) movie = tmdb.imdbLookup(id = identifier)
except: except:
pass pass
@@ -119,7 +119,7 @@ class TheMovieDb(MovieProvider):
movie = None movie = None
try: try:
log.debug('Getting info: %s' % cache_key) log.debug('Getting info: %s', cache_key)
movie = tmdb.getMovieInfo(id = id) movie = tmdb.getMovieInfo(id = id)
except: except:
pass pass
couchpotato/core/providers/nzb/mysterbin/main.py
+5 -5
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \ from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString simplifyString
@@ -49,21 +49,21 @@ class Mysterbin(NZBProvider):
try: try:
html = BeautifulSoup(data) html = BeautifulSoup(data)
resultable = html.find('table', attrs = {'class':'t'}) resultable = html.find('table', attrs = {'class':'t'})
for result in resultable.findAll('tr'): for result in resultable.find_all('tr'):
try: try:
myster_id = result.find('input', attrs = {'class': 'check4nzb'})['value'] myster_id = result.find('input', attrs = {'class': 'check4nzb'})['value']
# Age # Age
age = '' age = ''
for temp in result.find('td', attrs = {'class': 'cdetail'}).findAll(text = True): for temp in result.find('td', attrs = {'class': 'cdetail'}).find_all(text = True):
if 'days' in temp: if 'days' in temp:
age = tryInt(temp.split(' ')[0]) age = tryInt(temp.split(' ')[0])
break break
# size # size
size = None size = None
for temp in result.find('div', attrs = {'class': 'cdetail'}).findAll(text = True): for temp in result.find('div', attrs = {'class': 'cdetail'}).find_all(text = True):
if 'gb' in temp.lower() or 'mb' in temp.lower() or 'kb' in temp.lower(): if 'gb' in temp.lower() or 'mb' in temp.lower() or 'kb' in temp.lower():
size = self.parseSize(temp) size = self.parseSize(temp)
break break
@@ -74,7 +74,7 @@ class Mysterbin(NZBProvider):
new = { new = {
'id': myster_id, 'id': myster_id,
'name': ''.join(result.find('span', attrs = {'class': 'cname'}).findAll(text = True)), 'name': ''.join(result.find('span', attrs = {'class': 'cname'}).find_all(text = True)),
'type': 'nzb', 'type': 'nzb',
'provider': self.getName(), 'provider': self.getName(),
'age': age, 'age': age,
couchpotato/core/providers/nzb/newzbin/main.py
+5 -3
View File
@@ -57,6 +57,8 @@ class Newzbin(NZBProvider, RSS):
'category': '6', 'category': '6',
'ps_rb_video_format': str(cat_id), 'ps_rb_video_format': str(cat_id),
'ps_rb_source': str(format_id), 'ps_rb_source': str(format_id),
'u_post_larger_than': quality.get('size_min'),
'u_post_smaller_than': quality.get('size_max'),
}) })
url = "%s?%s" % (self.urls['search'], arguments) url = "%s?%s" % (self.urls['search'], arguments)
@@ -80,7 +82,7 @@ class Newzbin(NZBProvider, RSS):
data = XMLTree.fromstring(data) data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item') nzbs = self.getElements(data, 'channel/item')
except Exception, e: except Exception, e:
log.debug('%s, %s' % (self.getName(), e)) log.debug('%s, %s', (self.getName(), e))
return results return results
for nzb in nzbs: for nzb in nzbs:
@@ -131,7 +133,7 @@ class Newzbin(NZBProvider, RSS):
def download(self, url = '', nzb_id = ''): def download(self, url = '', nzb_id = ''):
try: try:
log.info('Download nzb from newzbin, report id: %s ' % nzb_id) log.info('Download nzb from newzbin, report id: %s ', nzb_id)
return self.urlopen(self.urls['download'], params = { return self.urlopen(self.urls['download'], params = {
'username' : self.conf('username'), 'username' : self.conf('username'),
@@ -139,7 +141,7 @@ class Newzbin(NZBProvider, RSS):
'reportid' : nzb_id 'reportid' : nzb_id
}, show_error = False) }, show_error = False)
except Exception, e: except Exception, e:
log.error('Failed downloading from newzbin, check credit: %s' % e) log.error('Failed downloading from newzbin, check credit: %s', e)
return False return False
def getFormatId(self, format): def getFormatId(self, format):
couchpotato/core/providers/nzb/newznab/main.py
+6 -6
View File
@@ -112,7 +112,7 @@ class Newznab(NZBProvider, RSS):
data = XMLTree.fromstring(data) data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item') nzbs = self.getElements(data, 'channel/item')
except Exception, e: except Exception, e:
log.debug('%s, %s' % (self.getName(), e)) log.debug('%s, %s', (self.getName(), e))
return results return results
results = [] results = []
@@ -126,8 +126,8 @@ class Newznab(NZBProvider, RSS):
elif item.attrib.get('name') == 'usenetdate': elif item.attrib.get('name') == 'usenetdate':
date = item.attrib.get('value') date = item.attrib.get('value')
if date is '': log.debug('Date not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title"))) if date is '': log.debug('Date not parsed properly or not available for %s: %s', (host['host'], self.getTextElement(nzb, "title")))
if size is 0: log.debug('Size not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title"))) if size is 0: log.debug('Size not parsed properly or not available for %s: %s', (host['host'], self.getTextElement(nzb, "title")))
id = self.getTextElement(nzb, "guid").split('/')[-1:].pop() id = self.getTextElement(nzb, "guid").split('/')[-1:].pop()
new = { new = {
@@ -157,7 +157,7 @@ class Newznab(NZBProvider, RSS):
return results return results
except SyntaxError: except SyntaxError:
log.error('Failed to parse XML response from Newznab: %s' % host) log.error('Failed to parse XML response from Newznab: %s', host)
return results return results
def getHosts(self): def getHosts(self):
@@ -216,9 +216,9 @@ class Newznab(NZBProvider, RSS):
response = e.read().lower() response = e.read().lower()
if 'maximum api' in response or 'download limit' in response: if 'maximum api' in response or 'download limit' in response:
if not self.limits_reached.get(host): if not self.limits_reached.get(host):
log.error('Limit reached for newznab provider: %s' % host) log.error('Limit reached for newznab provider: %s', host)
self.limits_reached[host] = time.time() self.limits_reached[host] = time.time()
return 'try_next' return 'try_next'
log.error('Failed download from %s' % (host, traceback.format_exc())) log.error('Failed download from %s', (host, traceback.format_exc()))
raise raise
couchpotato/core/providers/nzb/nzbclub/main.py
+8 -7
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \ from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString simplifyString
@@ -28,7 +28,7 @@ class NZBClub(NZBProvider, RSS):
if self.isDisabled(): if self.isDisabled():
return results return results
q = '"%s" %s %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier')) q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
for ignored in Env.setting('ignored_words', 'searcher').split(','): for ignored in Env.setting('ignored_words', 'searcher').split(','):
q = '%s -%s' % (q, ignored.strip()) q = '%s -%s' % (q, ignored.strip())
@@ -49,7 +49,7 @@ class NZBClub(NZBProvider, RSS):
data = XMLTree.fromstring(data) data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item') nzbs = self.getElements(data, 'channel/item')
except Exception, e: except Exception, e:
log.debug('%s, %s' % (self.getName(), e)) log.debug('%s, %s', (self.getName(), e))
return results return results
for nzb in nzbs: for nzb in nzbs:
@@ -62,9 +62,10 @@ class NZBClub(NZBProvider, RSS):
def extra_check(item): def extra_check(item):
full_description = self.getCache('nzbclub.%s' % nzbclub_id, item['detail_url'], cache_timeout = 25920000) full_description = self.getCache('nzbclub.%s' % nzbclub_id, item['detail_url'], cache_timeout = 25920000)
if 'ARCHIVE inside ARCHIVE' in full_description: for ignored in ['ARCHIVE inside ARCHIVE', 'Incomplete', 'repair impossible']:
log.info('Wrong: Seems to be passworded files: %s' % new['name']) if ignored in full_description:
return False log.info('Wrong: Seems to be passworded or corrupted files: %s', new['name'])
return False
return True return True
@@ -111,7 +112,7 @@ class NZBClub(NZBProvider, RSS):
full_description = self.getCache('nzbclub.%s' % item['id'], item['detail_url'], cache_timeout = 25920000) full_description = self.getCache('nzbclub.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
if 'ARCHIVE inside ARCHIVE' in full_description: if 'ARCHIVE inside ARCHIVE' in full_description:
log.info('Wrong: Seems to be passworded files: %s' % item['name']) log.info('Wrong: Seems to be passworded files: %s', item['name'])
return False return False
return True return True
couchpotato/core/providers/nzb/nzbindex/main.py
+11 -3
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \ from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString simplifyString
@@ -30,7 +30,7 @@ class NzbIndex(NZBProvider, RSS):
if self.isDisabled(): if self.isDisabled():
return results return results
q = '%s %s %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier')) q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
arguments = tryUrlencode({ arguments = tryUrlencode({
'q': q, 'q': q,
'age': Env.setting('retention', 'nzb'), 'age': Env.setting('retention', 'nzb'),
@@ -53,7 +53,7 @@ class NzbIndex(NZBProvider, RSS):
data = XMLTree.fromstring(data) data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item') nzbs = self.getElements(data, 'channel/item')
except Exception, e: except Exception, e:
log.debug('%s, %s' % (self.getName(), e)) log.debug('%s, %s', (self.getName(), e))
return results return results
for nzb in nzbs: for nzb in nzbs:
@@ -67,6 +67,13 @@ class NzbIndex(NZBProvider, RSS):
except: except:
description = '' description = ''
def extra_check(new):
if '#c20000' in new['description'].lower():
log.info('Wrong: Seems to be passworded: %s', new['name'])
return False
return True
new = { new = {
'id': nzbindex_id, 'id': nzbindex_id,
'type': 'nzb', 'type': 'nzb',
@@ -79,6 +86,7 @@ class NzbIndex(NZBProvider, RSS):
'detail_url': enclosure['url'].replace('/download/', '/release/'), 'detail_url': enclosure['url'].replace('/download/', '/release/'),
'description': description, 'description': description,
'get_more_info': self.getMoreInfo, 'get_more_info': self.getMoreInfo,
'extra_check': extra_check,
'check_nzb': True, 'check_nzb': True,
} }
couchpotato/core/providers/nzb/nzbmatrix/main.py
+4 -1
View File
@@ -58,7 +58,7 @@ class NZBMatrix(NZBProvider, RSS):
data = XMLTree.fromstring(data) data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item') nzbs = self.getElements(data, 'channel/item')
except Exception, e: except Exception, e:
log.debug('%s, %s' % (self.getName(), e)) log.debug('%s, %s', (self.getName(), e))
return results return results
for nzb in nzbs: for nzb in nzbs:
@@ -99,6 +99,9 @@ class NZBMatrix(NZBProvider, RSS):
return results return results
def download(self, url = '', nzb_id = ''):
return self.urlopen(url, headers = {'User-Agent': Env.getIdentifier()})
def getApiExt(self): def getApiExt(self):
return '&username=%s&apikey=%s' % (self.conf('username'), self.conf('api_key')) return '&username=%s&apikey=%s' % (self.conf('username'), self.conf('api_key'))
couchpotato/core/providers/torrent/kickasstorrents/main.py
+7 -7
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.variable import tryInt, getTitle from couchpotato.core.helpers.variable import tryInt, getTitle
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
@@ -47,14 +47,14 @@ class KickAssTorrents(TorrentProvider):
try: try:
html = BeautifulSoup(data) html = BeautifulSoup(data)
resultdiv = html.find('div', attrs = {'class':'tabs'}) resultdiv = html.find('div', attrs = {'class':'tabs'})
for result in resultdiv.findAll('div', recursive = False): for result in resultdiv.find_all('div', recursive = False):
if result.get('id').lower() not in cat_ids: if result.get('id').lower() not in cat_ids:
continue continue
try: try:
try: try:
for temp in result.findAll('tr'): for temp in result.find_all('tr'):
if temp['class'] is 'firstr' or not temp.get('id'): if temp['class'] is 'firstr' or not temp.get('id'):
continue continue
@@ -68,15 +68,15 @@ class KickAssTorrents(TorrentProvider):
} }
nr = 0 nr = 0
for td in temp.findAll('td'): for td in temp.find_all('td'):
column_name = table_order[nr] column_name = table_order[nr]
if column_name: if column_name:
if column_name is 'name': if column_name is 'name':
link = td.find('div', {'class': 'torrentname'}).findAll('a')[1] link = td.find('div', {'class': 'torrentname'}).find_all('a')[1]
new['id'] = temp.get('id')[-8:] new['id'] = temp.get('id')[-8:]
new['name'] = link.text new['name'] = link.text
new['url'] = td.findAll('a', 'idownload')[1]['href'] new['url'] = td.find_all('a', 'idownload')[1]['href']
if new['url'][:2] == '//': if new['url'][:2] == '//':
new['url'] = 'http:%s' % new['url'] new['url'] = 'http:%s' % new['url']
new['score'] = 20 if td.find('a', 'iverif') else 0 new['score'] = 20 if td.find('a', 'iverif') else 0
@@ -99,7 +99,7 @@ class KickAssTorrents(TorrentProvider):
results.append(new) results.append(new)
self.found(new) self.found(new)
except: except:
log.error('Failed parsing KickAssTorrents: %s' % traceback.format_exc()) log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc())
except: except:
pass pass
couchpotato/core/providers/torrent/thepiratebay/main.py
+5 -5
View File
@@ -36,11 +36,11 @@ class ThePirateBay(TorrentProvider):
url = self.apiUrl % (quote_plus(self.toSearchString(movie.name + ' ' + quality) + self.makeIgnoreString(type)), self.getCatId(type)) url = self.apiUrl % (quote_plus(self.toSearchString(movie.name + ' ' + quality) + self.makeIgnoreString(type)), self.getCatId(type))
log.info('Searching: %s' % url) log.info('Searching: %s', url)
data = self.urlopen(url) data = self.urlopen(url)
if not data: if not data:
log.error('Failed to get data from %s.' % url) log.error('Failed to get data from %s.', url)
return results return results
try: try:
@@ -104,7 +104,7 @@ class ThePirateBay(TorrentProvider):
new.content = self.getInfo(new.detailUrl) new.content = self.getInfo(new.detailUrl)
if self.isCorrectMovie(new, movie, type): if self.isCorrectMovie(new, movie, type):
results.append(new) results.append(new)
log.info('Found: %s' % new.name) log.info('Found: %s', new.name)
return results return results
@@ -127,11 +127,11 @@ class ThePirateBay(TorrentProvider):
def getInfo(self, url): def getInfo(self, url):
log.debug('Getting info: %s' % url) log.debug('Getting info: %s', url)
data = self.urlopen(url) data = self.urlopen(url)
if not data: if not data:
log.error('Failed to get data from %s.' % url) log.error('Failed to get data from %s.', url)
return '' return ''
div = SoupStrainer('div') div = SoupStrainer('div')
couchpotato/core/providers/trailer/hdtrailers/main.py
+8 -8
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import SoupStrainer, BeautifulSoup from bs4 import SoupStrainer, BeautifulSoup
from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import mergeDicts, getTitle from couchpotato.core.helpers.variable import mergeDicts, getTitle
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
@@ -51,13 +51,13 @@ class HDTrailers(TrailerProvider):
try: try:
tables = SoupStrainer('div') tables = SoupStrainer('div')
html = BeautifulSoup(data, parseOnlyThese = tables) html = BeautifulSoup(data, parse_only = tables)
result_table = html.findAll('h2', text = re.compile(movie_name)) result_table = html.find_all('h2', text = re.compile(movie_name))
for h2 in result_table: for h2 in result_table:
if 'trailer' in h2.lower(): if 'trailer' in h2.lower():
parent = h2.parent.parent.parent parent = h2.parent.parent.parent
trailerLinks = parent.findAll('a', text = re.compile('480p|720p|1080p')) trailerLinks = parent.find_all('a', text = re.compile('480p|720p|1080p'))
try: try:
for trailer in trailerLinks: for trailer in trailerLinks:
results[trailer].insert(0, trailer.parent['href']) results[trailer].insert(0, trailer.parent['href'])
@@ -74,11 +74,11 @@ class HDTrailers(TrailerProvider):
results = {'480p':[], '720p':[], '1080p':[]} results = {'480p':[], '720p':[], '1080p':[]}
try: try:
tables = SoupStrainer('table') tables = SoupStrainer('table')
html = BeautifulSoup(data, parseOnlyThese = tables) html = BeautifulSoup(data, parse_only = tables)
result_table = html.find('table', attrs = {'class':'bottomTable'}) result_table = html.find('table', attrs = {'class':'bottomTable'})
for tr in result_table.findAll('tr'): for tr in result_table.find_all('tr'):
trtext = str(tr).lower() trtext = str(tr).lower()
if 'clips' in trtext: if 'clips' in trtext:
break break
@@ -86,7 +86,7 @@ class HDTrailers(TrailerProvider):
nr = 0 nr = 0
if 'trailer' not in tr.find('span', 'standardTrailerName').text.lower(): if 'trailer' not in tr.find('span', 'standardTrailerName').text.lower():
continue continue
resolutions = tr.findAll('td', attrs = {'class':'bottomTableResolution'}) resolutions = tr.find_all('td', attrs = {'class':'bottomTableResolution'})
for res in resolutions: for res in resolutions:
results[str(res.a.contents[0])].insert(0, res.a['href']) results[str(res.a.contents[0])].insert(0, res.a['href'])
nr += 1 nr += 1
@@ -94,7 +94,7 @@ class HDTrailers(TrailerProvider):
return results return results
except AttributeError: except AttributeError:
log.debug('No trailers found in provider %s.' % provider) log.debug('No trailers found in provider %s.', provider)
results['404'] = True results['404'] = True
return results return results
couchpotato/core/providers/userscript/allocine/main.py
+1 -1
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup from bs4 import BeautifulSoup
from couchpotato.core.providers.userscript.base import UserscriptBase from couchpotato.core.providers.userscript.base import UserscriptBase
class AlloCine(UserscriptBase): class AlloCine(UserscriptBase):
couchpotato/core/providers/userscript/rottentomatoes/main.py
+1 -1
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent from couchpotato.core.event import fireEvent
from couchpotato.core.providers.userscript.base import UserscriptBase from couchpotato.core.providers.userscript.base import UserscriptBase
couchpotato/core/providers/userscript/youteather/main.py
+3 -3
View File
@@ -5,8 +5,8 @@ class YouTheater(UserscriptBase):
id_re = re.compile("view\.php\?id=(\d+)") id_re = re.compile("view\.php\?id=(\d+)")
includes = ['http://www.youtheater.com/view.php?id=*', 'http://youtheater.com/view.php?id=*', includes = ['http://www.youtheater.com/view.php?id=*', 'http://youtheater.com/view.php?id=*',
'http://www.sratim.co.il/view.php?id=*', 'http://sratim.co.il/view.php?id=*'] 'http://www.sratim.co.il/view.php?id=*', 'http://sratim.co.il/view.php?id=*']
def getMovie(self, url): def getMovie(self, url):
id = self.id_re.findall(url)[0] id = self.id_re.findall(url)[0]
url = "http://www.youtheater.com/view.php?id=%s" % id url = 'http://www.youtheater.com/view.php?id=%s' % id
return super(YouTheater, self).getMovie(url) return super(YouTheater, self).getMovie(url)
couchpotato/runner.py
+5 -5
View File
@@ -155,10 +155,10 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
import color_logs import color_logs
from couchpotato.core.logger import CPLog from couchpotato.core.logger import CPLog
log = CPLog(__name__) log = CPLog(__name__)
log.debug('Started with options %s' % options) log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file = None, line = None): def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s' % (category, message, filename, lineno)) log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn warnings.showwarning = customwarn
@@ -185,7 +185,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
current_db_version = db_version(db, repo) current_db_version = db_version(db, repo)
if current_db_version < latest_db_version and not debug: if current_db_version < latest_db_version and not debug:
log.info('Doing database upgrade. From %d to %d' % (current_db_version, latest_db_version)) log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version))
upgrade(db, repo) upgrade(db, repo)
# Configure Database # Configure Database
@@ -220,7 +220,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
app.register_blueprint(api, url_prefix = '%s/api/%s/' % (url_base, api_key)) app.register_blueprint(api, url_prefix = '%s/api/%s/' % (url_base, api_key))
# Some logging and fire load event # Some logging and fire load event
try: log.info('Starting server on port %(port)s' % config) try: log.info('Starting server on port %(port)s', config)
except: pass except: pass
fireEventAsync('app.load') fireEventAsync('app.load')
@@ -248,7 +248,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
try: try:
nr, msg = e nr, msg = e
if nr == 48: if nr == 48:
log.info('Already in use, try %s more time after few seconds' % restart_tries) log.info('Already in use, try %s more time after few seconds', restart_tries)
time.sleep(1) time.sleep(1)
restart_tries -= 1 restart_tries -= 1
couchpotato/static/images/icon.trailer.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 596 B

couchpotato/static/scripts/page/manage.js
+1 -1
View File
@@ -41,7 +41,7 @@ Page.Manage = new Class({
Api.request('manage.update', { Api.request('manage.update', {
'data': { 'data': {
'full': full ? 1 : null 'full': +full
} }
}) })
couchpotato/static/scripts/page/settings.js
+12 -7
View File
@@ -926,14 +926,19 @@ Option.Choice = new Class({
var mtches = [] var mtches = []
if(matches) if(matches)
matches.each(function(match, mnr){ matches.each(function(match, mnr){
var msplit = value.split(match); var pos = value.indexOf(match),
msplit.each(function(matchsplit, snr){ msplit = [value.substr(0, pos), value.substr(pos, match.length), value.substr(pos+match.length)];
if(msplit.length-1 == snr)
value = matchsplit;
mtches.append([value == matchsplit ? match : matchsplit]);
if(matches.length*2 == mtches.length) msplit.each(function(matchsplit, snr){
mtches.append([value]); if(msplit.length-1 == snr){
value = matchsplit;
if(matches.length-1 == mnr)
mtches.append([value]);
return;
}
mtches.append([value == matchsplit ? match : matchsplit]);
}); });
}); });
couchpotato/static/scripts/page/wanted.js
+5 -4
View File
@@ -28,8 +28,9 @@ var MovieActions = {};
window.addEvent('domready', function(){ window.addEvent('domready', function(){
MovieActions.Wanted = { MovieActions.Wanted = {
'IMBD': IMDBAction 'IMDB': IMDBAction
,'releases': ReleaseAction ,'Trailer': TrailerAction
,'Releases': ReleaseAction
,'Edit': new Class({ ,'Edit': new Class({
@@ -236,12 +237,12 @@ window.addEvent('domready', function(){
}; };
MovieActions.Snatched = { MovieActions.Snatched = {
'IMBD': IMDBAction 'IMDB': IMDBAction
,'Delete': MovieActions.Wanted.Delete ,'Delete': MovieActions.Wanted.Delete
}; };
MovieActions.Done = { MovieActions.Done = {
'IMBD': IMDBAction 'IMDB': IMDBAction
,'Edit': MovieActions.Wanted.Edit ,'Edit': MovieActions.Wanted.Edit
,'Files': new Class({ ,'Files': new Class({
couchpotato/static/style/main.css
+5 -1
View File
@@ -154,6 +154,7 @@ body > .spinner, .mask{
.icon.rating { background-image: url('../images/icon.rating.png'); } .icon.rating { background-image: url('../images/icon.rating.png'); }
.icon.files { background-image: url('../images/icon.files.png'); } .icon.files { background-image: url('../images/icon.files.png'); }
.icon.info { background-image: url('../images/icon.info.png'); } .icon.info { background-image: url('../images/icon.info.png'); }
.icon.trailer { background-image: url('../images/icon.trailer.png'); }
/*** Navigation ***/ /*** Navigation ***/
.header { .header {
@@ -166,6 +167,7 @@ body > .spinner, .mask{
z-index: 5; z-index: 5;
box-shadow: 0 20px 30px -30px rgba(0,0,0,0.05); box-shadow: 0 20px 30px -30px rgba(0,0,0,0.05);
transition: box-shadow .4s cubic-bezier(0.9,0,0.1,1); transition: box-shadow .4s cubic-bezier(0.9,0,0.1,1);
transform: translateZ(0);
} }
.header.with_shadow { .header.with_shadow {
box-shadow: 0 20px 30px -30px rgba(0,0,0,0.3); box-shadow: 0 20px 30px -30px rgba(0,0,0,0.3);
@@ -215,6 +217,7 @@ body > .spinner, .mask{
outline: none; outline: none;
box-shadow: inset 0 1px 8px rgba(0,0,0,0.05), 0 1px 0px rgba(255,255,255,0.15); box-shadow: inset 0 1px 8px rgba(0,0,0,0.05), 0 1px 0px rgba(255,255,255,0.15);
transition: all .4s cubic-bezier(0.9,0,0.1,1); transition: all .4s cubic-bezier(0.9,0,0.1,1);
transform: translateZ(0);
} }
.header .navigation li:hover a:after { background-color: #047792; } .header .navigation li:hover a:after { background-color: #047792; }
@@ -311,7 +314,7 @@ body > .spinner, .mask{
.header .message.update { .header .message.update {
text-align: center; text-align: center;
position: relative; position: relative;
top: -70px; top: -100px;
padding: 2px 0; padding: 2px 0;
background: #ff6134; background: #ff6134;
font-size: 12px; font-size: 12px;
@@ -522,6 +525,7 @@ body > .spinner, .mask{
width: 25px; width: 25px;
border: 1px solid rgba(0,0,0,0.3); border: 1px solid rgba(0,0,0,0.3);
transition: all 0.3s ease-in-out; transition: all 0.3s ease-in-out;
transform: translateZ(0);
} }
.more_menu.show > a:not(:active), .more_menu > a:hover:not(:active) { .more_menu.show > a:not(:active), .more_menu > a:hover:not(:active) {
background-color: #406db8; background-color: #406db8;
couchpotato/static/style/page/settings.css
+2
View File
@@ -35,6 +35,7 @@
padding: 11px 15px; padding: 11px 15px;
font-weight: normal; font-weight: normal;
transition: all 0.1s ease-in-out; transition: all 0.1s ease-in-out;
transform: translateZ(0);
color: rgba(255, 255, 255, 0.8); color: rgba(255, 255, 255, 0.8);
} }
.page.settings .tabs a:hover, .page.settings .tabs .active a { .page.settings .tabs a:hover, .page.settings .tabs .active a {
@@ -49,6 +50,7 @@
padding: 0; padding: 0;
overflow: hidden; overflow: hidden;
transition: all 1s ease-in-out; transition: all 1s ease-in-out;
transform: translateZ(0);
max-height: 0; max-height: 0;
} }
.page.settings .tabs > .active .subtabs { .page.settings .tabs > .active .subtabs {
couchpotato/templates/_desktop.html
+2
View File
@@ -45,6 +45,8 @@
<link href="{{ url_for('web.static', filename='images/favicon.ico') }}" rel="icon" type="image/x-icon" /> <link href="{{ url_for('web.static', filename='images/favicon.ico') }}" rel="icon" type="image/x-icon" />
<link rel="apple-touch-icon" href="{{ url_for('web.static', filename='images/homescreen.png') }}" /> <link rel="apple-touch-icon" href="{{ url_for('web.static', filename='images/homescreen.png') }}" />
<script type="text/javascript" src="https://www.youtube.com/player_api" defer="defer"></script>
<script type="text/javascript"> <script type="text/javascript">
window.addEvent('domready', function() { window.addEvent('domready', function() {
libs/BeautifulSoup.py
-2015
View File
File diff suppressed because it is too large Load Diff
libs/bs4/__init__.py
+355
View File
@@ -0,0 +1,355 @@
"""Beautiful Soup
Elixir and Tonic
"The Screen-Scraper's Friend"
http://www.crummy.com/software/BeautifulSoup/
Beautiful Soup uses a pluggable XML or HTML parser to parse a
(possibly invalid) document into a tree representation. Beautiful Soup
provides provides methods and Pythonic idioms that make it easy to
navigate, search, and modify the parse tree.
Beautiful Soup works with Python 2.6 and up. It works better if lxml
and/or html5lib is installed.
For more than you ever wanted to know about Beautiful Soup, see the
documentation:
http://www.crummy.com/software/BeautifulSoup/bs4/doc/
"""
__author__ = "Leonard Richardson (leonardr@segfault.org)"
__version__ = "4.1.0"
__copyright__ = "Copyright (c) 2004-2012 Leonard Richardson"
__license__ = "MIT"
__all__ = ['BeautifulSoup']
import re
import warnings
from .builder import builder_registry
from .dammit import UnicodeDammit
from .element import (
CData,
Comment,
DEFAULT_OUTPUT_ENCODING,
Declaration,
Doctype,
NavigableString,
PageElement,
ProcessingInstruction,
ResultSet,
SoupStrainer,
Tag,
)
# The very first thing we do is give a useful error if someone is
# running this code under Python 3 without converting it.
syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
class BeautifulSoup(Tag):
"""
This class defines the basic interface called by the tree builders.
These methods will be called by the parser:
reset()
feed(markup)
The tree builder may call these methods from its feed() implementation:
handle_starttag(name, attrs) # See note about return value
handle_endtag(name)
handle_data(data) # Appends to the current data node
endData(containerClass=NavigableString) # Ends the current data node
No matter how complicated the underlying parser is, you should be
able to build a tree using 'start tag' events, 'end tag' events,
'data' events, and "done with data" events.
If you encounter an empty-element tag (aka a self-closing tag,
like HTML's <br> tag), call handle_starttag and then
handle_endtag.
"""
ROOT_TAG_NAME = u'[document]'
# If the end-user gives no indication which tree builder they
# want, look for one with these features.
DEFAULT_BUILDER_FEATURES = ['html', 'fast']
# Used when determining whether a text node is all whitespace and
# can be replaced with a single space. A text node that contains
# fancy Unicode spaces (usually non-breaking) should be left
# alone.
STRIP_ASCII_SPACES = {9: None, 10: None, 12: None, 13: None, 32: None, }
def __init__(self, markup="", features=None, builder=None,
parse_only=None, from_encoding=None, **kwargs):
"""The Soup object is initialized as the 'root tag', and the
provided markup (which can be a string or a file-like object)
is fed into the underlying parser."""
if 'convertEntities' in kwargs:
warnings.warn(
"BS4 does not respect the convertEntities argument to the "
"BeautifulSoup constructor. Entities are always converted "
"to Unicode characters.")
if 'markupMassage' in kwargs:
del kwargs['markupMassage']
warnings.warn(
"BS4 does not respect the markupMassage argument to the "
"BeautifulSoup constructor. The tree builder is responsible "
"for any necessary markup massage.")
if 'smartQuotesTo' in kwargs:
del kwargs['smartQuotesTo']
warnings.warn(
"BS4 does not respect the smartQuotesTo argument to the "
"BeautifulSoup constructor. Smart quotes are always converted "
"to Unicode characters.")
if 'selfClosingTags' in kwargs:
del kwargs['selfClosingTags']
warnings.warn(
"BS4 does not respect the selfClosingTags argument to the "
"BeautifulSoup constructor. The tree builder is responsible "
"for understanding self-closing tags.")
if 'isHTML' in kwargs:
del kwargs['isHTML']
warnings.warn(
"BS4 does not respect the isHTML argument to the "
"BeautifulSoup constructor. You can pass in features='html' "
"or features='xml' to get a builder capable of handling "
"one or the other.")
def deprecated_argument(old_name, new_name):
if old_name in kwargs:
warnings.warn(
'The "%s" argument to the BeautifulSoup constructor '
'has been renamed to "%s."' % (old_name, new_name))
value = kwargs[old_name]
del kwargs[old_name]
return value
return None
parse_only = parse_only or deprecated_argument(
"parseOnlyThese", "parse_only")
from_encoding = from_encoding or deprecated_argument(
"fromEncoding", "from_encoding")
if len(kwargs) > 0:
arg = kwargs.keys().pop()
raise TypeError(
"__init__() got an unexpected keyword argument '%s'" % arg)
if builder is None:
if isinstance(features, basestring):
features = [features]
if features is None or len(features) == 0:
features = self.DEFAULT_BUILDER_FEATURES
builder_class = builder_registry.lookup(*features)
if builder_class is None:
raise ValueError(
"Couldn't find a tree builder with the features you "
"requested: %s. Do you need to install a parser library?"
% ",".join(features))
builder = builder_class()
self.builder = builder
self.is_xml = builder.is_xml
self.builder.soup = self
self.parse_only = parse_only
self.reset()
if hasattr(markup, 'read'): # It's a file-type object.
markup = markup.read()
(self.markup, self.original_encoding, self.declared_html_encoding,
self.contains_replacement_characters) = (
self.builder.prepare_markup(markup, from_encoding))
try:
self._feed()
except StopParsing:
pass
# Clear out the markup and remove the builder's circular
# reference to this object.
self.markup = None
self.builder.soup = None
def _feed(self):
# Convert the document to Unicode.
self.builder.reset()
self.builder.feed(self.markup)
# Close out any unfinished strings and close all the open tags.
self.endData()
while self.currentTag.name != self.ROOT_TAG_NAME:
self.popTag()
def reset(self):
Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
self.hidden = 1
self.builder.reset()
self.currentData = []
self.currentTag = None
self.tagStack = []
self.pushTag(self)
def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
"""Create a new tag associated with this soup."""
return Tag(None, self.builder, name, namespace, nsprefix, attrs)
def new_string(self, s):
"""Create a new NavigableString associated with this soup."""
navigable = NavigableString(s)
navigable.setup()
return navigable
def insert_before(self, successor):
raise ValueError("BeautifulSoup objects don't support insert_before().")
def insert_after(self, successor):
raise ValueError("BeautifulSoup objects don't support insert_after().")
def popTag(self):
tag = self.tagStack.pop()
#print "Pop", tag.name
if self.tagStack:
self.currentTag = self.tagStack[-1]
return self.currentTag
def pushTag(self, tag):
#print "Push", tag.name
if self.currentTag:
self.currentTag.contents.append(tag)
self.tagStack.append(tag)
self.currentTag = self.tagStack[-1]
def endData(self, containerClass=NavigableString):
if self.currentData:
currentData = u''.join(self.currentData)
if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and
not set([tag.name for tag in self.tagStack]).intersection(
self.builder.preserve_whitespace_tags)):
if '\n' in currentData:
currentData = '\n'
else:
currentData = ' '
self.currentData = []
if self.parse_only and len(self.tagStack) <= 1 and \
(not self.parse_only.text or \
not self.parse_only.search(currentData)):
return
o = containerClass(currentData)
self.object_was_parsed(o)
def object_was_parsed(self, o):
"""Add an object to the parse tree."""
o.setup(self.currentTag, self.previous_element)
if self.previous_element:
self.previous_element.next_element = o
self.previous_element = o
self.currentTag.contents.append(o)
def _popToTag(self, name, nsprefix=None, inclusivePop=True):
"""Pops the tag stack up to and including the most recent
instance of the given tag. If inclusivePop is false, pops the tag
stack up to but *not* including the most recent instqance of
the given tag."""
#print "Popping to %s" % name
if name == self.ROOT_TAG_NAME:
return
numPops = 0
mostRecentTag = None
for i in range(len(self.tagStack) - 1, 0, -1):
if (name == self.tagStack[i].name
and nsprefix == self.tagStack[i].nsprefix == nsprefix):
numPops = len(self.tagStack) - i
break
if not inclusivePop:
numPops = numPops - 1
for i in range(0, numPops):
mostRecentTag = self.popTag()
return mostRecentTag
def handle_starttag(self, name, namespace, nsprefix, attrs):
"""Push a start tag on to the stack.
If this method returns None, the tag was rejected by the
SoupStrainer. You should proceed as if the tag had not occured
in the document. For instance, if this was a self-closing tag,
don't call handle_endtag.
"""
# print "Start tag %s: %s" % (name, attrs)
self.endData()
if (self.parse_only and len(self.tagStack) <= 1
and (self.parse_only.text
or not self.parse_only.search_tag(name, attrs))):
return None
tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
self.currentTag, self.previous_element)
if tag is None:
return tag
if self.previous_element:
self.previous_element.next_element = tag
self.previous_element = tag
self.pushTag(tag)
return tag
def handle_endtag(self, name, nsprefix=None):
#print "End tag: " + name
self.endData()
self._popToTag(name, nsprefix)
def handle_data(self, data):
self.currentData.append(data)
def decode(self, pretty_print=False,
eventual_encoding=DEFAULT_OUTPUT_ENCODING,
formatter="minimal"):
"""Returns a string or Unicode representation of this document.
To get Unicode, pass None for encoding."""
if self.is_xml:
# Print the XML declaration
encoding_part = ''
if eventual_encoding != None:
encoding_part = ' encoding="%s"' % eventual_encoding
prefix = u'<?xml version="1.0"%s?>\n' % encoding_part
else:
prefix = u''
if not pretty_print:
indent_level = None
else:
indent_level = 0
return prefix + super(BeautifulSoup, self).decode(
indent_level, eventual_encoding, formatter)
class BeautifulStoneSoup(BeautifulSoup):
"""Deprecated interface to an XML parser."""
def __init__(self, *args, **kwargs):
kwargs['features'] = 'xml'
warnings.warn(
'The BeautifulStoneSoup class is deprecated. Instead of using '
'it, pass features="xml" into the BeautifulSoup constructor.')
super(BeautifulStoneSoup, self).__init__(*args, **kwargs)
class StopParsing(Exception):
pass
#By default, act as an HTML pretty-printer.
if __name__ == '__main__':
import sys
soup = BeautifulSoup(sys.stdin)
print soup.prettify()
libs/bs4/builder/__init__.py
+307
View File
@@ -0,0 +1,307 @@
from collections import defaultdict
import itertools
import sys
from bs4.element import (
CharsetMetaAttributeValue,
ContentMetaAttributeValue,
whitespace_re
)
__all__ = [
'HTMLTreeBuilder',
'SAXTreeBuilder',
'TreeBuilder',
'TreeBuilderRegistry',
]
# Some useful features for a TreeBuilder to have.
FAST = 'fast'
PERMISSIVE = 'permissive'
STRICT = 'strict'
XML = 'xml'
HTML = 'html'
HTML_5 = 'html5'
class TreeBuilderRegistry(object):
def __init__(self):
self.builders_for_feature = defaultdict(list)
self.builders = []
def register(self, treebuilder_class):
"""Register a treebuilder based on its advertised features."""
for feature in treebuilder_class.features:
self.builders_for_feature[feature].insert(0, treebuilder_class)
self.builders.insert(0, treebuilder_class)
def lookup(self, *features):
if len(self.builders) == 0:
# There are no builders at all.
return None
if len(features) == 0:
# They didn't ask for any features. Give them the most
# recently registered builder.
return self.builders[0]
# Go down the list of features in order, and eliminate any builders
# that don't match every feature.
features = list(features)
features.reverse()
candidates = None
candidate_set = None
while len(features) > 0:
feature = features.pop()
we_have_the_feature = self.builders_for_feature.get(feature, [])
if len(we_have_the_feature) > 0:
if candidates is None:
candidates = we_have_the_feature
candidate_set = set(candidates)
else:
# Eliminate any candidates that don't have this feature.
candidate_set = candidate_set.intersection(
set(we_have_the_feature))
# The only valid candidates are the ones in candidate_set.
# Go through the original list of candidates and pick the first one
# that's in candidate_set.
if candidate_set is None:
return None
for candidate in candidates:
if candidate in candidate_set:
return candidate
return None
# The BeautifulSoup class will take feature lists from developers and use them
# to look up builders in this registry.
builder_registry = TreeBuilderRegistry()
class TreeBuilder(object):
"""Turn a document into a Beautiful Soup object tree."""
features = []
is_xml = False
preserve_whitespace_tags = set()
empty_element_tags = None # A tag will be considered an empty-element
# tag when and only when it has no contents.
# A value for these tag/attribute combinations is a space- or
# comma-separated list of CDATA, rather than a single CDATA.
cdata_list_attributes = {}
def __init__(self):
self.soup = None
def reset(self):
pass
def can_be_empty_element(self, tag_name):
"""Might a tag with this name be an empty-element tag?
The final markup may or may not actually present this tag as
self-closing.
For instance: an HTMLBuilder does not consider a <p> tag to be
an empty-element tag (it's not in
HTMLBuilder.empty_element_tags). This means an empty <p> tag
will be presented as "<p></p>", not "<p />".
The default implementation has no opinion about which tags are
empty-element tags, so a tag will be presented as an
empty-element tag if and only if it has no contents.
"<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will
be left alone.
"""
if self.empty_element_tags is None:
return True
return tag_name in self.empty_element_tags
def feed(self, markup):
raise NotImplementedError()
def prepare_markup(self, markup, user_specified_encoding=None,
document_declared_encoding=None):
return markup, None, None, False
def test_fragment_to_document(self, fragment):
"""Wrap an HTML fragment to make it look like a document.
Different parsers do this differently. For instance, lxml
introduces an empty <head> tag, and html5lib
doesn't. Abstracting this away lets us write simple tests
which run HTML fragments through the parser and compare the
results against other HTML fragments.
This method should not be used outside of tests.
"""
return fragment
def set_up_substitutions(self, tag):
return False
def _replace_cdata_list_attribute_values(self, tag_name, attrs):
"""Replaces class="foo bar" with class=["foo", "bar"]
Modifies its input in place.
"""
if self.cdata_list_attributes:
universal = self.cdata_list_attributes.get('*', [])
tag_specific = self.cdata_list_attributes.get(
tag_name.lower(), [])
for cdata_list_attr in itertools.chain(universal, tag_specific):
if cdata_list_attr in dict(attrs):
# Basically, we have a "class" attribute whose
# value is a whitespace-separated list of CSS
# classes. Split it into a list.
value = attrs[cdata_list_attr]
values = whitespace_re.split(value)
attrs[cdata_list_attr] = values
return attrs
class SAXTreeBuilder(TreeBuilder):
"""A Beautiful Soup treebuilder that listens for SAX events."""
def feed(self, markup):
raise NotImplementedError()
def close(self):
pass
def startElement(self, name, attrs):
attrs = dict((key[1], value) for key, value in list(attrs.items()))
#print "Start %s, %r" % (name, attrs)
self.soup.handle_starttag(name, attrs)
def endElement(self, name):
#print "End %s" % name
self.soup.handle_endtag(name)
def startElementNS(self, nsTuple, nodeName, attrs):
# Throw away (ns, nodeName) for now.
self.startElement(nodeName, attrs)
def endElementNS(self, nsTuple, nodeName):
# Throw away (ns, nodeName) for now.
self.endElement(nodeName)
#handler.endElementNS((ns, node.nodeName), node.nodeName)
def startPrefixMapping(self, prefix, nodeValue):
# Ignore the prefix for now.
pass
def endPrefixMapping(self, prefix):
# Ignore the prefix for now.
# handler.endPrefixMapping(prefix)
pass
def characters(self, content):
self.soup.handle_data(content)
def startDocument(self):
pass
def endDocument(self):
pass
class HTMLTreeBuilder(TreeBuilder):
"""This TreeBuilder knows facts about HTML.
Such as which tags are empty-element tags.
"""
preserve_whitespace_tags = set(['pre', 'textarea'])
empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
'spacer', 'link', 'frame', 'base'])
# The HTML standard defines these attributes as containing a
# space-separated list of values, not a single value. That is,
# class="foo bar" means that the 'class' attribute has two values,
# 'foo' and 'bar', not the single value 'foo bar'. When we
# encounter one of these attributes, we will parse its value into
# a list of values if possible. Upon output, the list will be
# converted back into a string.
cdata_list_attributes = {
"*" : ['class', 'accesskey', 'dropzone'],
"a" : ['rel', 'rev'],
"link" : ['rel', 'rev'],
"td" : ["headers"],
"th" : ["headers"],
"td" : ["headers"],
"form" : ["accept-charset"],
"object" : ["archive"],
# These are HTML5 specific, as are *.accesskey and *.dropzone above.
"area" : ["rel"],
"icon" : ["sizes"],
"iframe" : ["sandbox"],
"output" : ["for"],
}
def set_up_substitutions(self, tag):
# We are only interested in <meta> tags
if tag.name != 'meta':
return False
http_equiv = tag.get('http-equiv')
content = tag.get('content')
charset = tag.get('charset')
# We are interested in <meta> tags that say what encoding the
# document was originally in. This means HTML 5-style <meta>
# tags that provide the "charset" attribute. It also means
# HTML 4-style <meta> tags that provide the "content"
# attribute and have "http-equiv" set to "content-type".
#
# In both cases we will replace the value of the appropriate
# attribute with a standin object that can take on any
# encoding.
meta_encoding = None
if charset is not None:
# HTML 5 style:
# <meta charset="utf8">
meta_encoding = charset
tag['charset'] = CharsetMetaAttributeValue(charset)
elif (content is not None and http_equiv is not None
and http_equiv.lower() == 'content-type'):
# HTML 4 style:
# <meta http-equiv="content-type" content="text/html; charset=utf8">
tag['content'] = ContentMetaAttributeValue(content)
return (meta_encoding is not None)
def register_treebuilders_from(module):
"""Copy TreeBuilders from the given module into this module."""
# I'm fairly sure this is not the best way to do this.
this_module = sys.modules['bs4.builder']
for name in module.__all__:
obj = getattr(module, name)
if issubclass(obj, TreeBuilder):
setattr(this_module, name, obj)
this_module.__all__.append(name)
# Register the builder while we're at it.
this_module.builder_registry.register(obj)
# Builders are registered in reverse order of priority, so that custom
# builder registrations will take precedence. In general, we want lxml
# to take precedence over html5lib, because it's faster. And we only
# want to use HTMLParser as a last result.
from . import _htmlparser
register_treebuilders_from(_htmlparser)
try:
from . import _html5lib
register_treebuilders_from(_html5lib)
except ImportError:
# They don't have html5lib installed.
pass
try:
from . import _lxml
register_treebuilders_from(_lxml)
except ImportError:
# They don't have lxml installed.
pass
libs/bs4/builder/_html5lib.py
+222
View File
@@ -0,0 +1,222 @@
__all__ = [
'HTML5TreeBuilder',
]
import warnings
from bs4.builder import (
PERMISSIVE,
HTML,
HTML_5,
HTMLTreeBuilder,
)
from bs4.element import NamespacedAttribute
import html5lib
from html5lib.constants import namespaces
from bs4.element import (
Comment,
Doctype,
NavigableString,
Tag,
)
class HTML5TreeBuilder(HTMLTreeBuilder):
"""Use html5lib to build a tree."""
features = ['html5lib', PERMISSIVE, HTML_5, HTML]
def prepare_markup(self, markup, user_specified_encoding):
# Store the user-specified encoding for use later on.
self.user_specified_encoding = user_specified_encoding
return markup, None, None, False
# These methods are defined by Beautiful Soup.
def feed(self, markup):
if self.soup.parse_only is not None:
warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
parser = html5lib.HTMLParser(tree=self.create_treebuilder)
doc = parser.parse(markup, encoding=self.user_specified_encoding)
# Set the character encoding detected by the tokenizer.
if isinstance(markup, unicode):
# We need to special-case this because html5lib sets
# charEncoding to UTF-8 if it gets Unicode input.
doc.original_encoding = None
else:
doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
def create_treebuilder(self, namespaceHTMLElements):
self.underlying_builder = TreeBuilderForHtml5lib(
self.soup, namespaceHTMLElements)
return self.underlying_builder
def test_fragment_to_document(self, fragment):
"""See `TreeBuilder`."""
return u'<html><head></head><body>%s</body></html>' % fragment
class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
def __init__(self, soup, namespaceHTMLElements):
self.soup = soup
super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
def documentClass(self):
self.soup.reset()
return Element(self.soup, self.soup, None)
def insertDoctype(self, token):
name = token["name"]
publicId = token["publicId"]
systemId = token["systemId"]
doctype = Doctype.for_name_and_ids(name, publicId, systemId)
self.soup.object_was_parsed(doctype)
def elementClass(self, name, namespace):
tag = self.soup.new_tag(name, namespace)
return Element(tag, self.soup, namespace)
def commentClass(self, data):
return TextNode(Comment(data), self.soup)
def fragmentClass(self):
self.soup = BeautifulSoup("")
self.soup.name = "[document_fragment]"
return Element(self.soup, self.soup, None)
def appendChild(self, node):
# XXX This code is not covered by the BS4 tests.
self.soup.append(node.element)
def getDocument(self):
return self.soup
def getFragment(self):
return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
class AttrList(object):
def __init__(self, element):
self.element = element
self.attrs = dict(self.element.attrs)
def __iter__(self):
return list(self.attrs.items()).__iter__()
def __setitem__(self, name, value):
"set attr", name, value
self.element[name] = value
def items(self):
return list(self.attrs.items())
def keys(self):
return list(self.attrs.keys())
def __len__(self):
return len(self.attrs)
def __getitem__(self, name):
return self.attrs[name]
def __contains__(self, name):
return name in list(self.attrs.keys())
class Element(html5lib.treebuilders._base.Node):
def __init__(self, element, soup, namespace):
html5lib.treebuilders._base.Node.__init__(self, element.name)
self.element = element
self.soup = soup
self.namespace = namespace
def appendChild(self, node):
if (node.element.__class__ == NavigableString and self.element.contents
and self.element.contents[-1].__class__ == NavigableString):
# Concatenate new text onto old text node
# XXX This has O(n^2) performance, for input like
# "a</a>a</a>a</a>..."
old_element = self.element.contents[-1]
new_element = self.soup.new_string(old_element + node.element)
old_element.replace_with(new_element)
else:
self.element.append(node.element)
node.parent = self
def getAttributes(self):
return AttrList(self.element)
def setAttributes(self, attributes):
if attributes is not None and len(attributes) > 0:
converted_attributes = []
for name, value in list(attributes.items()):
if isinstance(name, tuple):
new_name = NamespacedAttribute(*name)
del attributes[name]
attributes[new_name] = value
self.soup.builder._replace_cdata_list_attribute_values(
self.name, attributes)
for name, value in attributes.items():
self.element[name] = value
# The attributes may contain variables that need substitution.
# Call set_up_substitutions manually.
#
# The Tag constructor called this method when the Tag was created,
# but we just set/changed the attributes, so call it again.
self.soup.builder.set_up_substitutions(self.element)
attributes = property(getAttributes, setAttributes)
def insertText(self, data, insertBefore=None):
text = TextNode(self.soup.new_string(data), self.soup)
if insertBefore:
self.insertBefore(text, insertBefore)
else:
self.appendChild(text)
def insertBefore(self, node, refNode):
index = self.element.index(refNode.element)
if (node.element.__class__ == NavigableString and self.element.contents
and self.element.contents[index-1].__class__ == NavigableString):
# (See comments in appendChild)
old_node = self.element.contents[index-1]
new_str = self.soup.new_string(old_node + node.element)
old_node.replace_with(new_str)
else:
self.element.insert(index, node.element)
node.parent = self
def removeChild(self, node):
node.element.extract()
def reparentChildren(self, newParent):
while self.element.contents:
child = self.element.contents[0]
child.extract()
if isinstance(child, Tag):
newParent.appendChild(
Element(child, self.soup, namespaces["html"]))
else:
newParent.appendChild(
TextNode(child, self.soup))
def cloneNode(self):
tag = self.soup.new_tag(self.element.name, self.namespace)
node = Element(tag, self.soup, self.namespace)
for key,value in self.attributes:
node.attributes[key] = value
return node
def hasContent(self):
return self.element.contents
def getNameTuple(self):
if self.namespace == None:
return namespaces["html"], self.name
else:
return self.namespace, self.name
nameTuple = property(getNameTuple)
class TextNode(Element):
def __init__(self, element, soup):
html5lib.treebuilders._base.Node.__init__(self, None)
self.element = element
self.soup = soup
def cloneNode(self):
raise NotImplementedError
libs/bs4/builder/_htmlparser.py
+244
View File
@@ -0,0 +1,244 @@
"""Use the HTMLParser library to parse HTML files that aren't too bad."""
__all__ = [
'HTMLParserTreeBuilder',
]
from HTMLParser import (
HTMLParser,
HTMLParseError,
)
import sys
import warnings
# Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
# argument, which we'd like to set to False. Unfortunately,
# http://bugs.python.org/issue13273 makes strict=True a better bet
# before Python 3.2.3.
#
# At the end of this file, we monkeypatch HTMLParser so that
# strict=True works well on Python 3.2.2.
major, minor, release = sys.version_info[:3]
CONSTRUCTOR_TAKES_STRICT = (
major > 3
or (major == 3 and minor > 2)
or (major == 3 and minor == 2 and release >= 3))
from bs4.element import (
CData,
Comment,
Declaration,
Doctype,
ProcessingInstruction,
)
from bs4.dammit import EntitySubstitution, UnicodeDammit
from bs4.builder import (
HTML,
HTMLTreeBuilder,
STRICT,
)
HTMLPARSER = 'html.parser'
class BeautifulSoupHTMLParser(HTMLParser):
def handle_starttag(self, name, attrs):
# XXX namespace
self.soup.handle_starttag(name, None, None, dict(attrs))
def handle_endtag(self, name):
self.soup.handle_endtag(name)
def handle_data(self, data):
self.soup.handle_data(data)
def handle_charref(self, name):
# XXX workaround for a bug in HTMLParser. Remove this once
# it's fixed.
if name.startswith('x'):
real_name = int(name.lstrip('x'), 16)
else:
real_name = int(name)
try:
data = unichr(real_name)
except (ValueError, OverflowError), e:
data = u"\N{REPLACEMENT CHARACTER}"
self.handle_data(data)
def handle_entityref(self, name):
character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
if character is not None:
data = character
else:
data = "&%s;" % name
self.handle_data(data)
def handle_comment(self, data):
self.soup.endData()
self.soup.handle_data(data)
self.soup.endData(Comment)
def handle_decl(self, data):
self.soup.endData()
if data.startswith("DOCTYPE "):
data = data[len("DOCTYPE "):]
self.soup.handle_data(data)
self.soup.endData(Doctype)
def unknown_decl(self, data):
if data.upper().startswith('CDATA['):
cls = CData
data = data[len('CDATA['):]
else:
cls = Declaration
self.soup.endData()
self.soup.handle_data(data)
self.soup.endData(cls)
def handle_pi(self, data):
self.soup.endData()
if data.endswith("?") and data.lower().startswith("xml"):
# "An XHTML processing instruction using the trailing '?'
# will cause the '?' to be included in data." - HTMLParser
# docs.
#
# Strip the question mark so we don't end up with two
# question marks.
data = data[:-1]
self.soup.handle_data(data)
self.soup.endData(ProcessingInstruction)
class HTMLParserTreeBuilder(HTMLTreeBuilder):
is_xml = False
features = [HTML, STRICT, HTMLPARSER]
def __init__(self, *args, **kwargs):
if CONSTRUCTOR_TAKES_STRICT:
kwargs['strict'] = False
self.parser_args = (args, kwargs)
def prepare_markup(self, markup, user_specified_encoding=None,
document_declared_encoding=None):
"""
:return: A 4-tuple (markup, original encoding, encoding
declared within markup, whether any characters had to be
replaced with REPLACEMENT CHARACTER).
"""
if isinstance(markup, unicode):
return markup, None, None, False
try_encodings = [user_specified_encoding, document_declared_encoding]
dammit = UnicodeDammit(markup, try_encodings, is_html=True)
return (dammit.markup, dammit.original_encoding,
dammit.declared_html_encoding,
dammit.contains_replacement_characters)
def feed(self, markup):
args, kwargs = self.parser_args
parser = BeautifulSoupHTMLParser(*args, **kwargs)
parser.soup = self.soup
try:
parser.feed(markup)
except HTMLParseError, e:
warnings.warn(RuntimeWarning(
"Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
raise e
# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
# 3.2.3 code. This ensures they don't treat markup like <p></p> as a
# string.
#
# XXX This code can be removed once most Python 3 users are on 3.2.3.
if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
import re
attrfind_tolerant = re.compile(
r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant
locatestarttagend = re.compile(r"""
<[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
(?:\s+ # whitespace before attribute name
(?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name
(?:\s*=\s* # value indicator
(?:'[^']*' # LITA-enclosed value
|\"[^\"]*\" # LIT-enclosed value
|[^'\">\s]+ # bare value
)
)?
)
)*
\s* # trailing whitespace
""", re.VERBOSE)
BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend
from html.parser import tagfind, attrfind
def parse_starttag(self, i):
self.__starttag_text = None
endpos = self.check_for_whole_start_tag(i)
if endpos < 0:
return endpos
rawdata = self.rawdata
self.__starttag_text = rawdata[i:endpos]
# Now parse the data between i+1 and j into a tag and attrs
attrs = []
match = tagfind.match(rawdata, i+1)
assert match, 'unexpected call to parse_starttag()'
k = match.end()
self.lasttag = tag = rawdata[i+1:k].lower()
while k < endpos:
if self.strict:
m = attrfind.match(rawdata, k)
else:
m = attrfind_tolerant.match(rawdata, k)
if not m:
break
attrname, rest, attrvalue = m.group(1, 2, 3)
if not rest:
attrvalue = None
elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
attrvalue[:1] == '"' == attrvalue[-1:]:
attrvalue = attrvalue[1:-1]
if attrvalue:
attrvalue = self.unescape(attrvalue)
attrs.append((attrname.lower(), attrvalue))
k = m.end()
end = rawdata[k:endpos].strip()
if end not in (">", "/>"):
lineno, offset = self.getpos()
if "\n" in self.__starttag_text:
lineno = lineno + self.__starttag_text.count("\n")
offset = len(self.__starttag_text) \
- self.__starttag_text.rfind("\n")
else:
offset = offset + len(self.__starttag_text)
if self.strict:
self.error("junk characters in start tag: %r"
% (rawdata[k:endpos][:20],))
self.handle_data(rawdata[i:endpos])
return endpos
if end.endswith('/>'):
# XHTML-style empty tag: <span attr="value" />
self.handle_startendtag(tag, attrs)
else:
self.handle_starttag(tag, attrs)
if tag in self.CDATA_CONTENT_ELEMENTS:
self.set_cdata_mode(tag)
return endpos
def set_cdata_mode(self, elem):
self.cdata_elem = elem.lower()
self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
BeautifulSoupHTMLParser.parse_starttag = parse_starttag
BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode
CONSTRUCTOR_TAKES_STRICT = True
libs/bs4/builder/_lxml.py
+179
View File
@@ -0,0 +1,179 @@
__all__ = [
'LXMLTreeBuilderForXML',
'LXMLTreeBuilder',
]
from StringIO import StringIO
import collections
from lxml import etree
from bs4.element import Comment, Doctype, NamespacedAttribute
from bs4.builder import (
FAST,
HTML,
HTMLTreeBuilder,
PERMISSIVE,
TreeBuilder,
XML)
from bs4.dammit import UnicodeDammit
LXML = 'lxml'
class LXMLTreeBuilderForXML(TreeBuilder):
DEFAULT_PARSER_CLASS = etree.XMLParser
is_xml = True
# Well, it's permissive by XML parser standards.
features = [LXML, XML, FAST, PERMISSIVE]
CHUNK_SIZE = 512
@property
def default_parser(self):
# This can either return a parser object or a class, which
# will be instantiated with default arguments.
return etree.XMLParser(target=self, strip_cdata=False, recover=True)
def __init__(self, parser=None, empty_element_tags=None):
if empty_element_tags is not None:
self.empty_element_tags = set(empty_element_tags)
if parser is None:
# Use the default parser.
parser = self.default_parser
if isinstance(parser, collections.Callable):
# Instantiate the parser with default arguments
parser = parser(target=self, strip_cdata=False)
self.parser = parser
self.soup = None
self.nsmaps = None
def _getNsTag(self, tag):
# Split the namespace URL out of a fully-qualified lxml tag
# name. Copied from lxml's src/lxml/sax.py.
if tag[0] == '{':
return tuple(tag[1:].split('}', 1))
else:
return (None, tag)
def prepare_markup(self, markup, user_specified_encoding=None,
document_declared_encoding=None):
"""
:return: A 3-tuple (markup, original encoding, encoding
declared within markup).
"""
if isinstance(markup, unicode):
return markup, None, None, False
try_encodings = [user_specified_encoding, document_declared_encoding]
dammit = UnicodeDammit(markup, try_encodings, is_html=True)
return (dammit.markup, dammit.original_encoding,
dammit.declared_html_encoding,
dammit.contains_replacement_characters)
def feed(self, markup):
if isinstance(markup, basestring):
markup = StringIO(markup)
# Call feed() at least once, even if the markup is empty,
# or the parser won't be initialized.
data = markup.read(self.CHUNK_SIZE)
self.parser.feed(data)
while data != '':
# Now call feed() on the rest of the data, chunk by chunk.
data = markup.read(self.CHUNK_SIZE)
if data != '':
self.parser.feed(data)
self.parser.close()
def close(self):
self.nsmaps = None
def start(self, name, attrs, nsmap={}):
# Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
attrs = dict(attrs)
nsprefix = None
# Invert each namespace map as it comes in.
if len(nsmap) == 0 and self.nsmaps != None:
# There are no new namespaces for this tag, but namespaces
# are in play, so we need a separate tag stack to know
# when they end.
self.nsmaps.append(None)
elif len(nsmap) > 0:
# A new namespace mapping has come into play.
if self.nsmaps is None:
self.nsmaps = []
inverted_nsmap = dict((value, key) for key, value in nsmap.items())
self.nsmaps.append(inverted_nsmap)
# Also treat the namespace mapping as a set of attributes on the
# tag, so we can recreate it later.
attrs = attrs.copy()
for prefix, namespace in nsmap.items():
attribute = NamespacedAttribute(
"xmlns", prefix, "http://www.w3.org/2000/xmlns/")
attrs[attribute] = namespace
namespace, name = self._getNsTag(name)
if namespace is not None:
for inverted_nsmap in reversed(self.nsmaps):
if inverted_nsmap is not None and namespace in inverted_nsmap:
nsprefix = inverted_nsmap[namespace]
break
self.soup.handle_starttag(name, namespace, nsprefix, attrs)
def end(self, name):
self.soup.endData()
completed_tag = self.soup.tagStack[-1]
namespace, name = self._getNsTag(name)
nsprefix = None
if namespace is not None:
for inverted_nsmap in reversed(self.nsmaps):
if inverted_nsmap is not None and namespace in inverted_nsmap:
nsprefix = inverted_nsmap[namespace]
break
self.soup.handle_endtag(name, nsprefix)
if self.nsmaps != None:
# This tag, or one of its parents, introduced a namespace
# mapping, so pop it off the stack.
self.nsmaps.pop()
if len(self.nsmaps) == 0:
# Namespaces are no longer in play, so don't bother keeping
# track of the namespace stack.
self.nsmaps = None
def pi(self, target, data):
pass
def data(self, content):
self.soup.handle_data(content)
def doctype(self, name, pubid, system):
self.soup.endData()
doctype = Doctype.for_name_and_ids(name, pubid, system)
self.soup.object_was_parsed(doctype)
def comment(self, content):
"Handle comments as Comment objects."
self.soup.endData()
self.soup.handle_data(content)
self.soup.endData(Comment)
def test_fragment_to_document(self, fragment):
"""See `TreeBuilder`."""
return u'<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
features = [LXML, HTML, FAST, PERMISSIVE]
is_xml = False
@property
def default_parser(self):
return etree.HTMLParser
def feed(self, markup):
self.parser.feed(markup)
self.parser.close()
def test_fragment_to_document(self, fragment):
"""See `TreeBuilder`."""
return u'<html><body>%s</body></html>' % fragment
libs/bs4/dammit.py
+792
View File
@@ -0,0 +1,792 @@
# -*- coding: utf-8 -*-
"""Beautiful Soup bonus library: Unicode, Dammit
This class forces XML data into a standard format (usually to UTF-8 or
Unicode). It is heavily based on code from Mark Pilgrim's Universal
Feed Parser. It does not rewrite the XML or HTML to reflect a new
encoding; that's the tree builder's job.
"""
import codecs
from htmlentitydefs import codepoint2name
import re
import warnings
# Autodetects character encodings. Very useful.
# Download from http://chardet.feedparser.org/
# or 'apt-get install python-chardet'
# or 'easy_install chardet'
try:
import chardet
#import chardet.constants
#chardet.constants._debug = 1
except ImportError:
chardet = None
# Available from http://cjkpython.i18n.org/.
try:
import iconv_codec
except ImportError:
pass
xml_encoding_re = re.compile(
'^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
html_meta_re = re.compile(
'<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
class EntitySubstitution(object):
"""Substitute XML or HTML entities for the corresponding characters."""
def _populate_class_variables():
lookup = {}
reverse_lookup = {}
characters_for_re = []
for codepoint, name in list(codepoint2name.items()):
character = unichr(codepoint)
if codepoint != 34:
# There's no point in turning the quotation mark into
# &quot;, unless it happens within an attribute value, which
# is handled elsewhere.
characters_for_re.append(character)
lookup[character] = name
# But we do want to turn &quot; into the quotation mark.
reverse_lookup[name] = character
re_definition = "[%s]" % "".join(characters_for_re)
return lookup, reverse_lookup, re.compile(re_definition)
(CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
CHARACTER_TO_XML_ENTITY = {
"'": "apos",
'"': "quot",
"&": "amp",
"<": "lt",
">": "gt",
}
BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
"&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
")")
@classmethod
def _substitute_html_entity(cls, matchobj):
entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
return "&%s;" % entity
@classmethod
def _substitute_xml_entity(cls, matchobj):
"""Used with a regular expression to substitute the
appropriate XML entity for an XML special character."""
entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
return "&%s;" % entity
@classmethod
def quoted_attribute_value(self, value):
"""Make a value into a quoted XML attribute, possibly escaping it.
Most strings will be quoted using double quotes.
Bob's Bar -> "Bob's Bar"
If a string contains double quotes, it will be quoted using
single quotes.
Welcome to "my bar" -> 'Welcome to "my bar"'
If a string contains both single and double quotes, the
double quotes will be escaped, and the string will be quoted
using double quotes.
Welcome to "Bob's Bar" -> "Welcome to &quot;Bob's bar&quot;
"""
quote_with = '"'
if '"' in value:
if "'" in value:
# The string contains both single and double
# quotes. Turn the double quotes into
# entities. We quote the double quotes rather than
# the single quotes because the entity name is
# "&quot;" whether this is HTML or XML. If we
# quoted the single quotes, we'd have to decide
# between &apos; and &squot;.
replace_with = "&quot;"
value = value.replace('"', replace_with)
else:
# There are double quotes but no single quotes.
# We can use single quotes to quote the attribute.
quote_with = "'"
return quote_with + value + quote_with
@classmethod
def substitute_xml(cls, value, make_quoted_attribute=False):
"""Substitute XML entities for special XML characters.
:param value: A string to be substituted. The less-than sign will
become &lt;, the greater-than sign will become &gt;, and any
ampersands that are not part of an entity defition will
become &amp;.
:param make_quoted_attribute: If True, then the string will be
quoted, as befits an attribute value.
"""
# Escape angle brackets, and ampersands that aren't part of
# entities.
value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
cls._substitute_xml_entity, value)
if make_quoted_attribute:
value = cls.quoted_attribute_value(value)
return value
@classmethod
def substitute_html(cls, s):
"""Replace certain Unicode characters with named HTML entities.
This differs from data.encode(encoding, 'xmlcharrefreplace')
in that the goal is to make the result more readable (to those
with ASCII displays) rather than to recover from
errors. There's absolutely nothing wrong with a UTF-8 string
containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
character with "&eacute;" will make it more readable to some
people.
"""
return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
cls._substitute_html_entity, s)
class UnicodeDammit:
"""A class for detecting the encoding of a *ML document and
converting it to a Unicode string. If the source encoding is
windows-1252, can replace MS smart quotes with their HTML or XML
equivalents."""
# This dictionary maps commonly seen values for "charset" in HTML
# meta tags to the corresponding Python codec names. It only covers
# values that aren't in Python's aliases and can't be determined
# by the heuristics in find_codec.
CHARSET_ALIASES = {"macintosh": "mac-roman",
"x-sjis": "shift-jis"}
ENCODINGS_WITH_SMART_QUOTES = [
"windows-1252",
"iso-8859-1",
"iso-8859-2",
]
def __init__(self, markup, override_encodings=[],
smart_quotes_to=None, is_html=False):
self.declared_html_encoding = None
self.smart_quotes_to = smart_quotes_to
self.tried_encodings = []
self.contains_replacement_characters = False
if markup == '' or isinstance(markup, unicode):
self.markup = markup
self.unicode_markup = unicode(markup)
self.original_encoding = None
return
new_markup, document_encoding, sniffed_encoding = \
self._detectEncoding(markup, is_html)
self.markup = new_markup
u = None
if new_markup != markup:
# _detectEncoding modified the markup, then converted it to
# Unicode and then to UTF-8. So convert it from UTF-8.
u = self._convert_from("utf8")
self.original_encoding = sniffed_encoding
if not u:
for proposed_encoding in (
override_encodings + [document_encoding, sniffed_encoding]):
if proposed_encoding is not None:
u = self._convert_from(proposed_encoding)
if u:
break
# If no luck and we have auto-detection library, try that:
if not u and chardet and not isinstance(self.markup, unicode):
u = self._convert_from(chardet.detect(self.markup)['encoding'])
# As a last resort, try utf-8 and windows-1252:
if not u:
for proposed_encoding in ("utf-8", "windows-1252"):
u = self._convert_from(proposed_encoding)
if u:
break
# As an absolute last resort, try the encodings again with
# character replacement.
if not u:
for proposed_encoding in (
override_encodings + [
document_encoding, sniffed_encoding, "utf-8", "windows-1252"]):
if proposed_encoding != "ascii":
u = self._convert_from(proposed_encoding, "replace")
if u is not None:
warnings.warn(
UnicodeWarning(
"Some characters could not be decoded, and were "
"replaced with REPLACEMENT CHARACTER."))
self.contains_replacement_characters = True
break
# We could at this point force it to ASCII, but that would
# destroy so much data that I think giving up is better
self.unicode_markup = u
if not u:
self.original_encoding = None
def _sub_ms_char(self, match):
"""Changes a MS smart quote character to an XML or HTML
entity, or an ASCII character."""
orig = match.group(1)
if self.smart_quotes_to == 'ascii':
sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
else:
sub = self.MS_CHARS.get(orig)
if type(sub) == tuple:
if self.smart_quotes_to == 'xml':
sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
else:
sub = '&'.encode() + sub[0].encode() + ';'.encode()
else:
sub = sub.encode()
return sub
def _convert_from(self, proposed, errors="strict"):
proposed = self.find_codec(proposed)
if not proposed or (proposed, errors) in self.tried_encodings:
return None
self.tried_encodings.append((proposed, errors))
markup = self.markup
# Convert smart quotes to HTML if coming from an encoding
# that might have them.
if (self.smart_quotes_to is not None
and proposed.lower() in self.ENCODINGS_WITH_SMART_QUOTES):
smart_quotes_re = b"([\x80-\x9f])"
smart_quotes_compiled = re.compile(smart_quotes_re)
markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
try:
#print "Trying to convert document to %s (errors=%s)" % (
# proposed, errors)
u = self._to_unicode(markup, proposed, errors)
self.markup = u
self.original_encoding = proposed
except Exception as e:
#print "That didn't work!"
#print e
return None
#print "Correct encoding: %s" % proposed
return self.markup
def _to_unicode(self, data, encoding, errors="strict"):
'''Given a string and its encoding, decodes the string into Unicode.
%encoding is a string recognized by encodings.aliases'''
# strip Byte Order Mark (if present)
if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
and (data[2:4] != '\x00\x00'):
encoding = 'utf-16be'
data = data[2:]
elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
and (data[2:4] != '\x00\x00'):
encoding = 'utf-16le'
data = data[2:]
elif data[:3] == '\xef\xbb\xbf':
encoding = 'utf-8'
data = data[3:]
elif data[:4] == '\x00\x00\xfe\xff':
encoding = 'utf-32be'
data = data[4:]
elif data[:4] == '\xff\xfe\x00\x00':
encoding = 'utf-32le'
data = data[4:]
newdata = unicode(data, encoding, errors)
return newdata
def _detectEncoding(self, xml_data, is_html=False):
"""Given a document, tries to detect its XML encoding."""
xml_encoding = sniffed_xml_encoding = None
try:
if xml_data[:4] == b'\x4c\x6f\xa7\x94':
# EBCDIC
xml_data = self._ebcdic_to_ascii(xml_data)
elif xml_data[:4] == b'\x00\x3c\x00\x3f':
# UTF-16BE
sniffed_xml_encoding = 'utf-16be'
xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xfe\xff') \
and (xml_data[2:4] != b'\x00\x00'):
# UTF-16BE with BOM
sniffed_xml_encoding = 'utf-16be'
xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
elif xml_data[:4] == b'\x3c\x00\x3f\x00':
# UTF-16LE
sniffed_xml_encoding = 'utf-16le'
xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xff\xfe') and \
(xml_data[2:4] != b'\x00\x00'):
# UTF-16LE with BOM
sniffed_xml_encoding = 'utf-16le'
xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
elif xml_data[:4] == b'\x00\x00\x00\x3c':
# UTF-32BE
sniffed_xml_encoding = 'utf-32be'
xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
elif xml_data[:4] == b'\x3c\x00\x00\x00':
# UTF-32LE
sniffed_xml_encoding = 'utf-32le'
xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
elif xml_data[:4] == b'\x00\x00\xfe\xff':
# UTF-32BE with BOM
sniffed_xml_encoding = 'utf-32be'
xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
elif xml_data[:4] == b'\xff\xfe\x00\x00':
# UTF-32LE with BOM
sniffed_xml_encoding = 'utf-32le'
xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
elif xml_data[:3] == b'\xef\xbb\xbf':
# UTF-8 with BOM
sniffed_xml_encoding = 'utf-8'
xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
else:
sniffed_xml_encoding = 'ascii'
pass
except:
xml_encoding_match = None
xml_encoding_match = xml_encoding_re.match(xml_data)
if not xml_encoding_match and is_html:
xml_encoding_match = html_meta_re.search(xml_data)
if xml_encoding_match is not None:
xml_encoding = xml_encoding_match.groups()[0].decode(
'ascii').lower()
if is_html:
self.declared_html_encoding = xml_encoding
if sniffed_xml_encoding and \
(xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
'iso-10646-ucs-4', 'ucs-4', 'csucs4',
'utf-16', 'utf-32', 'utf_16', 'utf_32',
'utf16', 'u16')):
xml_encoding = sniffed_xml_encoding
return xml_data, xml_encoding, sniffed_xml_encoding
def find_codec(self, charset):
return self._codec(self.CHARSET_ALIASES.get(charset, charset)) \
or (charset and self._codec(charset.replace("-", ""))) \
or (charset and self._codec(charset.replace("-", "_"))) \
or charset
def _codec(self, charset):
if not charset:
return charset
codec = None
try:
codecs.lookup(charset)
codec = charset
except (LookupError, ValueError):
pass
return codec
EBCDIC_TO_ASCII_MAP = None
def _ebcdic_to_ascii(self, s):
c = self.__class__
if not c.EBCDIC_TO_ASCII_MAP:
emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,
201,202,106,107,108,109,110,111,112,113,114,203,204,205,
206,207,208,209,126,115,116,117,118,119,120,121,122,210,
211,212,213,214,215,216,217,218,219,220,221,222,223,224,
225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72,
73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81,
82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89,
90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57,
250,251,252,253,254,255)
import string
c.EBCDIC_TO_ASCII_MAP = string.maketrans(
''.join(map(chr, list(range(256)))), ''.join(map(chr, emap)))
return s.translate(c.EBCDIC_TO_ASCII_MAP)
# A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
MS_CHARS = {b'\x80': ('euro', '20AC'),
b'\x81': ' ',
b'\x82': ('sbquo', '201A'),
b'\x83': ('fnof', '192'),
b'\x84': ('bdquo', '201E'),
b'\x85': ('hellip', '2026'),
b'\x86': ('dagger', '2020'),
b'\x87': ('Dagger', '2021'),
b'\x88': ('circ', '2C6'),
b'\x89': ('permil', '2030'),
b'\x8A': ('Scaron', '160'),
b'\x8B': ('lsaquo', '2039'),
b'\x8C': ('OElig', '152'),
b'\x8D': '?',
b'\x8E': ('#x17D', '17D'),
b'\x8F': '?',
b'\x90': '?',
b'\x91': ('lsquo', '2018'),
b'\x92': ('rsquo', '2019'),
b'\x93': ('ldquo', '201C'),
b'\x94': ('rdquo', '201D'),
b'\x95': ('bull', '2022'),
b'\x96': ('ndash', '2013'),
b'\x97': ('mdash', '2014'),
b'\x98': ('tilde', '2DC'),
b'\x99': ('trade', '2122'),
b'\x9a': ('scaron', '161'),
b'\x9b': ('rsaquo', '203A'),
b'\x9c': ('oelig', '153'),
b'\x9d': '?',
b'\x9e': ('#x17E', '17E'),
b'\x9f': ('Yuml', ''),}
# A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
# horrors like stripping diacritical marks to turn á into a, but also
# contains non-horrors like turning “ into ".
MS_CHARS_TO_ASCII = {
b'\x80' : 'EUR',
b'\x81' : ' ',
b'\x82' : ',',
b'\x83' : 'f',
b'\x84' : ',,',
b'\x85' : '...',
b'\x86' : '+',
b'\x87' : '++',
b'\x88' : '^',
b'\x89' : '%',
b'\x8a' : 'S',
b'\x8b' : '<',
b'\x8c' : 'OE',
b'\x8d' : '?',
b'\x8e' : 'Z',
b'\x8f' : '?',
b'\x90' : '?',
b'\x91' : "'",
b'\x92' : "'",
b'\x93' : '"',
b'\x94' : '"',
b'\x95' : '*',
b'\x96' : '-',
b'\x97' : '--',
b'\x98' : '~',
b'\x99' : '(TM)',
b'\x9a' : 's',
b'\x9b' : '>',
b'\x9c' : 'oe',
b'\x9d' : '?',
b'\x9e' : 'z',
b'\x9f' : 'Y',
b'\xa0' : ' ',
b'\xa1' : '!',
b'\xa2' : 'c',
b'\xa3' : 'GBP',
b'\xa4' : '$', #This approximation is especially parochial--this is the
#generic currency symbol.
b'\xa5' : 'YEN',
b'\xa6' : '|',
b'\xa7' : 'S',
b'\xa8' : '..',
b'\xa9' : '',
b'\xaa' : '(th)',
b'\xab' : '<<',
b'\xac' : '!',
b'\xad' : ' ',
b'\xae' : '(R)',
b'\xaf' : '-',
b'\xb0' : 'o',
b'\xb1' : '+-',
b'\xb2' : '2',
b'\xb3' : '3',
b'\xb4' : ("'", 'acute'),
b'\xb5' : 'u',
b'\xb6' : 'P',
b'\xb7' : '*',
b'\xb8' : ',',
b'\xb9' : '1',
b'\xba' : '(th)',
b'\xbb' : '>>',
b'\xbc' : '1/4',
b'\xbd' : '1/2',
b'\xbe' : '3/4',
b'\xbf' : '?',
b'\xc0' : 'A',
b'\xc1' : 'A',
b'\xc2' : 'A',
b'\xc3' : 'A',
b'\xc4' : 'A',
b'\xc5' : 'A',
b'\xc6' : 'AE',
b'\xc7' : 'C',
b'\xc8' : 'E',
b'\xc9' : 'E',
b'\xca' : 'E',
b'\xcb' : 'E',
b'\xcc' : 'I',
b'\xcd' : 'I',
b'\xce' : 'I',
b'\xcf' : 'I',
b'\xd0' : 'D',
b'\xd1' : 'N',
b'\xd2' : 'O',
b'\xd3' : 'O',
b'\xd4' : 'O',
b'\xd5' : 'O',
b'\xd6' : 'O',
b'\xd7' : '*',
b'\xd8' : 'O',
b'\xd9' : 'U',
b'\xda' : 'U',
b'\xdb' : 'U',
b'\xdc' : 'U',
b'\xdd' : 'Y',
b'\xde' : 'b',
b'\xdf' : 'B',
b'\xe0' : 'a',
b'\xe1' : 'a',
b'\xe2' : 'a',
b'\xe3' : 'a',
b'\xe4' : 'a',
b'\xe5' : 'a',
b'\xe6' : 'ae',
b'\xe7' : 'c',
b'\xe8' : 'e',
b'\xe9' : 'e',
b'\xea' : 'e',
b'\xeb' : 'e',
b'\xec' : 'i',
b'\xed' : 'i',
b'\xee' : 'i',
b'\xef' : 'i',
b'\xf0' : 'o',
b'\xf1' : 'n',
b'\xf2' : 'o',
b'\xf3' : 'o',
b'\xf4' : 'o',
b'\xf5' : 'o',
b'\xf6' : 'o',
b'\xf7' : '/',
b'\xf8' : 'o',
b'\xf9' : 'u',
b'\xfa' : 'u',
b'\xfb' : 'u',
b'\xfc' : 'u',
b'\xfd' : 'y',
b'\xfe' : 'b',
b'\xff' : 'y',
}
# A map used when removing rogue Windows-1252/ISO-8859-1
# characters in otherwise UTF-8 documents.
#
# Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
# Windows-1252.
WINDOWS_1252_TO_UTF8 = {
0x80 : b'\xe2\x82\xac', # €
0x82 : b'\xe2\x80\x9a', # ‚
0x83 : b'\xc6\x92', # ƒ
0x84 : b'\xe2\x80\x9e', # „
0x85 : b'\xe2\x80\xa6', # …
0x86 : b'\xe2\x80\xa0', # †
0x87 : b'\xe2\x80\xa1', # ‡
0x88 : b'\xcb\x86', # ˆ
0x89 : b'\xe2\x80\xb0', # ‰
0x8a : b'\xc5\xa0', # Š
0x8b : b'\xe2\x80\xb9', # ‹
0x8c : b'\xc5\x92', # Œ
0x8e : b'\xc5\xbd', # Ž
0x91 : b'\xe2\x80\x98', # ‘
0x92 : b'\xe2\x80\x99', # ’
0x93 : b'\xe2\x80\x9c', # “
0x94 : b'\xe2\x80\x9d', # ”
0x95 : b'\xe2\x80\xa2', # •
0x96 : b'\xe2\x80\x93', # –
0x97 : b'\xe2\x80\x94', # —
0x98 : b'\xcb\x9c', # ˜
0x99 : b'\xe2\x84\xa2', # ™
0x9a : b'\xc5\xa1', # š
0x9b : b'\xe2\x80\xba', # ›
0x9c : b'\xc5\x93', # œ
0x9e : b'\xc5\xbe', # ž
0x9f : b'\xc5\xb8', # Ÿ
0xa0 : b'\xc2\xa0', #  
0xa1 : b'\xc2\xa1', # ¡
0xa2 : b'\xc2\xa2', # ¢
0xa3 : b'\xc2\xa3', # £
0xa4 : b'\xc2\xa4', # ¤
0xa5 : b'\xc2\xa5', # ¥
0xa6 : b'\xc2\xa6', # ¦
0xa7 : b'\xc2\xa7', # §
0xa8 : b'\xc2\xa8', # ¨
0xa9 : b'\xc2\xa9', # ©
0xaa : b'\xc2\xaa', # ª
0xab : b'\xc2\xab', # «
0xac : b'\xc2\xac', # ¬
0xad : b'\xc2\xad', # ­
0xae : b'\xc2\xae', # ®
0xaf : b'\xc2\xaf', # ¯
0xb0 : b'\xc2\xb0', # °
0xb1 : b'\xc2\xb1', # ±
0xb2 : b'\xc2\xb2', # ²
0xb3 : b'\xc2\xb3', # ³
0xb4 : b'\xc2\xb4', # ´
0xb5 : b'\xc2\xb5', # µ
0xb6 : b'\xc2\xb6', # ¶
0xb7 : b'\xc2\xb7', # ·
0xb8 : b'\xc2\xb8', # ¸
0xb9 : b'\xc2\xb9', # ¹
0xba : b'\xc2\xba', # º
0xbb : b'\xc2\xbb', # »
0xbc : b'\xc2\xbc', # ¼
0xbd : b'\xc2\xbd', # ½
0xbe : b'\xc2\xbe', # ¾
0xbf : b'\xc2\xbf', # ¿
0xc0 : b'\xc3\x80', # À
0xc1 : b'\xc3\x81', # Á
0xc2 : b'\xc3\x82', # Â
0xc3 : b'\xc3\x83', # Ã
0xc4 : b'\xc3\x84', # Ä
0xc5 : b'\xc3\x85', # Å
0xc6 : b'\xc3\x86', # Æ
0xc7 : b'\xc3\x87', # Ç
0xc8 : b'\xc3\x88', # È
0xc9 : b'\xc3\x89', # É
0xca : b'\xc3\x8a', # Ê
0xcb : b'\xc3\x8b', # Ë
0xcc : b'\xc3\x8c', # Ì
0xcd : b'\xc3\x8d', # Í
0xce : b'\xc3\x8e', # Î
0xcf : b'\xc3\x8f', # Ï
0xd0 : b'\xc3\x90', # Ð
0xd1 : b'\xc3\x91', # Ñ
0xd2 : b'\xc3\x92', # Ò
0xd3 : b'\xc3\x93', # Ó
0xd4 : b'\xc3\x94', # Ô
0xd5 : b'\xc3\x95', # Õ
0xd6 : b'\xc3\x96', # Ö
0xd7 : b'\xc3\x97', # ×
0xd8 : b'\xc3\x98', # Ø
0xd9 : b'\xc3\x99', # Ù
0xda : b'\xc3\x9a', # Ú
0xdb : b'\xc3\x9b', # Û
0xdc : b'\xc3\x9c', # Ü
0xdd : b'\xc3\x9d', # Ý
0xde : b'\xc3\x9e', # Þ
0xdf : b'\xc3\x9f', # ß
0xe0 : b'\xc3\xa0', # à
0xe1 : b'\xa1', # á
0xe2 : b'\xc3\xa2', # â
0xe3 : b'\xc3\xa3', # ã
0xe4 : b'\xc3\xa4', # ä
0xe5 : b'\xc3\xa5', # å
0xe6 : b'\xc3\xa6', # æ
0xe7 : b'\xc3\xa7', # ç
0xe8 : b'\xc3\xa8', # è
0xe9 : b'\xc3\xa9', # é
0xea : b'\xc3\xaa', # ê
0xeb : b'\xc3\xab', # ë
0xec : b'\xc3\xac', # ì
0xed : b'\xc3\xad', # í
0xee : b'\xc3\xae', # î
0xef : b'\xc3\xaf', # ï
0xf0 : b'\xc3\xb0', # ð
0xf1 : b'\xc3\xb1', # ñ
0xf2 : b'\xc3\xb2', # ò
0xf3 : b'\xc3\xb3', # ó
0xf4 : b'\xc3\xb4', # ô
0xf5 : b'\xc3\xb5', # õ
0xf6 : b'\xc3\xb6', # ö
0xf7 : b'\xc3\xb7', # ÷
0xf8 : b'\xc3\xb8', # ø
0xf9 : b'\xc3\xb9', # ù
0xfa : b'\xc3\xba', # ú
0xfb : b'\xc3\xbb', # û
0xfc : b'\xc3\xbc', # ü
0xfd : b'\xc3\xbd', # ý
0xfe : b'\xc3\xbe', # þ
}
MULTIBYTE_MARKERS_AND_SIZES = [
(0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
(0xe0, 0xef, 3), # 3-byte characters start with E0-EF
(0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
]
FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]
@classmethod
def detwingle(cls, in_bytes, main_encoding="utf8",
embedded_encoding="windows-1252"):
"""Fix characters from one encoding embedded in some other encoding.
Currently the only situation supported is Windows-1252 (or its
subset ISO-8859-1), embedded in UTF-8.
The input must be a bytestring. If you've already converted
the document to Unicode, you're too late.
The output is a bytestring in which `embedded_encoding`
characters have been converted to their `main_encoding`
equivalents.
"""
if embedded_encoding.replace('_', '-').lower() not in (
'windows-1252', 'windows_1252'):
raise NotImplementedError(
"Windows-1252 and ISO-8859-1 are the only currently supported "
"embedded encodings.")
if main_encoding.lower() not in ('utf8', 'utf-8'):
raise NotImplementedError(
"UTF-8 is the only currently supported main encoding.")
byte_chunks = []
chunk_start = 0
pos = 0
while pos < len(in_bytes):
byte = in_bytes[pos]
if not isinstance(byte, int):
# Python 2.x
byte = ord(byte)
if (byte >= cls.FIRST_MULTIBYTE_MARKER
and byte <= cls.LAST_MULTIBYTE_MARKER):
# This is the start of a UTF-8 multibyte character. Skip
# to the end.
for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
if byte >= start and byte <= end:
pos += size
break
elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
# We found a Windows-1252 character!
# Save the string up to this point as a chunk.
byte_chunks.append(in_bytes[chunk_start:pos])
# Now translate the Windows-1252 character into UTF-8
# and add it as another, one-byte chunk.
byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
pos += 1
chunk_start = pos
else:
# Go on to the next character.
pos += 1
if chunk_start == 0:
# The string is unchanged.
return in_bytes
else:
# Store the final chunk.
byte_chunks.append(in_bytes[chunk_start:])
return b''.join(byte_chunks)
libs/bs4/element.py
+1347
View File
File diff suppressed because it is too large Load Diff
libs/bs4/testing.py
+515
View File
@@ -0,0 +1,515 @@
"""Helper classes for tests."""
import copy
import functools
import unittest
from unittest import TestCase
from bs4 import BeautifulSoup
from bs4.element import (
CharsetMetaAttributeValue,
Comment,
ContentMetaAttributeValue,
Doctype,
SoupStrainer,
)
from bs4.builder import HTMLParserTreeBuilder
default_builder = HTMLParserTreeBuilder
class SoupTest(unittest.TestCase):
@property
def default_builder(self):
return default_builder()
def soup(self, markup, **kwargs):
"""Build a Beautiful Soup object from markup."""
builder = kwargs.pop('builder', self.default_builder)
return BeautifulSoup(markup, builder=builder, **kwargs)
def document_for(self, markup):
"""Turn an HTML fragment into a document.
The details depend on the builder.
"""
return self.default_builder.test_fragment_to_document(markup)
def assertSoupEquals(self, to_parse, compare_parsed_to=None):
builder = self.default_builder
obj = BeautifulSoup(to_parse, builder=builder)
if compare_parsed_to is None:
compare_parsed_to = to_parse
self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
class HTMLTreeBuilderSmokeTest(object):
"""A basic test of a treebuilder's competence.
Any HTML treebuilder, present or future, should be able to pass
these tests. With invalid markup, there's room for interpretation,
and different parsers can handle it differently. But with the
markup in these tests, there's not much room for interpretation.
"""
def assertDoctypeHandled(self, doctype_fragment):
"""Assert that a given doctype string is handled correctly."""
doctype_str, soup = self._document_with_doctype(doctype_fragment)
# Make sure a Doctype object was created.
doctype = soup.contents[0]
self.assertEqual(doctype.__class__, Doctype)
self.assertEqual(doctype, doctype_fragment)
self.assertEqual(str(soup)[:len(doctype_str)], doctype_str)
# Make sure that the doctype was correctly associated with the
# parse tree and that the rest of the document parsed.
self.assertEqual(soup.p.contents[0], 'foo')
def _document_with_doctype(self, doctype_fragment):
"""Generate and parse a document with the given doctype."""
doctype = '<!DOCTYPE %s>' % doctype_fragment
markup = doctype + '\n<p>foo</p>'
soup = self.soup(markup)
return doctype, soup
def test_normal_doctypes(self):
"""Make sure normal, everyday HTML doctypes are handled correctly."""
self.assertDoctypeHandled("html")
self.assertDoctypeHandled(
'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"')
def test_public_doctype_with_url(self):
doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
self.assertDoctypeHandled(doctype)
def test_system_doctype(self):
self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"')
def test_namespaced_system_doctype(self):
# We can handle a namespaced doctype with a system ID.
self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"')
def test_namespaced_public_doctype(self):
# Test a namespaced doctype with a public id.
self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"')
def test_real_xhtml_document(self):
"""A real XHTML document should come out more or less the same as it went in."""
markup = b"""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><title>Hello.</title></head>
<body>Goodbye.</body>
</html>"""
soup = self.soup(markup)
self.assertEqual(
soup.encode("utf-8").replace(b"\n", b""),
markup.replace(b"\n", b""))
def test_deepcopy(self):
"""Make sure you can copy the tree builder.
This is important because the builder is part of a
BeautifulSoup object, and we want to be able to copy that.
"""
copy.deepcopy(self.default_builder)
def test_p_tag_is_never_empty_element(self):
"""A <p> tag is never designated as an empty-element tag.
Even if the markup shows it as an empty-element tag, it
shouldn't be presented that way.
"""
soup = self.soup("<p/>")
self.assertFalse(soup.p.is_empty_element)
self.assertEqual(str(soup.p), "<p></p>")
def test_unclosed_tags_get_closed(self):
"""A tag that's not closed by the end of the document should be closed.
This applies to all tags except empty-element tags.
"""
self.assertSoupEquals("<p>", "<p></p>")
self.assertSoupEquals("<b>", "<b></b>")
self.assertSoupEquals("<br>", "<br/>")
def test_br_is_always_empty_element_tag(self):
"""A <br> tag is designated as an empty-element tag.
Some parsers treat <br></br> as one <br/> tag, some parsers as
two tags, but it should always be an empty-element tag.
"""
soup = self.soup("<br></br>")
self.assertTrue(soup.br.is_empty_element)
self.assertEqual(str(soup.br), "<br/>")
def test_nested_formatting_elements(self):
self.assertSoupEquals("<em><em></em></em>")
def test_comment(self):
# Comments are represented as Comment objects.
markup = "<p>foo<!--foobar-->baz</p>"
self.assertSoupEquals(markup)
soup = self.soup(markup)
comment = soup.find(text="foobar")
self.assertEqual(comment.__class__, Comment)
def test_preserved_whitespace_in_pre_and_textarea(self):
"""Whitespace must be preserved in <pre> and <textarea> tags."""
self.assertSoupEquals("<pre> </pre>")
self.assertSoupEquals("<textarea> woo </textarea>")
def test_nested_inline_elements(self):
"""Inline elements can be nested indefinitely."""
b_tag = "<b>Inside a B tag</b>"
self.assertSoupEquals(b_tag)
nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>"
self.assertSoupEquals(nested_b_tag)
double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>"
self.assertSoupEquals(nested_b_tag)
def test_nested_block_level_elements(self):
"""Block elements can be nested."""
soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>')
blockquote = soup.blockquote
self.assertEqual(blockquote.p.b.string, 'Foo')
self.assertEqual(blockquote.b.string, 'Foo')
def test_correctly_nested_tables(self):
"""One table can go inside another one."""
markup = ('<table id="1">'
'<tr>'
"<td>Here's another table:"
'<table id="2">'
'<tr><td>foo</td></tr>'
'</table></td>')
self.assertSoupEquals(
markup,
'<table id="1"><tr><td>Here\'s another table:'
'<table id="2"><tr><td>foo</td></tr></table>'
'</td></tr></table>')
self.assertSoupEquals(
"<table><thead><tr><td>Foo</td></tr></thead>"
"<tbody><tr><td>Bar</td></tr></tbody>"
"<tfoot><tr><td>Baz</td></tr></tfoot></table>")
def test_angle_brackets_in_attribute_values_are_escaped(self):
self.assertSoupEquals('<a b="<a>"></a>', '<a b="&lt;a&gt;"></a>')
def test_entities_in_attributes_converted_to_unicode(self):
expect = u'<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
self.assertSoupEquals('<p id="pi&#241;ata"></p>', expect)
self.assertSoupEquals('<p id="pi&#xf1;ata"></p>', expect)
self.assertSoupEquals('<p id="pi&ntilde;ata"></p>', expect)
def test_entities_in_text_converted_to_unicode(self):
expect = u'<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
self.assertSoupEquals("<p>pi&#241;ata</p>", expect)
self.assertSoupEquals("<p>pi&#xf1;ata</p>", expect)
self.assertSoupEquals("<p>pi&ntilde;ata</p>", expect)
def test_quot_entity_converted_to_quotation_mark(self):
self.assertSoupEquals("<p>I said &quot;good day!&quot;</p>",
'<p>I said "good day!"</p>')
def test_out_of_range_entity(self):
expect = u"\N{REPLACEMENT CHARACTER}"
self.assertSoupEquals("&#10000000000000;", expect)
self.assertSoupEquals("&#x10000000000000;", expect)
self.assertSoupEquals("&#1000000000;", expect)
def test_basic_namespaces(self):
"""Parsers don't need to *understand* namespaces, but at the
very least they should not choke on namespaces or lose
data."""
markup = b'<html xmlns="http://www.w3.org/1999/xhtml" xmlns:mathml="http://www.w3.org/1998/Math/MathML" xmlns:svg="http://www.w3.org/2000/svg"><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill="red"></b></body></html>'
soup = self.soup(markup)
self.assertEqual(markup, soup.encode())
html = soup.html
self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
self.assertEqual(
'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
self.assertEqual(
'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])
def test_multivalued_attribute_value_becomes_list(self):
markup = b'<a class="foo bar">'
soup = self.soup(markup)
self.assertEqual(['foo', 'bar'], soup.a['class'])
#
# Generally speaking, tests below this point are more tests of
# Beautiful Soup than tests of the tree builders. But parsers are
# weird, so we run these tests separately for every tree builder
# to detect any differences between them.
#
def test_soupstrainer(self):
"""Parsers should be able to work with SoupStrainers."""
strainer = SoupStrainer("b")
soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>",
parse_only=strainer)
self.assertEqual(soup.decode(), "<b>bold</b>")
def test_single_quote_attribute_values_become_double_quotes(self):
self.assertSoupEquals("<foo attr='bar'></foo>",
'<foo attr="bar"></foo>')
def test_attribute_values_with_nested_quotes_are_left_alone(self):
text = """<foo attr='bar "brawls" happen'>a</foo>"""
self.assertSoupEquals(text)
def test_attribute_values_with_double_nested_quotes_get_quoted(self):
text = """<foo attr='bar "brawls" happen'>a</foo>"""
soup = self.soup(text)
soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
self.assertSoupEquals(
soup.foo.decode(),
"""<foo attr="Brawls happen at &quot;Bob\'s Bar&quot;">a</foo>""")
def test_ampersand_in_attribute_value_gets_escaped(self):
self.assertSoupEquals('<this is="really messed up & stuff"></this>',
'<this is="really messed up &amp; stuff"></this>')
self.assertSoupEquals(
'<a href="http://example.org?a=1&b=2;3">foo</a>',
'<a href="http://example.org?a=1&amp;b=2;3">foo</a>')
def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
self.assertSoupEquals('<a href="http://example.org?a=1&amp;b=2;3"></a>')
def test_entities_in_strings_converted_during_parsing(self):
# Both XML and HTML entities are converted to Unicode characters
# during parsing.
text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
self.assertSoupEquals(text, expected)
def test_smart_quotes_converted_on_the_way_in(self):
# Microsoft smart quotes are converted to Unicode characters during
# parsing.
quote = b"<p>\x91Foo\x92</p>"
soup = self.soup(quote)
self.assertEqual(
soup.p.string,
u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
def test_non_breaking_spaces_converted_on_the_way_in(self):
soup = self.soup("<a>&nbsp;&nbsp;</a>")
self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
def test_entities_converted_on_the_way_out(self):
text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>".encode("utf-8")
soup = self.soup(text)
self.assertEqual(soup.p.encode("utf-8"), expected)
def test_real_iso_latin_document(self):
# Smoke test of interrelated functionality, using an
# easy-to-understand document.
# Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
# That's because we're going to encode it into ISO-Latin-1, and use
# that to test.
iso_latin_html = unicode_html.encode("iso-8859-1")
# Parse the ISO-Latin-1 HTML.
soup = self.soup(iso_latin_html)
# Encode it to UTF-8.
result = soup.encode("utf-8")
# What do we expect the result to look like? Well, it would
# look like unicode_html, except that the META tag would say
# UTF-8 instead of ISO-Latin-1.
expected = unicode_html.replace("ISO-Latin-1", "utf-8")
# And, of course, it would be in UTF-8, not Unicode.
expected = expected.encode("utf-8")
# Ta-da!
self.assertEqual(result, expected)
def test_real_shift_jis_document(self):
# Smoke test to make sure the parser can handle a document in
# Shift-JIS encoding, without choking.
shift_jis_html = (
b'<html><head></head><body><pre>'
b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
b'</pre></body></html>')
unicode_html = shift_jis_html.decode("shift-jis")
soup = self.soup(unicode_html)
# Make sure the parse tree is correctly encoded to various
# encodings.
self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
def test_real_hebrew_document(self):
# A real-world test to make sure we can convert ISO-8859-9 (a
# Hebrew encoding) to UTF-8.
hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
soup = self.soup(
hebrew_document, from_encoding="iso8859-8")
self.assertEqual(soup.original_encoding, 'iso8859-8')
self.assertEqual(
soup.encode('utf-8'),
hebrew_document.decode("iso8859-8").encode("utf-8"))
def test_meta_tag_reflects_current_encoding(self):
# Here's the <meta> tag saying that a document is
# encoded in Shift-JIS.
meta_tag = ('<meta content="text/html; charset=x-sjis" '
'http-equiv="Content-type"/>')
# Here's a document incorporating that meta tag.
shift_jis_html = (
'<html><head>\n%s\n'
'<meta http-equiv="Content-language" content="ja"/>'
'</head><body>Shift-JIS markup goes here.') % meta_tag
soup = self.soup(shift_jis_html)
# Parse the document, and the charset is seemingly unaffected.
parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
content = parsed_meta['content']
self.assertEqual('text/html; charset=x-sjis', content)
# But that value is actually a ContentMetaAttributeValue object.
self.assertTrue(isinstance(content, ContentMetaAttributeValue))
# And it will take on a value that reflects its current
# encoding.
self.assertEqual('text/html; charset=utf8', content.encode("utf8"))
# For the rest of the story, see TestSubstitutions in
# test_tree.py.
def test_html5_style_meta_tag_reflects_current_encoding(self):
# Here's the <meta> tag saying that a document is
# encoded in Shift-JIS.
meta_tag = ('<meta id="encoding" charset="x-sjis" />')
# Here's a document incorporating that meta tag.
shift_jis_html = (
'<html><head>\n%s\n'
'<meta http-equiv="Content-language" content="ja"/>'
'</head><body>Shift-JIS markup goes here.') % meta_tag
soup = self.soup(shift_jis_html)
# Parse the document, and the charset is seemingly unaffected.
parsed_meta = soup.find('meta', id="encoding")
charset = parsed_meta['charset']
self.assertEqual('x-sjis', charset)
# But that value is actually a CharsetMetaAttributeValue object.
self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))
# And it will take on a value that reflects its current
# encoding.
self.assertEqual('utf8', charset.encode("utf8"))
def test_tag_with_no_attributes_can_have_attributes_added(self):
data = self.soup("<a>text</a>")
data.a['foo'] = 'bar'
self.assertEqual('<a foo="bar">text</a>', data.a.decode())
class XMLTreeBuilderSmokeTest(object):
def test_docstring_generated(self):
soup = self.soup("<root/>")
self.assertEqual(
soup.encode(), b'<?xml version="1.0" encoding="utf-8"?>\n<root/>')
def test_real_xhtml_document(self):
"""A real XHTML document should come out *exactly* the same as it went in."""
markup = b"""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><title>Hello.</title></head>
<body>Goodbye.</body>
</html>"""
soup = self.soup(markup)
self.assertEqual(
soup.encode("utf-8"), markup)
def test_docstring_includes_correct_encoding(self):
soup = self.soup("<root/>")
self.assertEqual(
soup.encode("latin1"),
b'<?xml version="1.0" encoding="latin1"?>\n<root/>')
def test_large_xml_document(self):
"""A large XML document should come out the same as it went in."""
markup = (b'<?xml version="1.0" encoding="utf-8"?>\n<root>'
+ b'0' * (2**12)
+ b'</root>')
soup = self.soup(markup)
self.assertEqual(soup.encode("utf-8"), markup)
def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
self.assertSoupEquals("<p>", "<p/>")
self.assertSoupEquals("<p>foo</p>")
def test_namespaces_are_preserved(self):
markup = '<root xmlns:a="http://example.com/" xmlns:b="http://example.net/"><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root>'
soup = self.soup(markup)
root = soup.root
self.assertEqual("http://example.com/", root['xmlns:a'])
self.assertEqual("http://example.net/", root['xmlns:b'])
class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
"""Smoke test for a tree builder that supports HTML5."""
def test_real_xhtml_document(self):
# Since XHTML is not HTML5, HTML5 parsers are not tested to handle
# XHTML documents in any particular way.
pass
def test_html_tags_have_namespace(self):
markup = "<a>"
soup = self.soup(markup)
self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)
def test_svg_tags_have_namespace(self):
markup = '<svg><circle/></svg>'
soup = self.soup(markup)
namespace = "http://www.w3.org/2000/svg"
self.assertEqual(namespace, soup.svg.namespace)
self.assertEqual(namespace, soup.circle.namespace)
def test_mathml_tags_have_namespace(self):
markup = '<math><msqrt>5</msqrt></math>'
soup = self.soup(markup)
namespace = 'http://www.w3.org/1998/Math/MathML'
self.assertEqual(namespace, soup.math.namespace)
self.assertEqual(namespace, soup.msqrt.namespace)
def skipIf(condition, reason):
def nothing(test, *args, **kwargs):
return None
def decorator(test_item):
if condition:
return nothing
else:
return test_item
return decorator
libs/certifi/__init__.py
+1
View File
@@ -0,0 +1 @@
from .core import where
libs/certifi/cacert.pem
+3338
View File
File diff suppressed because it is too large Load Diff
libs/certifi/core.py
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
ceritfi.py
~~~~~~~~~~
This module returns the installation location of cacert.pem.
"""
import os
def where():
f = os.path.split(__file__)[0]
return os.path.join(f, 'cacert.pem')
if __name__ == '__main__':
print(where())
libs/guessit/ISO-3166-1_utf8.txt
Regular → Executable
View File
libs/guessit/ISO-639-2_utf-8.txt
Regular → Executable
View File
libs/guessit/__init__.py
Regular → Executable
+1 -1
View File
@@ -18,7 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
__version__ = '0.4' __version__ = '0.5-dev'
__all__ = ['Guess', 'Language', __all__ = ['Guess', 'Language',
'guess_file_info', 'guess_video_info', 'guess_file_info', 'guess_video_info',
'guess_movie_info', 'guess_episode_info'] 'guess_movie_info', 'guess_episode_info']
libs/guessit/country.py
Regular → Executable
+3 -1
View File
@@ -20,6 +20,7 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from guessit import fileutils from guessit import fileutils
from guessit.textutils import to_unicode
import logging import logging
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -66,7 +67,8 @@ class Country(object):
""" """
def __init__(self, country, strict=False): def __init__(self, country, strict=False):
self.alpha3 = country_to_alpha3.get(country.lower()) country = to_unicode(country.strip().lower())
self.alpha3 = country_to_alpha3.get(country)
if self.alpha3 is None and strict: if self.alpha3 is None and strict:
msg = 'The given string "%s" could not be identified as a country' msg = 'The given string "%s" could not be identified as a country'
libs/guessit/date.py
Regular → Executable
View File
libs/guessit/fileutils.py
Regular → Executable
+1 -6
View File
@@ -29,6 +29,7 @@ def split_path(path):
If the given path was an absolute path, the first element will always be: If the given path was an absolute path, the first element will always be:
- the '/' root folder on Unix systems - the '/' root folder on Unix systems
- the drive letter on Windows systems (eg: r'C:\') - the drive letter on Windows systems (eg: r'C:\')
- the mount point '\\' on Windows systems (eg: r'\\host\share')
>>> split_path('/usr/bin/smewt') >>> split_path('/usr/bin/smewt')
['/', 'usr', 'bin', 'smewt'] ['/', 'usr', 'bin', 'smewt']
@@ -36,12 +37,6 @@ def split_path(path):
>>> split_path('relative_path/to/my_folder/') >>> split_path('relative_path/to/my_folder/')
['relative_path', 'to', 'my_folder'] ['relative_path', 'to', 'my_folder']
>>> split_path(r'C:\Program Files\Smewt\smewt.exe')
['C:\\', 'Program Files', 'Smewt', 'smewt.exe']
>>> split_path(r'Documents and Settings\User\config\\')
['Documents and Settings', 'User', 'config']
""" """
result = [] result = []
while True: while True:
libs/guessit/guess.py
Regular → Executable
View File
libs/guessit/hash_ed2k.py
Regular → Executable
View File
libs/guessit/hash_mpc.py
Regular → Executable
View File

Some files were not shown because too many files have changed in this diff Show More