dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'refs/heads/develop'

Browse Source
This commit is contained in:
Ruud
2012-06-16 18:09:34 +02:00
parent 7367f7da79 79e842951e
commit 1c0b83eb75
187 changed files with 23312 additions and 3314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
couchpotato/__init__.py
+2 -2
View File
@@ -69,10 +69,10 @@ def getApiKey():
@app.errorhandler(404)
def page_not_found(error):
index_url = url_for('web.index')
url = getattr(request, 'path')[len(index_url):]
url = request.path[len(index_url):]
if url[:3] != 'api':
return redirect(index_url + '#' + url)
return redirect(request.url.replace(request.path, index_url + '#' + url))
else:
time.sleep(0.1)
return 'Wrong API key used', 404
couchpotato/core/_base/_core/main.py
+3 -3
View File
@@ -96,7 +96,7 @@ class Core(Plugin):
while loop:
log.debug('Asking who is running')
still_running = fireEvent('plugin.running', merge = True)
log.debug('Still running: %s' % still_running)
log.debug('Still running: %s', still_running)
if len(still_running) == 0:
break
@@ -105,7 +105,7 @@ class Core(Plugin):
running = list(set(still_running) - set(self.ignore_restart))
if len(running) > 0:
log.info('Waiting on plugins to finish: %s' % running)
log.info('Waiting on plugins to finish: %s', running)
else:
loop = False
@@ -118,7 +118,7 @@ class Core(Plugin):
except RuntimeError:
pass
except:
log.error('Failed shutting down the server: %s' % traceback.format_exc())
log.error('Failed shutting down the server: %s', traceback.format_exc())
fireEvent('app.after_shutdown', restart = restart)
couchpotato/core/_base/scheduler/main.py
+5 -5
View File
@@ -28,7 +28,7 @@ class Scheduler(Plugin):
for type in ['interval', 'cron']:
try:
self.sched.unschedule_job(getattr(self, type)[identifier]['job'])
log.debug('%s unscheduled %s' % (type.capitalize(), identifier))
log.debug('%s unscheduled %s', (type.capitalize(), identifier))
except:
pass
@@ -45,7 +45,7 @@ class Scheduler(Plugin):
job = self.sched.add_cron_job(cron['handle'], day = cron['day'], hour = cron['hour'], minute = cron['minute'])
cron['job'] = job
except ValueError, e:
log.error("Failed adding cronjob: %s" % e)
log.error('Failed adding cronjob: %s', e)
# Intervals
for identifier in self.intervals:
@@ -55,7 +55,7 @@ class Scheduler(Plugin):
job = self.sched.add_interval_job(interval['handle'], hours = interval['hours'], minutes = interval['minutes'], seconds = interval['seconds'])
interval['job'] = job
except ValueError, e:
log.error("Failed adding interval cronjob: %s" % e)
log.error('Failed adding interval cronjob: %s', e)
# Start it
log.debug('Starting scheduler')
@@ -75,7 +75,7 @@ class Scheduler(Plugin):
self.started = False
def cron(self, identifier = '', handle = None, day = '*', hour = '*', minute = '*'):
log.info('Scheduling "%s", cron: day = %s, hour = %s, minute = %s' % (identifier, day, hour, minute))
log.info('Scheduling "%s", cron: day = %s, hour = %s, minute = %s', (identifier, day, hour, minute))
self.remove(identifier)
self.crons[identifier] = {
@@ -86,7 +86,7 @@ class Scheduler(Plugin):
}
def interval(self, identifier = '', handle = None, hours = 0, minutes = 0, seconds = 0):
log.info('Scheduling %s, interval: hours = %s, minutes = %s, seconds = %s' % (identifier, hours, minutes, seconds))
log.info('Scheduling %s, interval: hours = %s, minutes = %s, seconds = %s', (identifier, hours, minutes, seconds))
self.remove(identifier)
self.intervals[identifier] = {
couchpotato/core/_base/updater/main.py
+19 -18
View File
@@ -1,5 +1,6 @@
from couchpotato.api import addApiView
from couchpotato.core.event import addEvent, fireEvent, fireEventAsync
from couchpotato.core.helpers.encoding import ss
from couchpotato.core.helpers.request import jsonified
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
@@ -120,7 +121,7 @@ class BaseUpdater(Plugin):
def deletePyc(self, only_excess = True):
for root, dirs, files in os.walk(Env.get('app_dir')):
for root, dirs, files in os.walk(ss(Env.get('app_dir'))):
pyc_files = filter(lambda filename: filename.endswith('.pyc'), files)
py_files = set(filter(lambda filename: filename.endswith('.py'), files))
@@ -128,11 +129,11 @@ class BaseUpdater(Plugin):
for excess_pyc_file in excess_pyc_files:
full_path = os.path.join(root, excess_pyc_file)
log.debug('Removing old PYC file: %s' % full_path)
log.debug('Removing old PYC file: %s', full_path)
try:
os.remove(full_path)
except:
log.error('Couldn\'t remove %s: %s' % (full_path, traceback.format_exc()))
log.error('Couldn\'t remove %s: %s', (full_path, traceback.format_exc()))
for dir_name in dirs:
full_path = os.path.join(root, dir_name)
@@ -140,7 +141,7 @@ class BaseUpdater(Plugin):
try:
os.rmdir(full_path)
except:
log.error('Couldn\'t remove empty directory %s: %s' % (full_path, traceback.format_exc()))
log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
@@ -170,7 +171,7 @@ class GitUpdater(BaseUpdater):
return True
except:
log.error('Failed updating via GIT: %s' % traceback.format_exc())
log.error('Failed updating via GIT: %s', traceback.format_exc())
self.update_failed = True
@@ -181,14 +182,14 @@ class GitUpdater(BaseUpdater):
if not self.version:
try:
output = self.repo.getHead() # Yes, please
log.debug('Git version output: %s' % output.hash)
log.debug('Git version output: %s', output.hash)
self.version = {
'hash': output.hash[:8],
'date': output.getDate(),
'type': 'git',
}
except Exception, e:
log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s' % e)
log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s', e)
return 'No GIT'
return self.version
@@ -198,7 +199,7 @@ class GitUpdater(BaseUpdater):
if self.update_version:
return
log.info('Checking for new version on github for %s' % self.repo_name)
log.info('Checking for new version on github for %s', self.repo_name)
if not Env.get('dev'):
self.repo.fetch()
@@ -210,7 +211,7 @@ class GitUpdater(BaseUpdater):
local = self.repo.getHead()
remote = branch.getHead()
log.info('Versions, local:%s, remote:%s' % (local.hash[:8], remote.hash[:8]))
log.info('Versions, local:%s, remote:%s', (local.hash[:8], remote.hash[:8]))
if local.getDate() < remote.getDate():
self.update_version = {
@@ -263,13 +264,13 @@ class SourceUpdater(BaseUpdater):
return True
except:
log.error('Failed updating: %s' % traceback.format_exc())
log.error('Failed updating: %s', traceback.format_exc())
self.update_failed = True
return False
def replaceWith(self, path):
app_dir = Env.get('app_dir')
app_dir = ss(Env.get('app_dir'))
# Get list of files we want to overwrite
self.deletePyc(only_excess = False)
@@ -296,7 +297,7 @@ class SourceUpdater(BaseUpdater):
except ValueError:
pass
except Exception, e:
log.error('Failed overwriting file: %s' % e)
log.error('Failed overwriting file: %s', e)
def removeDir(self, path):
@@ -315,11 +316,11 @@ class SourceUpdater(BaseUpdater):
output = json.loads(f.read())
f.close()
log.debug('Source version output: %s' % output)
log.debug('Source version output: %s', output)
self.version = output
self.version['type'] = 'source'
except Exception, e:
log.error('Failed using source updater. %s' % e)
log.error('Failed using source updater. %s', e)
return {}
return self.version
@@ -336,7 +337,7 @@ class SourceUpdater(BaseUpdater):
self.last_check = time.time()
except:
log.error('Failed updating via source: %s' % traceback.format_exc())
log.error('Failed updating via source: %s', traceback.format_exc())
return self.update_version is not None
@@ -351,7 +352,7 @@ class SourceUpdater(BaseUpdater):
'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())),
}
except:
log.error('Failed getting latest request from github: %s' % traceback.format_exc())
log.error('Failed getting latest request from github: %s', traceback.format_exc())
return {}
@@ -372,7 +373,7 @@ class DesktopUpdater(BaseUpdater):
if e['status'] == 'done':
fireEventAsync('app.restart')
else:
log.error('Failed updating desktop: %s' % e['exception'])
log.error('Failed updating desktop: %s', e['exception'])
self.update_failed = True
self.desktop._esky.auto_update(callback = do_restart)
@@ -403,7 +404,7 @@ class DesktopUpdater(BaseUpdater):
self.last_check = time.time()
except:
log.error('Failed updating desktop: %s' % traceback.format_exc())
log.error('Failed updating desktop: %s', traceback.format_exc())
return self.update_version is not None
couchpotato/core/downloaders/blackhole/main.py
+5 -5
View File
@@ -16,7 +16,7 @@ class Blackhole(Downloader):
directory = self.conf('directory')
if not directory or not os.path.isdir(directory):
log.error('No directory set for blackhole %s download.' % data.get('type'))
log.error('No directory set for blackhole %s download.', data.get('type'))
else:
try:
if not filedata or len(filedata) < 50:
@@ -27,19 +27,19 @@ class Blackhole(Downloader):
try:
if not os.path.isfile(fullPath):
log.info('Downloading %s to %s.' % (data.get('type'), fullPath))
log.info('Downloading %s to %s.', (data.get('type'), fullPath))
with open(fullPath, 'wb') as f:
f.write(filedata)
return True
else:
log.info('File %s already exists.' % fullPath)
log.info('File %s already exists.', fullPath)
return True
except:
log.error('Failed to download to blackhole %s' % traceback.format_exc())
log.error('Failed to download to blackhole %s', traceback.format_exc())
pass
except:
log.info('Failed to download file %s: %s' % (data.get('name'), traceback.format_exc()))
log.info('Failed to download file %s: %s', (data.get('name'), traceback.format_exc()))
return False
return False
couchpotato/core/downloaders/nzbget/main.py
+4 -4
View File
@@ -20,10 +20,10 @@ class NZBGet(Downloader):
return
if not filedata:
log.error('Unable to get NZB file: %s' % traceback.format_exc())
log.error('Unable to get NZB file: %s', traceback.format_exc())
return False
log.info('Sending "%s" to NZBGet.' % data.get('name'))
log.info('Sending "%s" to NZBGet.', data.get('name'))
url = self.url % {'host': self.conf('host'), 'password': self.conf('password')}
nzb_name = '%s.nzb' % self.createNzbName(data, movie)
@@ -41,12 +41,12 @@ class NZBGet(Downloader):
if e.errcode == 401:
log.error('Password is incorrect.')
else:
log.error('Protocol Error: %s' % e)
log.error('Protocol Error: %s', e)
return False
if rpc.append(nzb_name, self.conf('category'), False, standard_b64encode(filedata.strip())):
log.info('NZB sent successfully to NZBGet')
return True
else:
log.error('NZBGet could not add %s to the queue.' % nzb_name)
log.error('NZBGet could not add %s to the queue.', nzb_name)
return False
couchpotato/core/downloaders/sabnzbd/__init__.py
-6
View File
@@ -33,12 +33,6 @@ config = [{
'label': 'Category',
'description': 'The category CP places the nzb in. Like <strong>movies</strong> or <strong>couchpotato</strong>',
},
{
'advanced': True,
'name': 'pp_directory',
'type': 'directory',
'description': 'Your Post-Processing Script directory, set in Sabnzbd > Config > Directories.',
},
{
'name': 'manual',
'default': 0,
couchpotato/core/downloaders/sabnzbd/main.py
+1 -67
View File
@@ -2,11 +2,6 @@ from couchpotato.core.downloaders.base import Downloader
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import cleanHost
from couchpotato.core.logger import CPLog
from inspect import ismethod, isfunction
from tempfile import mkstemp
import base64
import os
import re
import traceback
log = CPLog(__name__)
@@ -20,20 +15,7 @@ class Sabnzbd(Downloader):
if self.isDisabled(manual) or not self.isCorrectType(data.get('type')):
return
log.info("Sending '%s' to SABnzbd." % data.get('name'))
if self.conf('ppDir') and data.get('imdb_id'):
try:
pp_script_fn = self.buildPp(data.get('imdb_id'))
except:
log.info("Failed to create post-processing script.")
pp_script_fn = False
if not pp_script_fn:
pp = False
else:
pp = True
else:
pp = False
log.info('Sending "%s" to SABnzbd.', data.get('name'))
params = {
'apikey': self.conf('api_key'),
@@ -53,9 +35,6 @@ class Sabnzbd(Downloader):
else:
params['name'] = data.get('url')
if pp:
params['script'] = pp_script_fn
url = cleanHost(self.conf('host')) + "api?" + tryUrlencode(params)
try:
@@ -82,48 +61,3 @@ class Sabnzbd(Downloader):
else:
log.error("Unknown error: " + result[:40])
return False
def buildPp(self, imdb_id):
pp_script_path = self.getPpFile()
scriptB64 = '''IyEvdXNyL2Jpbi9weXRob24KaW1wb3J0IG9zCmltcG9ydCBzeXMKcHJpbnQgIkNyZWF0aW5nIGNwLmNw
bmZvIGZvciAlcyIgJSBzeXMuYXJndlsxXQppbWRiSWQgPSB7W0lNREJJREhFUkVdfQpwYXRoID0gb3Mu
cGF0aC5qb2luKHN5cy5hcmd2WzFdLCAiY3AuY3BuZm8iKQp0cnk6CiBmID0gb3BlbihwYXRoLCAndycp
CmV4Y2VwdCBJT0Vycm9yOgogcHJpbnQgIlVuYWJsZSB0byBvcGVuICVzIGZvciB3cml0aW5nIiAlIHBh
dGgKIHN5cy5leGl0KDEpCnRyeToKIGYud3JpdGUob3MucGF0aC5iYXNlbmFtZShzeXMuYXJndlswXSkr
IlxuIitpbWRiSWQpCmV4Y2VwdDoKIHByaW50ICJVbmFibGUgdG8gd3JpdGUgdG8gZmlsZTogJXMiICUg
cGF0aAogc3lzLmV4aXQoMikKZi5jbG9zZSgpCnByaW50ICJXcm90ZSBpbWRiIGlkLCAlcywgdG8gZmls
ZTogJXMiICUgKGltZGJJZCwgcGF0aCkK'''
script = re.sub(r"\{\[IMDBIDHERE\]\}", "'%s'" % imdb_id, base64.b64decode(scriptB64))
try:
f = open(pp_script_path, 'wb')
except:
log.info("Unable to open post-processing script for writing. Check permissions: %s" % pp_script_path)
return False
try:
f.write(script)
f.close()
except:
log.info("Unable to write to post-processing script. Check permissions: %s" % pp_script_path)
return False
log.info("Wrote post-processing script to: %s" % pp_script_path)
return os.path.basename(pp_script_path)
def getPpFile(self):
pp_script_handle, pp_script_path = mkstemp(suffix = '.py', dir = self.conf('ppDir'))
pp_sh = os.fdopen(pp_script_handle)
pp_sh.close()
try:
os.chmod(pp_script_path, int('777', 8))
except:
log.info("Unable to set post-processing script permissions to 777 (may still work correctly): %s" % pp_script_path)
return pp_script_path
couchpotato/core/downloaders/transmission/main.py
+3 -3
View File
@@ -16,7 +16,7 @@ class Transmission(Downloader):
if self.isDisabled(manual) or not self.isCorrectType(data.get('type')):
return
log.info('Sending "%s" to Transmission.' % data.get('name'))
log.info('Sending "%s" to Transmission.', data.get('name'))
# Load host from config and split out port.
host = self.conf('host').split(':')
@@ -42,10 +42,10 @@ class Transmission(Downloader):
torrent.seed_ratio_limit = self.conf('ratio')
torrent.seed_ratio_mode = 'single' if self.conf('ratio') else 'global'
except transmissionrpc.TransmissionError, e:
log.error('Failed to change settings for transfer in transmission: %s' % e)
log.error('Failed to change settings for transfer in transmission: %s', e)
return True
except transmissionrpc.TransmissionError, e:
log.error('Failed to send link to transmission: %s' % e)
log.error('Failed to send link to transmission: %s', e)
return False
couchpotato/core/event.py
+6 -6
View File
@@ -12,7 +12,7 @@ def runHandler(name, handler, *args, **kwargs):
return handler(*args, **kwargs)
except:
from couchpotato.environment import Env
log.error('Error in event "%s", that wasn\'t caught: %s%s' % (name, traceback.format_exc(), Env.all()))
log.error('Error in event "%s", that wasn\'t caught: %s%s', (name, traceback.format_exc(), Env.all()))
def addEvent(name, handler, priority = 100):
@@ -43,7 +43,7 @@ def removeEvent(name, handler):
def fireEvent(name, *args, **kwargs):
if not events.get(name): return
#log.debug('Firing event %s' % name)
#log.debug('Firing event %s', name)
try:
# Fire after event
@@ -100,7 +100,7 @@ def fireEvent(name, *args, **kwargs):
elif r[1]:
errorHandler(r[1])
else:
log.debug('Assume disabled eventhandler for: %s' % name)
log.debug('Assume disabled eventhandler for: %s', name)
else:
results = []
@@ -130,7 +130,7 @@ def fireEvent(name, *args, **kwargs):
modified_results = fireEvent('result.modify.%s' % name, results, single = True)
if modified_results:
log.debug('Return modified results for %s' % name)
log.debug('Return modified results for %s', name)
results = modified_results
if not is_after_event:
@@ -143,7 +143,7 @@ def fireEvent(name, *args, **kwargs):
except KeyError, e:
pass
except Exception:
log.error('%s: %s' % (name, traceback.format_exc()))
log.error('%s: %s', (name, traceback.format_exc()))
def fireEventAsync(*args, **kwargs):
try:
@@ -152,7 +152,7 @@ def fireEventAsync(*args, **kwargs):
my_thread.start()
return True
except Exception, e:
log.error('%s: %s' % (args[0], e))
log.error('%s: %s', (args[0], e))
def errorHandler(error):
etype, value, tb = error
couchpotato/core/helpers/encoding.py
+5 -1
View File
@@ -31,10 +31,14 @@ def toUnicode(original, *args):
except:
raise
except UnicodeDecodeError:
log.error('Unable to decode value: %s... ' % repr(original)[:20])
log.error('Unable to decode value: %s... ', repr(original)[:20])
ascii_text = str(original).encode('string_escape')
return toUnicode(ascii_text)
def ss(original, *args):
from couchpotato.environment import Env
return toUnicode(original, *args).encode(Env.get('encoding'))
def ek(original, *args):
if isinstance(original, (str, unicode)):
try:
couchpotato/core/helpers/rss.py
+1 -1
View File
@@ -47,5 +47,5 @@ class RSS(object):
try:
return XMLTree.parse(data).findall(path)
except Exception, e:
log.error('Error parsing RSS. %s' % e)
log.error('Error parsing RSS. %s', e)
return []
couchpotato/core/helpers/variable.py
+2 -2
View File
@@ -118,10 +118,10 @@ def getTitle(library_dict):
try:
return library_dict['titles'][0]['title']
except:
log.error('Could not get title for %s' % library_dict['identifier'])
log.error('Could not get title for %s', library_dict['identifier'])
return None
except:
log.error('Could not get title for library item: %s' % library_dict)
log.error('Could not get title for library item: %s', library_dict)
return None
def randomString(size = 8, chars = string.ascii_uppercase + string.digits):
couchpotato/core/loader.py
+6 -6
View File
@@ -45,7 +45,7 @@ class Loader(object):
try:
m = getattr(self.loadModule(module_name), plugin.get('name'))
log.info("Loading %s: %s" % (plugin['type'], plugin['name']))
log.info('Loading %s: %s', (plugin['type'], plugin['name']))
# Save default settings for plugin/provider
did_save += self.loadSettings(m, module_name, save = False)
@@ -57,10 +57,10 @@ class Loader(object):
log.error(e.message)
pass
# todo:: this needs to be more descriptive.
log.error('Import error, remove the empty folder: %s' % plugin.get('module'))
log.debug('Can\'t import %s: %s' % (module_name, traceback.format_exc()))
log.error('Import error, remove the empty folder: %s', plugin.get('module'))
log.debug('Can\'t import %s: %s', (module_name, traceback.format_exc()))
except:
log.error('Can\'t import %s: %s' % (module_name, traceback.format_exc()))
log.error('Can\'t import %s: %s', (module_name, traceback.format_exc()))
if did_save:
fireEvent('settings.save')
@@ -84,7 +84,7 @@ class Loader(object):
fireEvent('settings.register', section_name = section['name'], options = options, save = save)
return True
except:
log.debug("Failed loading settings for '%s': %s" % (name, traceback.format_exc()))
log.debug('Failed loading settings for "%s": %s', (name, traceback.format_exc()))
return False
def loadPlugins(self, module, name):
@@ -97,7 +97,7 @@ class Loader(object):
return True
except Exception, e:
log.error("Failed loading plugin '%s': %s" % (module.__file__, traceback.format_exc()))
log.error('Failed loading plugin "%s": %s', (module.__file__, traceback.format_exc()))
return False
def addModule(self, priority, plugin_type, module, name):
couchpotato/core/logger.py
+29 -17
View File
@@ -1,5 +1,6 @@
import logging
import re
import traceback
class CPLog(object):
@@ -13,31 +14,42 @@ class CPLog(object):
self.context = context
self.logger = logging.getLogger()
def info(self, msg):
self.logger.info(self.addContext(msg))
def info(self, msg, replace_tuple = ()):
self.logger.info(self.addContext(msg, replace_tuple))
def debug(self, msg):
self.logger.debug(self.addContext(msg))
def debug(self, msg, replace_tuple = ()):
self.logger.debug(self.addContext(msg, replace_tuple))
def error(self, msg):
self.logger.error(self.addContext(msg))
def error(self, msg, replace_tuple = ()):
self.logger.error(self.addContext(msg, replace_tuple))
def warning(self, msg):
self.logger.warning(self.addContext(msg))
def warning(self, msg, replace_tuple = ()):
self.logger.warning(self.addContext(msg, replace_tuple))
def critical(self, msg):
self.logger.critical(self.addContext(msg), exc_info = 1)
def critical(self, msg, replace_tuple = ()):
self.logger.critical(self.addContext(msg, replace_tuple), exc_info = 1)
def addContext(self, msg):
return '[%+25.25s] %s' % (self.context[-25:], self.removePrivateData(msg))
def addContext(self, msg, replace_tuple = ()):
return '[%+25.25s] %s' % (self.context[-25:], self.safeMessage(msg, replace_tuple))
def removePrivateData(self, msg):
try:
msg = unicode(msg)
except:
pass
def safeMessage(self, msg, replace_tuple = ()):
from couchpotato.environment import Env
from couchpotato.core.helpers.encoding import ss
msg = ss(msg)
try:
msg = msg % replace_tuple
except:
try:
if isinstance(replace_tuple, tuple):
msg = msg % tuple([ss(x) for x in list(replace_tuple)])
else:
msg = msg % ss(replace_tuple)
except:
self.error('Failed encoding stuff to log: %s' % traceback.format_exc())
if not Env.get('dev'):
for replace in self.replace_private:
couchpotato/core/notifications/base.py
+1 -1
View File
@@ -41,7 +41,7 @@ class Notification(Plugin):
test_type = self.testNotifyName()
log.info('Sending test to %s' % test_type)
log.info('Sending test to %s', test_type)
success = self.notify(
message = self.test_message,
couchpotato/core/notifications/growl/main.py
+1 -1
View File
@@ -38,7 +38,7 @@ class Growl(Notification):
self.growl.register()
self.registered = True
except:
log.error('Failed register of growl: %s' % traceback.format_exc())
log.error('Failed register of growl: %s', traceback.format_exc())
def notify(self, message = '', data = {}, listener = None):
if self.isDisabled(): return
couchpotato/core/notifications/nmj/main.py
+8 -8
View File
@@ -33,10 +33,10 @@ class NMJ(Notification):
try:
terminal = telnetlib.Telnet(host)
except Exception:
log.error('Warning: unable to get a telnet session to %s' % (host))
log.error('Warning: unable to get a telnet session to %s', (host))
return self.failed()
log.debug('Connected to %s via telnet' % (host))
log.debug('Connected to %s via telnet', (host))
terminal.read_until('sh-3.00# ')
terminal.write('cat /tmp/source\n')
terminal.write('cat /tmp/netshare\n')
@@ -48,9 +48,9 @@ class NMJ(Notification):
if match:
database = match.group(1)
device = match.group(2)
log.info('Found NMJ database %s on device %s' % (database, device))
log.info('Found NMJ database %s on device %s', (database, device))
else:
log.error('Could not get current NMJ database on %s, NMJ is probably not running!' % (host))
log.error('Could not get current NMJ database on %s, NMJ is probably not running!', (host))
return self.failed()
if device.startswith('NETWORK_SHARE/'):
@@ -58,7 +58,7 @@ class NMJ(Notification):
if match:
mount = match.group().replace('127.0.0.1', host)
log.info('Found mounting url on the Popcorn Hour in configuration: %s' % (mount))
log.info('Found mounting url on the Popcorn Hour in configuration: %s', (mount))
else:
log.error('Detected a network share on the Popcorn Hour, but could not get the mounting url')
return self.failed()
@@ -77,7 +77,7 @@ class NMJ(Notification):
database = self.conf('database')
if self.mount:
log.debug('Try to mount network drive via url: %s' % (mount))
log.debug('Try to mount network drive via url: %s', (mount))
try:
data = self.urlopen(mount)
except:
@@ -102,11 +102,11 @@ class NMJ(Notification):
et = etree.fromstring(response)
result = et.findtext('returnValue')
except SyntaxError, e:
log.error('Unable to parse XML returned from the Popcorn Hour: %s' % (e))
log.error('Unable to parse XML returned from the Popcorn Hour: %s', (e))
return False
if int(result) > 0:
log.error('Popcorn Hour returned an errorcode: %s' % (result))
log.error('Popcorn Hour returned an errorcode: %s', (result))
return False
else:
log.info('NMJ started background scan')
couchpotato/core/notifications/notifo/main.py
+1 -1
View File
@@ -32,7 +32,7 @@ class Notifo(Notification):
raise Exception
except:
log.error('Notification failed: %s' % traceback.format_exc())
log.error('Notification failed: %s', traceback.format_exc())
return False
log.info('Notifo notification successful.')
couchpotato/core/notifications/notifymyandroid/main.py
+1 -1
View File
@@ -23,6 +23,6 @@ class NotifyMyAndroid(Notification):
for key in keys:
if not response[str(key)]['code'] == u'200':
log.error('Could not send notification to NotifyMyAndroid (%s). %s' % (key, response[key]['message']))
log.error('Could not send notification to NotifyMyAndroid (%s). %s', (key, response[key]['message']))
return response
couchpotato/core/notifications/notifymywp/main.py
+1 -1
View File
@@ -17,7 +17,7 @@ class NotifyMyWP(Notification):
for key in keys:
if not response[key]['Code'] == u'200':
log.error('Could not send notification to NotifyMyWindowsPhone (%s). %s' % (key, response[key]['message']))
log.error('Could not send notification to NotifyMyWindowsPhone (%s). %s', (key, response[key]['message']))
return False
return response
couchpotato/core/notifications/plex/__init__.py
+3 -2
View File
@@ -17,8 +17,9 @@ config = [{
},
{
'name': 'host',
'default': 'localhost:32400',
'description': 'Default should be on localhost:32400',
'default': 'localhost',
'description': 'Default should be on localhost',
'advanced': True,
},
],
}
couchpotato/core/notifications/plex/main.py
+27 -2
View File
@@ -1,4 +1,5 @@
from couchpotato.core.event import addEvent
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import cleanHost
from couchpotato.core.logger import CPLog
from couchpotato.core.notifications.base import Notification
@@ -11,13 +12,14 @@ log = CPLog(__name__)
class Plex(Notification):
def __init__(self):
super(Plex, self).__init__()
addEvent('renamer.after', self.addToLibrary)
def addToLibrary(self, group = {}):
if self.isDisabled(): return
log.info('Sending notification to Plex')
hosts = [cleanHost(x.strip()) for x in self.conf('host').split(",")]
hosts = [cleanHost(x.strip() + ':32400') for x in self.conf('host').split(",")]
for host in hosts:
@@ -36,7 +38,30 @@ class Plex(Notification):
x = self.urlopen(url)
except:
log.error('Plex library update failed for %s: %s' % (host, traceback.format_exc()))
log.error('Plex library update failed for %s: %s', (host, traceback.format_exc()))
return False
return True
def notify(self, message = '', data = {}, listener = None):
if self.isDisabled(): return
for host in [x.strip() + ':3000' for x in self.conf('host').split(",")]:
self.send({'command': 'ExecBuiltIn', 'parameter': 'Notification(CouchPotato, %s)' % message}, host)
return True
def send(self, command, host):
url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, tryUrlencode(command))
headers = {}
try:
self.urlopen(url, headers = headers, show_error = False)
except:
log.error("Couldn't sent command to Plex")
return False
log.info('Plex notification to %s successful.', host)
return True
couchpotato/core/notifications/prowl/main.py
+1 -1
View File
@@ -32,7 +32,7 @@ class Prowl(Notification):
log.info('Prowl notifications sent.')
return True
elif request_status == 401:
log.error('Prowl auth failed: %s' % response.reason)
log.error('Prowl auth failed: %s', response.reason)
return False
else:
log.error('Prowl notification failed.')
couchpotato/core/notifications/pushover/main.py
+1 -1
View File
@@ -35,7 +35,7 @@ class Pushover(Notification):
log.info('Pushover notifications sent.')
return True
elif request_status == 401:
log.error('Pushover auth failed: %s' % response.reason)
log.error('Pushover auth failed: %s', response.reason)
return False
else:
log.error('Pushover notification failed.')
couchpotato/core/notifications/synoindex/main.py
+3 -3
View File
@@ -15,14 +15,14 @@ class Synoindex(Notification):
if self.isDisabled(): return
command = ['/usr/syno/bin/synoindex', '-A', group.get('destination_dir')]
log.info(u'Executing synoindex command: %s ' % command)
log.info(u'Executing synoindex command: %s ', command)
try:
p = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
out = p.communicate()
log.info('Result from synoindex: %s' % str(out))
log.info('Result from synoindex: %s', str(out))
return True
except OSError, e:
log.error('Unable to run synoindex: %s' % e)
log.error('Unable to run synoindex: %s', e)
return False
return True
couchpotato/core/notifications/twitter/main.py
+5 -5
View File
@@ -55,7 +55,7 @@ class Twitter(Notification):
else:
api.PostUpdate('[%s] %s' % (self.default_title, message))
except Exception, e:
log.error('Error sending tweet: %s' % e)
log.error('Error sending tweet: %s', e)
return False
return True
@@ -71,7 +71,7 @@ class Twitter(Notification):
resp, content = oauth_client.request(self.urls['request'], 'POST', body = tryUrlencode({'oauth_callback': callback_url}))
if resp['status'] != '200':
log.error('Invalid response from Twitter requesting temp token: %s' % resp['status'])
log.error('Invalid response from Twitter requesting temp token: %s', resp['status'])
return jsonified({
'success': False,
})
@@ -80,7 +80,7 @@ class Twitter(Notification):
auth_url = self.urls['authorize'] + ("?oauth_token=%s" % self.request_token['oauth_token'])
log.info('Redirecting to "%s"' % auth_url)
log.info('Redirecting to "%s"', auth_url)
return jsonified({
'success': True,
'url': auth_url,
@@ -100,10 +100,10 @@ class Twitter(Notification):
access_token = dict(parse_qsl(content))
if resp['status'] != '200':
log.error('The request for an access token did not succeed: %s' % resp['status'])
log.error('The request for an access token did not succeed: %s', resp['status'])
return 'Twitter auth failed'
else:
log.debug('Tokens: %s, %s' % (access_token['oauth_token'], access_token['oauth_token_secret']))
log.debug('Tokens: %s, %s', (access_token['oauth_token'], access_token['oauth_token_secret']))
self.conf('access_token_key', value = access_token['oauth_token'])
self.conf('access_token_secret', value = access_token['oauth_token_secret'])
couchpotato/core/notifications/xbmc/main.py
+1 -1
View File
@@ -35,5 +35,5 @@ class XBMC(Notification):
log.error("Couldn't sent command to XBMC")
return False
log.info('XBMC notification to %s successful.' % host)
log.info('XBMC notification to %s successful.', host)
return True
couchpotato/core/plugins/base.py
+15 -12
View File
@@ -1,6 +1,6 @@
from couchpotato import addView
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import tryUrlencode, simplifyString
from couchpotato.core.helpers.encoding import tryUrlencode, simplifyString, ss
from couchpotato.core.helpers.variable import getExt
from couchpotato.core.logger import CPLog
from couchpotato.environment import Env
@@ -71,6 +71,7 @@ class Plugin(object):
return send_from_directory(d, filename)
def createFile(self, path, content, binary = False):
path = ss(path)
self.makeDir(os.path.dirname(path))
@@ -80,15 +81,16 @@ class Plugin(object):
f.close()
os.chmod(path, Env.getPermission('file'))
except Exception, e:
log.error('Unable writing to file "%s": %s' % (path, e))
log.error('Unable writing to file "%s": %s', (path, e))
def makeDir(self, path):
path = ss(path)
try:
if not os.path.isdir(path):
os.makedirs(path, Env.getPermission('folder'))
return True
except Exception, e:
log.error('Unable to create folder "%s": %s' % (path, e))
log.error('Unable to create folder "%s": %s', (path, e))
return False
@@ -106,7 +108,7 @@ class Plugin(object):
# Don't try for failed requests
if self.http_failed_disabled.get(host, 0) > 0:
if self.http_failed_disabled[host] > (time.time() - 900):
log.info('Disabled calls to %s for 15 minutes because so many failed requests.' % host)
log.info('Disabled calls to %s for 15 minutes because so many failed requests.', host)
raise Exception
else:
del self.http_failed_request[host]
@@ -116,7 +118,7 @@ class Plugin(object):
try:
if multipart:
log.info('Opening multipart url: %s, params: %s' % (url, [x for x in params.iterkeys()]))
log.info('Opening multipart url: %s, params: %s', (url, [x for x in params.iterkeys()]))
request = urllib2.Request(url, params, headers)
cookies = cookielib.CookieJar()
@@ -124,7 +126,7 @@ class Plugin(object):
data = opener.open(request, timeout = timeout).read()
else:
log.info('Opening url: %s, params: %s' % (url, [x for x in params.iterkeys()]))
log.info('Opening url: %s, params: %s', (url, [x for x in params.iterkeys()]))
data = tryUrlencode(params) if len(params) > 0 else None
request = urllib2.Request(url, data, headers)
@@ -133,7 +135,7 @@ class Plugin(object):
self.http_failed_request[host] = 0
except IOError:
if show_error:
log.error('Failed opening url in %s: %s %s' % (self.getName(), url, traceback.format_exc(1)))
log.error('Failed opening url in %s: %s %s', (self.getName(), url, traceback.format_exc(1)))
# Save failed requests by hosts
try:
@@ -147,7 +149,7 @@ class Plugin(object):
self.http_failed_disabled[host] = time.time()
except:
log.debug('Failed logging failed requests for %s: %s' % (url, traceback.format_exc()))
log.debug('Failed logging failed requests for %s: %s', (url, traceback.format_exc()))
raise
@@ -163,7 +165,7 @@ class Plugin(object):
wait = math.ceil(last_use - now + self.http_time_between_calls)
if wait > 0:
log.debug('Waiting for %s, %d seconds' % (self.getName(), wait))
log.debug('Waiting for %s, %d seconds', (self.getName(), wait))
time.sleep(last_use - now + self.http_time_between_calls)
def beforeCall(self, handler):
@@ -202,7 +204,7 @@ class Plugin(object):
cache_key = simplifyString(cache_key)
cache = Env.get('cache').get(cache_key)
if cache:
if not Env.get('dev'): log.debug('Getting cache %s' % cache_key)
if not Env.get('dev'): log.debug('Getting cache %s', cache_key)
return cache
if url:
@@ -214,13 +216,14 @@ class Plugin(object):
del kwargs['cache_timeout']
data = self.urlopen(url, **kwargs)
self.setCache(cache_key, data, timeout = cache_timeout)
if data:
self.setCache(cache_key, data, timeout = cache_timeout)
return data
except:
pass
def setCache(self, cache_key, value, timeout = 300):
log.debug('Setting cache %s' % cache_key)
log.debug('Setting cache %s', cache_key)
Env.get('cache').set(cache_key, value, timeout)
return value
couchpotato/core/plugins/file/main.py
+2
View File
@@ -8,6 +8,7 @@ from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import FileType, File
from couchpotato.environment import Env
import os.path
import traceback
log = CPLog(__name__)
@@ -46,6 +47,7 @@ class FileManager(Plugin):
try:
filedata = self.urlopen(url, **urlopen_kwargs)
except:
log.error('Failed downloading file %s: %s', (url, traceback.format_exc()))
return False
self.createFile(dest, filedata, binary = True)
couchpotato/core/plugins/library/main.py
+10 -9
View File
@@ -78,7 +78,7 @@ class LibraryPlugin(Plugin):
except: pass
if not info or len(info) == 0:
log.error('Could not update, no movie info to work with: %s' % identifier)
log.error('Could not update, no movie info to work with: %s', identifier)
return False
# Main info
@@ -95,7 +95,7 @@ class LibraryPlugin(Plugin):
db.commit()
titles = info.get('titles', [])
log.debug('Adding titles: %s' % titles)
log.debug('Adding titles: %s', titles)
for title in titles:
if not title:
continue
@@ -117,13 +117,14 @@ class LibraryPlugin(Plugin):
continue
file_path = fireEvent('file.download', url = image, single = True)
file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', type), single = True)
try:
file_obj = db.query(File).filter_by(id = file_obj.get('id')).one()
library.files.append(file_obj)
db.commit()
except:
log.debug('Failed to attach to library: %s' % traceback.format_exc())
if file_path:
file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', type), single = True)
try:
file_obj = db.query(File).filter_by(id = file_obj.get('id')).one()
library.files.append(file_obj)
db.commit()
except:
log.debug('Failed to attach to library: %s', traceback.format_exc())
library_dict = library.to_dict(self.default_dict)
couchpotato/core/plugins/log/main.py
+5 -4
View File
@@ -1,5 +1,7 @@
from couchpotato.api import addApiView
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.helpers.request import jsonified, getParam, getParams
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
@@ -78,7 +80,7 @@ class Logging(Plugin):
def partial(self):
log_type = getParam('type', 'all')
total_lines = getParam('lines', 30)
total_lines = tryInt(getParam('lines', 30))
log_lines = []
@@ -92,13 +94,12 @@ class Logging(Plugin):
reversed_lines = []
f = open(path, 'r')
reversed_lines = f.read().split('[0m\n')
reversed_lines = toUnicode(f.read()).split('[0m\n')
reversed_lines.reverse()
brk = False
for line in reversed_lines:
#print '%s ' % log_type in line.lower()
if log_type == 'all' or '%s ' % log_type.upper() in line:
log_lines.append(line)
@@ -149,7 +150,7 @@ class Logging(Plugin):
except:
log.error(log_message)
except:
log.error('Couldn\'t log via API: %s' % params)
log.error('Couldn\'t log via API: %s', params)
return jsonified({
couchpotato/core/plugins/manage/main.py
+11 -10
View File
@@ -1,6 +1,6 @@
from couchpotato.api import addApiView
from couchpotato.core.event import fireEvent, addEvent, fireEventAsync
from couchpotato.core.helpers.request import jsonified, getParams
from couchpotato.core.helpers.request import jsonified, getParam
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
@@ -25,13 +25,14 @@ class Manage(Plugin):
})
if not Env.get('dev'):
addEvent('app.load', self.updateLibrary)
def updateLibrary():
self.updateLibrary(full = False)
addEvent('app.load', updateLibrary)
def updateLibraryView(self):
params = getParams()
fireEventAsync('manage.update', full = params.get('full', True))
full = getParam('full', default = 1)
fireEventAsync('manage.update', full = True if full == '1' else False)
return jsonified({
'success': True
@@ -51,11 +52,11 @@ class Manage(Plugin):
if not os.path.isdir(directory):
if len(directory) > 0:
log.error('Directory doesn\'t exist: %s' % directory)
log.error('Directory doesn\'t exist: %s', directory)
continue
log.info('Updating manage library: %s' % directory)
identifiers = fireEvent('scanner.folder', folder = directory, newer_than = last_update, single = True)
log.info('Updating manage library: %s', directory)
identifiers = fireEvent('scanner.folder', folder = directory, newer_than = last_update if not full else 0, single = True)
if identifiers:
added_identifiers.extend(identifiers)
@@ -67,11 +68,11 @@ class Manage(Plugin):
if self.conf('cleanup') and full and not self.shuttingDown():
# Get movies with done status
done_movies = fireEvent('movie.list', status = 'done', single = True)
total_movies, done_movies = fireEvent('movie.list', status = 'done', single = True)
for done_movie in done_movies:
if done_movie['library']['identifier'] not in added_identifiers:
fireEvent('movie.delete', movie_id = done_movie['id'])
fireEvent('movie.delete', movie_id = done_movie['id'], delete_from = 'all')
Env.prop('manage.last_update', time.time())
couchpotato/core/plugins/movie/main.py
+14 -8
View File
@@ -130,6 +130,8 @@ class MoviePlugin(Plugin):
.filter(or_(*[Movie.status.has(identifier = s) for s in status])) \
.group_by(Movie.id)
total_count = q.count()
filter_or = []
if starts_with:
starts_with = toUnicode(starts_with.lower())
@@ -156,8 +158,7 @@ class MoviePlugin(Plugin):
.options(joinedload_all('library.titles')) \
.options(joinedload_all('library.files')) \
.options(joinedload_all('status')) \
.options(joinedload_all('files')) \
.options(joinedload_all('files'))
if limit_offset:
splt = [x.strip() for x in limit_offset.split(',')]
@@ -165,7 +166,6 @@ class MoviePlugin(Plugin):
offset = 0 if len(splt) is 1 else splt[1]
q2 = q2.limit(limit).offset(offset)
results = q2.all()
movies = []
for movie in results:
@@ -178,7 +178,7 @@ class MoviePlugin(Plugin):
movies.append(temp)
#db.close()
return movies
return (total_count, movies)
def availableChars(self, status = ['active']):
@@ -214,11 +214,12 @@ class MoviePlugin(Plugin):
starts_with = params.get('starts_with', None)
search = params.get('search', None)
movies = self.list(status = status, limit_offset = limit_offset, starts_with = starts_with, search = search)
total_movies, movies = self.list(status = status, limit_offset = limit_offset, starts_with = starts_with, search = search)
return jsonified({
'success': True,
'empty': len(movies) == 0,
'total': total_movies,
'movies': movies,
})
@@ -279,6 +280,10 @@ class MoviePlugin(Plugin):
def add(self, params = {}, force_readd = True, search_after = True):
if not params.get('identifier'):
log.error('Can\'t add movie without imdb identifier.')
return False
library = fireEvent('library.add', single = True, attrs = params, update_after = False)
# Status
@@ -314,7 +319,7 @@ class MoviePlugin(Plugin):
m.profile_id = params.get('profile_id', default_profile.get('id'))
else:
log.debug('Movie already exists, not updating: %s' % params)
log.debug('Movie already exists, not updating: %s', params)
added = False
if force_readd:
@@ -351,7 +356,7 @@ class MoviePlugin(Plugin):
return jsonified({
'success': True,
'added': True,
'added': True if movie_dict else False,
'movie': movie_dict,
})
@@ -436,6 +441,7 @@ class MoviePlugin(Plugin):
db.commit()
elif new_movie_status:
new_status = fireEvent('status.get', new_movie_status, single = True)
movie.profile_id = None
movie.status_id = new_status.get('id')
db.commit()
else:
@@ -456,7 +462,7 @@ class MoviePlugin(Plugin):
log.debug('Can\'t restatus movie, doesn\'t seem to exist.')
return False
log.debug('Changing status for %s' % (m.library.titles[0].title))
log.debug('Changing status for %s', (m.library.titles[0].title))
if not m.profile:
m.status_id = done_status.get('id')
else:
couchpotato/core/plugins/movie/static/list.js
+14 -2
View File
@@ -72,7 +72,7 @@ var MovieList = new Class({
self.created = true;
},
addMovies: function(movies){
addMovies: function(movies, total){
var self = this;
if(!self.created) self.create();
@@ -86,8 +86,19 @@ var MovieList = new Class({
Object.each(movies, function(movie){
self.createMovie(movie);
});
self.setCounter(total);
},
setCounter: function(count){
var self = this;
if(!self.navigation_counter) return;
self.navigation_counter.set('text', (count || 0));
},
createMovie: function(movie, inject_at){
var self = this;
@@ -118,6 +129,7 @@ var MovieList = new Class({
self.navigation = new Element('div.alph_nav').adopt(
self.navigation_actions = new Element('ul.inlay.actions.reversed'),
self.navigation_counter = new Element('span.counter[title=Total]'),
self.navigation_alpha = new Element('ul.numbers', {
'events': {
'click:relay(li)': function(e, el){
@@ -443,7 +455,7 @@ var MovieList = new Class({
}, self.filter),
'onComplete': function(json){
self.store(json.movies);
self.addMovies(json.movies);
self.addMovies(json.movies, json.total);
self.load_more.set('text', 'load more movies');
if(self.scrollspy) self.scrollspy.start();
}
couchpotato/core/plugins/movie/static/movie.css
+51 -6
View File
@@ -13,6 +13,7 @@
overflow: hidden;
width: 100%;
transition: all 0.2s linear;
transform: translateZ(0);
}
.movies .movie.list_view, .movies .movie.mass_edit_view {
margin: 1px 0;
@@ -24,6 +25,10 @@
.movies .movie.list_view:hover, .movies .movie.mass_edit_view:hover {
background: rgba(255,255,255,0.03);
}
.movies .movie_container {
overflow: hidden;
}
.movies .data {
padding: 20px;
@@ -32,8 +37,8 @@
position: relative;
float: right;
border-radius: 0;
overflow: hidden;
transition: all 0.2s linear;
transform: translateZ(0);
}
.movies .list_view .data, .movies .mass_edit_view .data {
height: 30px;
@@ -62,6 +67,7 @@
height: 180px;
border-radius: 4px 0 0 4px;
transition: all 0.2s linear;
transform: translateZ(0);
}
.movies .list_view .poster, .movies .mass_edit_view .poster {
@@ -84,6 +90,7 @@
float: left;
width: 90%;
transition: all 0.2s linear;
transform: translateZ(0);
}
.movies .list_view .info .title, .movies .mass_edit_view .info .title {
font-size: 16px;
@@ -100,6 +107,7 @@
width: 10%;
text-align: right;
transition: all 0.2s linear;
transform: translateZ(0);
}
.movies .list_view .info .year, .movies .mass_edit_view .info .year {
font-size: 16px;
@@ -310,6 +318,35 @@
padding-bottom: 4px;
height: auto;
}
.movies .movie .trailer_container {
width: 100%;
background: #000;
text-align: center;
transition: all .6s cubic-bezier(0.9,0,0.1,1);
transform: translateZ(0);
overflow: hidden;
}
.movies .movie .trailer_container.hide {
height: 0 !important;
}
.movies .movie .hide_trailer {
position: absolute;
top: 0;
left: 50%;
margin-left: -50px;
width: 100px;
text-align: center;
padding: 3px 10px;
background: #4e5969;
border-radius: 0 0 2px 2px;
transition: all .6s cubic-bezier(0.9,0,0.1,1) .2s;
transform: translateZ(0);
}
.movies .movie .hide_trailer.hide {
top: -30px;
}
.movies .load_more {
display: block;
@@ -323,6 +360,7 @@
.movies .alph_nav {
transition: box-shadow .4s linear;
transform: translateZ(0);
position: fixed;
z-index: 2;
top: 0;
@@ -338,7 +376,9 @@
background: #4e5969;
}
.movies .alph_nav ul.numbers, .movies .alph_nav ul.actions {
.movies .alph_nav ul.numbers,
.movies .alph_nav .counter,
.movies .alph_nav ul.actions {
list-style: none;
padding: 0 0 1px;
margin: 0;
@@ -346,10 +386,15 @@
user-select: none;
}
.movies .alph_nav .counter {
width: 60px;
text-align: center;
}
.movies .alph_nav .numbers li, .movies .alph_nav .actions li {
display: inline-block;
vertical-align: top;
width: 22px;
width: 20px;
height: 24px;
line-height: 26px;
text-align: center;
@@ -357,11 +402,11 @@
color: rgba(255,255,255,0.2);
border: 1px solid transparent;
transition: all 0.1s ease-in-out;
transform: translateZ(0);
text-shadow: none;
}
.movies .alph_nav .numbers li:first-child {
width: 43px;
margin-left: 7px;
}
.movies .alph_nav li.available {
color: rgba(255,255,255,0.8);
@@ -370,8 +415,8 @@
}
.movies .alph_nav li.active.available, .movies .alph_nav li.available:hover {
color: #fff;
font-size: 24px;
line-height: 24px;
font-size: 20px;
line-height: 20px;
}
.movies .alph_nav input {
couchpotato/core/plugins/movie/static/movie.js
+110 -1
View File
@@ -77,7 +77,7 @@ var Movie = new Class({
self.profile = Quality.getProfile(self.data.profile_id) || {};
self.create();
self.busy(false);
},
@@ -246,6 +246,10 @@ var Movie = new Class({
isSelected: function(){
return this.select_checkbox.get('checked');
},
toElement: function(){
return this.el;
}
});
@@ -425,4 +429,109 @@ var ReleaseAction = new Class({
}
});
var TrailerAction = new Class({
Extends: MovieAction,
id: null,
create: function(){
var self = this;
self.el = new Element('a.trailer', {
'title': 'Watch the trailer of ' + self.movie.getTitle(),
'events': {
'click': self.watch.bind(self)
}
});
},
watch: function(offset){
var self = this;
var data_url = 'http://gdata.youtube.com/feeds/videos?vq="{title}" {year} trailer&max-results=1&alt=json-in-script&orderby=relevance&sortorder=descending&format=5&fmt=18'
var url = data_url.substitute({
'title': self.movie.getTitle(),
'year': self.movie.get('year'),
'offset': offset || 1
}),
size = $(self.movie).getSize(),
height = (size.x/16)*9,
id = 'trailer-'+randomString();
self.player_container = new Element('div[id='+id+']');
self.container = new Element('div.hide.trailer_container')
.adopt(self.player_container)
.inject(self.movie.container, 'top');
self.container.setStyle('height', 0);
self.container.removeClass('hide');
self.close_button = new Element('a.hide.hide_trailer', {
'text': 'Hide trailer',
'events': {
'click': self.stop.bind(self)
}
}).inject(self.movie);
setTimeout(function(){
$(self.movie).setStyle('max-height', height);
self.container.setStyle('height', height);
}, 100)
new Request.JSONP({
'url': url,
'onComplete': function(json){
var video_url = json.feed.entry[0].id.$t.split('/'),
video_id = video_url[video_url.length-1];
self.player = new YT.Player(id, {
'height': height,
'width': size.x,
'videoId': video_id,
'playerVars': {
'autoplay': 1,
'showsearch': 0,
'wmode': 'transparent',
'iv_load_policy': 3
}
});
self.close_button.removeClass('hide');
var quality_set = false;
var change_quality = function(state){
if(!quality_set && (state.data == 1 || state.data || 2)){
try {
self.player.setPlaybackQuality('hd720');
quality_set = true;
}
catch(e){
}
}
}
self.player.addEventListener('onStateChange', change_quality);
}
}).send()
},
stop: function(){
var self = this;
self.player.stopVideo();
self.container.addClass('hide');
self.close_button.addClass('hide');
setTimeout(function(){
self.container.destroy()
self.close_button.destroy();
}, 1800)
}
});
couchpotato/core/plugins/movie/static/search.js
+2 -2
View File
@@ -301,11 +301,11 @@ Block.Search.Item = new Class({
'title': self.title_select.get('value'),
'profile_id': self.profile_select.get('value')
},
'onComplete': function(){
'onComplete': function(json){
self.options.empty();
self.options.adopt(
new Element('div.message', {
'text': 'Movie succesfully added.'
'text': json.added ? 'Movie succesfully added.' : 'Movie didn\'t add properly. Check logs'
})
);
},
couchpotato/core/plugins/profile/main.py
+2 -3
View File
@@ -135,8 +135,7 @@ class ProfilePlugin(Plugin):
success = True
except Exception, e:
message = 'Failed deleting Profile: %s' % e
log.error(message)
message = log.error('Failed deleting Profile: %s', e)
#db.close()
@@ -163,7 +162,7 @@ class ProfilePlugin(Plugin):
# Create default quality profile
order = -2
for profile in profiles:
log.info('Creating default profile: %s' % profile.get('label'))
log.info('Creating default profile: %s', profile.get('label'))
p = Profile(
label = toUnicode(profile.get('label')),
order = order
couchpotato/core/plugins/quality/main.py
+11 -11
View File
@@ -17,8 +17,8 @@ class QualityPlugin(Plugin):
qualities = [
{'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]},
{'identifier': '1080p', 'hd': True, 'size': (5000, 20000), 'label': '1080P', 'width': 1920, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts']},
{'identifier': '720p', 'hd': True, 'size': (3500, 10000), 'label': '720P', 'width': 1280, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts']},
{'identifier': '1080p', 'hd': True, 'size': (5000, 20000), 'label': '1080P', 'width': 1920, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts']},
{'identifier': '720p', 'hd': True, 'size': (3500, 10000), 'label': '720P', 'width': 1280, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts']},
{'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p'], 'ext':['avi']},
{'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': [], 'allow': [], 'ext':['iso', 'img'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts']},
{'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': ['dvdrip'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']},
@@ -26,7 +26,7 @@ class QualityPlugin(Plugin):
{'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': [], 'allow': ['dvdr'], 'ext':['avi', 'mpg', 'mpeg']},
{'identifier': 'tc', 'size': (600, 1000), 'label': 'TeleCine', 'alternative': ['telecine'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']},
{'identifier': 'ts', 'size': (600, 1000), 'label': 'TeleSync', 'alternative': ['telesync'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']},
{'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': [], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}
{'identifier': 'cam', 'size': (600, 1000), 'label': 'Cam', 'alternative': ['camrip', 'hdcam'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}
]
pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr']
@@ -116,7 +116,7 @@ class QualityPlugin(Plugin):
quality = db.query(Quality).filter_by(identifier = q.get('identifier')).first()
if not quality:
log.info('Creating quality: %s' % q.get('label'))
log.info('Creating quality: %s', q.get('label'))
quality = Quality()
db.add(quality)
@@ -133,7 +133,7 @@ class QualityPlugin(Plugin):
).all()
if not profile:
log.info('Creating profile: %s' % q.get('label'))
log.info('Creating profile: %s', q.get('label'))
profile = Profile(
core = True,
label = toUnicode(quality.label),
@@ -170,20 +170,20 @@ class QualityPlugin(Plugin):
# Check tags
if quality['identifier'] in words:
log.debug('Found via identifier "%s" in %s' % (quality['identifier'], cur_file))
log.debug('Found via identifier "%s" in %s', (quality['identifier'], cur_file))
return self.setCache(hash, quality)
if list(set(quality.get('alternative', [])) & set(words)):
log.debug('Found %s via alt %s in %s' % (quality['identifier'], quality.get('alternative'), cur_file))
log.debug('Found %s via alt %s in %s', (quality['identifier'], quality.get('alternative'), cur_file))
return self.setCache(hash, quality)
for tag in quality.get('tags', []):
if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words):
log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file))
log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file))
return self.setCache(hash, quality)
if list(set(quality.get('tags', [])) & set(words)):
log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file))
log.debug('Found %s via tag %s in %s', (quality['identifier'], quality.get('tags'), cur_file))
return self.setCache(hash, quality)
# Try again with loose testing
@@ -191,7 +191,7 @@ class QualityPlugin(Plugin):
if quality:
return self.setCache(hash, quality)
log.debug('Could not identify quality for: %s' % files)
log.debug('Could not identify quality for: %s', files)
return None
def guessLoose(self, hash, extra):
@@ -200,7 +200,7 @@ class QualityPlugin(Plugin):
# Last check on resolution only
if quality.get('width', 480) == extra.get('resolution_width', 0):
log.debug('Found %s via resolution_width: %s == %s' % (quality['identifier'], quality.get('width', 480), extra.get('resolution_width', 0)))
log.debug('Found %s via resolution_width: %s == %s', (quality['identifier'], quality.get('width', 480), extra.get('resolution_width', 0)))
return self.setCache(hash, quality)
if 480 <= extra.get('resolution_width', 0) <= 720:
couchpotato/core/plugins/release/main.py
+2 -2
View File
@@ -79,7 +79,7 @@ class Release(Plugin):
rel.files.append(added_file)
db.commit()
except Exception, e:
log.debug('Failed to attach "%s" to release: %s' % (cur_file, e))
log.debug('Failed to attach "%s" to release: %s', (cur_file, e))
fireEvent('movie.restatus', movie.id)
@@ -158,7 +158,7 @@ class Release(Plugin):
'success': True
})
else:
log.error('Couldn\'t find release with id: %s' % id)
log.error('Couldn\'t find release with id: %s', id)
#db.close()
return jsonified({
couchpotato/core/plugins/renamer/main.py
+20 -20
View File
@@ -100,7 +100,7 @@ class Renamer(Plugin):
else:
group['library'] = fireEvent('library.update', identifier = group['library']['identifier'], single = True)
if not group['library']:
log.error('Could not rename, no library item to work with: %s' % group_identifier)
log.error('Could not rename, no library item to work with: %s', group_identifier)
continue
library = group['library']
@@ -138,7 +138,7 @@ class Renamer(Plugin):
# Move nfo depending on settings
if file_type is 'nfo' and not self.conf('rename_nfo'):
log.debug('Skipping, renaming of %s disabled' % file_type)
log.debug('Skipping, renaming of %s disabled', file_type)
if self.conf('cleanup'):
for current_file in group['files'][file_type]:
remove_files.append(current_file)
@@ -197,7 +197,7 @@ class Renamer(Plugin):
break
if not found:
log.error('Could not determine dvd structure for: %s' % current_file)
log.error('Could not determine dvd structure for: %s', current_file)
# Do rename others
else:
@@ -272,7 +272,7 @@ class Renamer(Plugin):
movie.status_id = done_status.get('id')
db.commit()
except Exception, e:
log.error('Failed marking movie finished: %s %s' % (e, traceback.format_exc()))
log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc()))
# Go over current movie releases
for release in movie.releases:
@@ -282,20 +282,20 @@ class Renamer(Plugin):
# This is where CP removes older, lesser quality releases
if release.quality.order > group['meta_data']['quality']['order']:
log.info('Removing lesser quality %s for %s.' % (movie.library.titles[0].title, release.quality.label))
log.info('Removing lesser quality %s for %s.', (movie.library.titles[0].title, release.quality.label))
for current_file in release.files:
remove_files.append(current_file)
remove_releases.append(release)
# Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc
elif release.quality.order is group['meta_data']['quality']['order']:
log.info('Same quality release already exists for %s, with quality %s. Assuming repack.' % (movie.library.titles[0].title, release.quality.label))
log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (movie.library.titles[0].title, release.quality.label))
for current_file in release.files:
remove_files.append(current_file)
remove_releases.append(release)
# Downloaded a lower quality, rename the newly downloaded files/folder to exclude them from scan
else:
log.info('Better quality release already exists for %s, with quality %s' % (movie.library.titles[0].title, release.quality.label))
log.info('Better quality release already exists for %s, with quality %s', (movie.library.titles[0].title, release.quality.label))
# Add _EXISTS_ to the parent dir
if group['dirname']:
@@ -333,18 +333,18 @@ class Renamer(Plugin):
if isinstance(src, File):
src = src.path
log.info('Removing "%s"' % src)
log.info('Removing "%s"', src)
try:
os.remove(src)
except:
log.error('Failed removing %s: %s' % (src, traceback.format_exc()))
log.error('Failed removing %s: %s', (src, traceback.format_exc()))
# Rename all files marked
group['renamed_files'] = []
for src in rename_files:
if rename_files[src]:
dst = rename_files[src]
log.info('Renaming "%s" to "%s"' % (src, dst))
log.info('Renaming "%s" to "%s"', (src, dst))
# Create dir
self.makeDir(os.path.dirname(dst))
@@ -353,22 +353,22 @@ class Renamer(Plugin):
self.moveFile(src, dst)
group['renamed_files'].append(dst)
except:
log.error('Failed moving the file "%s" : %s' % (os.path.basename(src), traceback.format_exc()))
log.error('Failed moving the file "%s" : %s', (os.path.basename(src), traceback.format_exc()))
# Remove matching releases
for release in remove_releases:
log.debug('Removing release %s' % release.identifier)
log.debug('Removing release %s', release.identifier)
try:
db.delete(release)
except:
log.error('Failed removing %s: %s' % (release.identifier, traceback.format_exc()))
log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc()))
if group['dirname'] and group['parentdir']:
try:
log.info('Deleting folder: %s' % group['parentdir'])
log.info('Deleting folder: %s', group['parentdir'])
self.deleteEmptyFolder(group['parentdir'])
except:
log.error('Failed removing %s: %s' % (group['parentdir'], traceback.format_exc()))
log.error('Failed removing %s: %s', (group['parentdir'], traceback.format_exc()))
if not unknown:
# Search for trailers etc
@@ -406,12 +406,12 @@ class Renamer(Plugin):
shutil.move(old, dest)
try:
os.chmod(dest, Env.getPermission('folder'))
os.chmod(dest, Env.getPermission('file'))
except:
log.error('Failed setting permissions for file: %s' % dest)
log.error('Failed setting permissions for file: %s, %s', (dest, traceback.format_exc(1)))
except:
log.error("Couldn't move file '%s' to '%s': %s" % (old, dest, traceback.format_exc()))
log.error('Couldn\'t move file "%s" to "%s": %s', (old, dest, traceback.format_exc()))
raise Exception
return True
@@ -447,9 +447,9 @@ class Renamer(Plugin):
try:
os.rmdir(full_path)
except:
log.error('Couldn\'t remove empty directory %s: %s' % (full_path, traceback.format_exc()))
log.error('Couldn\'t remove empty directory %s: %s', (full_path, traceback.format_exc()))
try:
os.rmdir(folder)
except:
log.error('Couldn\'t remove empty directory %s: %s' % (folder, traceback.format_exc()))
log.error('Couldn\'t remove empty directory %s: %s', (folder, traceback.format_exc()))
couchpotato/core/plugins/scanner/main.py
+71 -53
View File
@@ -1,11 +1,10 @@
from couchpotato import get_session
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import toUnicode, simplifyString
from couchpotato.core.helpers.encoding import toUnicode, simplifyString, ss
from couchpotato.core.helpers.variable import getExt, getImdb, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import File, Movie
from couchpotato.environment import Env
from enzyme.exceptions import NoParserError, ParseError
from guessit import guess_movie_info
from subliminal.videos import Video
@@ -89,9 +88,9 @@ class Scanner(Plugin):
addEvent('scanner.partnumber', self.getPartNumber)
def after_rename(group):
return self.scanFilesToLibrary(self, folder = group['destination_dir'], files = group['renamed_files'])
return self.scanFilesToLibrary(folder = group['destination_dir'], files = group['renamed_files'])
addEvent('rename.after', after_rename)
addEvent('renamer.after', after_rename)
def scanFilesToLibrary(self, folder = None, files = None):
@@ -103,14 +102,14 @@ class Scanner(Plugin):
if group['library']:
fireEvent('release.add', group = group)
def scanFolderToLibrary(self, folder = None, newer_than = None, simple = True):
def scanFolderToLibrary(self, folder = None, newer_than = 0, simple = True):
folder = os.path.normpath(folder)
if not os.path.isdir(folder):
return
groups = self.scan(folder = folder, simple = simple)
groups = self.scan(folder = folder, simple = simple, newer_than = newer_than)
added_identifier = []
while True and not self.shuttingDown():
@@ -131,12 +130,12 @@ class Scanner(Plugin):
return added_identifier
def scan(self, folder = None, files = [], simple = False):
def scan(self, folder = None, files = [], simple = False, newer_than = 0):
folder = os.path.normpath(folder)
folder = ss(os.path.normpath(folder))
if not folder or not os.path.isdir(folder):
log.error('Folder doesn\'t exists: %s' % folder)
log.error('Folder doesn\'t exists: %s', folder)
return {}
# Get movie "master" files
@@ -147,19 +146,11 @@ class Scanner(Plugin):
if len(files) == 0:
try:
files = []
for root, dirs, walk_files in os.walk(toUnicode(folder)):
for root, dirs, walk_files in os.walk(folder):
for filename in walk_files:
files.append(os.path.join(root, filename))
except:
try:
files = []
folder = toUnicode(folder).encode(Env.get('encoding'))
log.info('Trying to convert unicode to str path: %s, %s' % (folder, type(folder)))
for root, dirs, walk_files in os.walk(folder):
for filename in walk_files:
files.append(os.path.join(root, filename))
except:
log.error('Failed getting files from %s: %s' % (folder, traceback.format_exc()))
log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
db = get_session()
@@ -215,7 +206,7 @@ class Scanner(Plugin):
for identifier, group in movie_files.iteritems():
if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier)
log.debug('Grouping files: %s' % identifier)
log.debug('Grouping files: %s', identifier)
for file_path in group['unsorted_files']:
wo_ext = file_path[:-(len(getExt(file_path)) + 1)]
@@ -241,7 +232,7 @@ class Scanner(Plugin):
# Group the files based on the identifier
delete_identifiers = []
for identifier, found_files in self.path_identifiers.iteritems():
log.debug('Grouping files on identifier: %s' % identifier)
log.debug('Grouping files on identifier: %s', identifier)
group = movie_files.get(identifier)
if group:
@@ -257,13 +248,14 @@ class Scanner(Plugin):
# Cleaning up used
for identifier in delete_identifiers:
del self.path_identifiers[identifier]
if self.path_identifiers.get(identifier):
del self.path_identifiers[identifier]
del delete_identifiers
# Group based on folder
delete_identifiers = []
for identifier, found_files in self.path_identifiers.iteritems():
log.debug('Grouping files on foldername: %s' % identifier)
log.debug('Grouping files on foldername: %s', identifier)
for ff in found_files:
new_identifier = self.createStringIdentifier(os.path.dirname(ff), folder)
@@ -282,7 +274,8 @@ class Scanner(Plugin):
# Cleaning up used
for identifier in delete_identifiers:
del self.path_identifiers[identifier]
if self.path_identifiers.get(identifier):
del self.path_identifiers[identifier]
del delete_identifiers
# Determine file types
@@ -296,13 +289,38 @@ class Scanner(Plugin):
# Check if movie is fresh and maybe still unpacking, ignore files new then 1 minute
file_too_new = False
for cur_file in group['unsorted_files']:
file_time = os.path.getmtime(cur_file)
if file_time > time.time() - 60:
file_too_new = tryInt(time.time() - file_time)
if not os.path.isfile(cur_file):
file_too_new = time.time()
break
file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)]
for t in file_time:
if t > time.time() - 60:
file_too_new = tryInt(time.time() - t)
break
if file_too_new:
break
if file_too_new:
log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s' % (time.ctime(file_time), identifier))
log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time.ctime(file_time[0]), identifier))
# Delete the unsorted list
del group['unsorted_files']
continue
# Only process movies newer than x
if newer_than and newer_than > 0:
for cur_file in group['unsorted_files']:
file_time = [os.path.getmtime(cur_file), os.path.getctime(cur_file)]
if file_time[0] > time.time() or file_time[1] > time.time():
break
log.debug('None of the files have changed since %s for %s, skipping.', (time.ctime(newer_than), identifier))
# Delete the unsorted list
del group['unsorted_files']
continue
# Group extra (and easy) files first
@@ -324,10 +342,10 @@ class Scanner(Plugin):
group['files']['movie'] = self.getMediaFiles(group['unsorted_files'])
if len(group['files']['movie']) == 0:
log.error('Couldn\t find any movie files for %s' % identifier)
log.error('Couldn\t find any movie files for %s', identifier)
continue
log.debug('Getting metadata for %s' % identifier)
log.debug('Getting metadata for %s', identifier)
group['meta_data'] = self.getMetaData(group)
# Subtitle meta
@@ -360,7 +378,7 @@ class Scanner(Plugin):
# Determine movie
group['library'] = self.determineMovie(group)
if not group['library']:
log.error('Unable to determine movie: %s' % group['identifiers'])
log.error('Unable to determine movie: %s', group['identifiers'])
else:
movie = db.query(Movie).filter_by(library_id = group['library']['id']).first()
group['movie_id'] = None if not movie else movie.id
@@ -373,9 +391,9 @@ class Scanner(Plugin):
self.path_identifiers = {}
if len(processed_movies) > 0:
log.info('Found %s movies in the folder %s' % (len(processed_movies), folder))
log.info('Found %s movies in the folder %s', (len(processed_movies), folder))
else:
log.debug('Found no movies in the folder %s' % (folder))
log.debug('Found no movies in the folder %s', (folder))
return processed_movies
def getMetaData(self, group):
@@ -395,7 +413,7 @@ class Scanner(Plugin):
data['resolution_height'] = meta.get('resolution_height', 480)
data['aspect'] = meta.get('resolution_width', 720) / meta.get('resolution_height', 480)
except:
log.debug('Error parsing metadata: %s %s' % (cur_file, traceback.format_exc()))
log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc()))
pass
if data.get('audio'): break
@@ -423,11 +441,11 @@ class Scanner(Plugin):
'resolution_height': tryInt(p.video[0].height),
}
except ParseError:
log.debug('Failed to parse meta for %s' % filename)
log.debug('Failed to parse meta for %s', filename)
except NoParserError:
log.debug('No parser found for %s' % filename)
log.debug('No parser found for %s', filename)
except:
log.debug('Failed parsing %s' % filename)
log.debug('Failed parsing %s', filename)
return {}
@@ -449,7 +467,7 @@ class Scanner(Plugin):
if s.language and s.path not in paths:
detected_languages[s.path] = [s.language]
except:
log.debug('Failed parsing subtitle languages for %s: %s' % (paths, traceback.format_exc()))
log.debug('Failed parsing subtitle languages for %s: %s', (paths, traceback.format_exc()))
# IDX
for extra in group['files']['subtitle_extra']:
@@ -465,7 +483,7 @@ class Scanner(Plugin):
if len(idx_langs) > 0 and os.path.isfile(sub_file):
detected_languages[sub_file] = idx_langs
except:
log.error('Failed parsing subtitle idx for %s: %s' % (extra, traceback.format_exc()))
log.error('Failed parsing subtitle idx for %s: %s', (extra, traceback.format_exc()))
return detected_languages
@@ -478,7 +496,7 @@ class Scanner(Plugin):
for cur_file in files['movie']:
imdb_id = self.getCPImdb(cur_file)
if imdb_id:
log.debug('Found movie via CP tag: %s' % cur_file)
log.debug('Found movie via CP tag: %s', cur_file)
break
# Check and see if nfo contains the imdb-id
@@ -487,7 +505,7 @@ class Scanner(Plugin):
for nfo_file in files['nfo']:
imdb_id = getImdb(nfo_file)
if imdb_id:
log.debug('Found movie via nfo file: %s' % nfo_file)
log.debug('Found movie via nfo file: %s', nfo_file)
break
except:
pass
@@ -499,7 +517,7 @@ class Scanner(Plugin):
for filetype_file in files[filetype]:
imdb_id = getImdb(filetype_file, check_inside = False)
if imdb_id:
log.debug('Found movie via imdb in filename: %s' % nfo_file)
log.debug('Found movie via imdb in filename: %s', nfo_file)
break
except:
pass
@@ -511,7 +529,7 @@ class Scanner(Plugin):
f = db.query(File).filter_by(path = toUnicode(cur_file)).first()
try:
imdb_id = f.library[0].identifier
log.debug('Found movie via database: %s' % cur_file)
log.debug('Found movie via database: %s', cur_file)
break
except:
pass
@@ -525,7 +543,7 @@ class Scanner(Plugin):
if len(movie) > 0:
imdb_id = movie[0]['imdb']
if imdb_id:
log.debug('Found movie via OpenSubtitleHash: %s' % cur_file)
log.debug('Found movie via OpenSubtitleHash: %s', cur_file)
break
# Search based on identifiers
@@ -542,17 +560,17 @@ class Scanner(Plugin):
if len(movie) > 0:
imdb_id = movie[0]['imdb']
log.debug('Found movie via search: %s' % cur_file)
log.debug('Found movie via search: %s', cur_file)
if imdb_id: break
else:
log.debug('Identifier to short to use for search: %s' % identifier)
log.debug('Identifier to short to use for search: %s', identifier)
if imdb_id:
return fireEvent('library.add', attrs = {
'identifier': imdb_id
}, update_after = False, single = True)
log.error('No imdb_id found for %s. Add a NFO file with IMDB id or add the year to the filename.' % group['identifiers'])
log.error('No imdb_id found for %s. Add a NFO file with IMDB id or add the year to the filename.', group['identifiers'])
return {}
def getCPImdb(self, string):
@@ -641,17 +659,17 @@ class Scanner(Plugin):
# ignoredpaths
for i in self.ignored_in_path:
if i in filename.lower():
log.debug('Ignored "%s" contains "%s".' % (filename, i))
log.debug('Ignored "%s" contains "%s".', (filename, i))
return False
# Sample file
if self.isSampleFile(filename):
log.debug('Is sample file "%s".' % filename)
log.debug('Is sample file "%s".', filename)
return False
# Minimal size
if self.filesizeBetween(filename, self.minimal_filesize['media']):
log.debug('File to small: %s' % filename)
log.debug('File to small: %s', filename)
return False
# All is OK
@@ -659,14 +677,14 @@ class Scanner(Plugin):
def isSampleFile(self, filename):
is_sample = re.search('(^|[\W_])sample\d*[\W_]', filename.lower())
if is_sample: log.debug('Is sample file: %s' % filename)
if is_sample: log.debug('Is sample file: %s', filename)
return is_sample
def filesizeBetween(self, file, min = 0, max = 100000):
try:
return (min * 1048576) < os.path.getsize(file) < (max * 1048576)
except:
log.error('Couldn\'t get filesize of %s.' % file)
log.error('Couldn\'t get filesize of %s.', file)
return False
@@ -769,7 +787,7 @@ class Scanner(Plugin):
'year': guess.get('year'),
}
except:
log.debug('Could not detect via guessit "%s": %s' % (file_name, traceback.format_exc()))
log.debug('Could not detect via guessit "%s": %s', (file_name, traceback.format_exc()))
# Backup to simple
cleaned = ' '.join(re.split('\W+', simplifyString(release_name)))
couchpotato/core/plugins/score/main.py
+9 -1
View File
@@ -4,7 +4,8 @@ from couchpotato.core.helpers.variable import getTitle
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.plugins.score.scores import nameScore, nameRatioScore, \
sizeScore, providerScore, duplicateScore
sizeScore, providerScore, duplicateScore, partialIgnoredScore, namePositionScore, \
halfMultipartScore
log = CPLog(__name__)
@@ -21,6 +22,7 @@ class Score(Plugin):
for movie_title in movie['library']['titles']:
score += nameRatioScore(toUnicode(nzb['name']), toUnicode(movie_title['title']))
score += namePositionScore(toUnicode(nzb['name']), toUnicode(movie_title['title']))
score += sizeScore(nzb['size'])
@@ -38,6 +40,12 @@ class Score(Plugin):
# Duplicates in name
score += duplicateScore(nzb['name'], getTitle(movie['library']))
# Partial ignored words
score += partialIgnoredScore(nzb['name'], getTitle(movie['library']))
# Ignore single downloads from multipart
score += halfMultipartScore(nzb['name'])
# Extra provider specific check
extra_score = nzb.get('extra_score')
if extra_score:
couchpotato/core/plugins/score/scores.py
+88 -4
View File
@@ -1,5 +1,7 @@
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import simplifyString
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.plugins.scanner.main import Scanner
from couchpotato.environment import Env
import re
@@ -16,11 +18,12 @@ name_scores = [
'german:-10', 'french:-10', 'spanish:-10', 'swesub:-20', 'danish:-10', 'dutch:-10',
# Release groups
'imbt:1', 'cocain:1', 'vomit:1', 'fico:1', 'arrow:1', 'pukka:1', 'prism:1', 'devise:1', 'esir:1', 'ctrlhd:1',
'metis:1', 'diamond:1', 'wiki:1', 'cbgb:1', 'crossbow:1', 'sinners:1', 'amiable:1', 'refined:1', 'twizted:1', 'felony:1', 'hubris:1', 'machd:1',
'metis:10', 'diamond:10', 'wiki:10', 'cbgb:10', 'crossbow:1', 'sinners:10', 'amiable:10', 'refined:1', 'twizted:1', 'felony:1', 'hubris:1', 'machd:1',
# Extras
'extras:-40', 'trilogy:-40',
]
def nameScore(name, year):
''' Calculate score for words in the NZB name '''
@@ -47,8 +50,8 @@ def nameScore(name, year):
return score
def nameRatioScore(nzb_name, movie_name):
def nameRatioScore(nzb_name, movie_name):
nzb_words = re.split('\W+', fireEvent('scanner.create_file_identifier', nzb_name, single = True))
movie_words = re.split('\W+', simplifyString(movie_name))
@@ -56,15 +59,68 @@ def nameRatioScore(nzb_name, movie_name):
return 10 - len(left_over)
def namePositionScore(nzb_name, movie_name):
score = 0
nzb_words = re.split('\W+', simplifyString(nzb_name))
qualities = fireEvent('quality.all', single = True)
try:
nzb_name = re.search(r'([\'"])[^\1]*\1', nzb_name).group(0)
except:
pass
name_year = fireEvent('scanner.name_year', nzb_name, single = True)
# Give points for movies beginning with the correct name
name_split = simplifyString(nzb_name).split(simplifyString(movie_name))
if name_split[0].strip() == '':
score += 10
# If year is second in line, give more points
if len(name_split) > 1 and name_year:
after_name = name_split[1].strip()
if tryInt(after_name[:4]) == name_year.get('year', None):
score += 10
after_name = after_name[4:]
# Give -point to crap between year and quality
found_quality = None
for quality in qualities:
# Main in words
if quality['identifier'] in nzb_words:
found_quality = quality['identifier']
# Alt in words
for alt in quality['alternative']:
if alt in nzb_words:
found_quality = alt
break
if not found_quality:
return score - 20
allowed = []
for value in name_scores:
name, sc = value.split(':')
allowed.append(name)
inbetween = re.split('\W+', after_name.split(found_quality)[0].strip())
score -= (10 * len(set(inbetween) - set(allowed)))
return score
def sizeScore(size):
return 0 if size else -20
def providerScore(provider):
if provider in ['NZBMatrix', 'Nzbs', 'Newzbin']:
return 30
return 20
if provider in ['Newznab', 'Moovee', 'X264']:
if provider in ['Newznab']:
return 10
return 0
@@ -79,3 +135,31 @@ def duplicateScore(nzb_name, movie_name):
duplicates = [x for i, x in enumerate(nzb_words) if nzb_words[i:].count(x) > 1]
return len(list(set(duplicates) - set(movie_words))) * -4
def partialIgnoredScore(nzb_name, movie_name):
nzb_name = nzb_name.lower()
movie_name = movie_name.lower()
ignored_words = [x.strip().lower() for x in Env.setting('ignored_words', section = 'searcher').split(',')]
score = 0
for ignored_word in ignored_words:
if ignored_word in nzb_name and ignored_word not in movie_name:
score -= 5
return score
def halfMultipartScore(nzb_name):
wrong_found = 0
for nr in [1, 2, 3, 4, 5, 'i', 'ii', 'iii', 'iv', 'v', 'a', 'b', 'c', 'd', 'e']:
for wrong in ['cd', 'part', 'dis', 'disc', 'dvd']:
if '%s%s' % (wrong, nr) in nzb_name.lower():
wrong_found += 1
if wrong_found == 1:
return -30
return 0
couchpotato/core/plugins/searcher/main.py
+27 -22
View File
@@ -56,7 +56,7 @@ class Searcher(Plugin):
except IndexError:
fireEvent('library.update', movie_dict['library']['identifier'], force = True)
except:
log.error('Search failed for %s: %s' % (movie_dict['library']['identifier'], traceback.format_exc()))
log.error('Search failed for %s: %s', (movie_dict['library']['identifier'], traceback.format_exc()))
# Break if CP wants to shut down
if self.shuttingDown():
@@ -88,7 +88,7 @@ class Searcher(Plugin):
ret = False
for quality_type in movie['profile']['types']:
if not self.couldBeReleased(quality_type['quality']['identifier'], release_dates, pre_releases):
log.info('To early to search for %s, %s' % (quality_type['quality']['identifier'], default_title))
log.info('To early to search for %s, %s', (quality_type['quality']['identifier'], default_title))
continue
has_better_quality = 0
@@ -101,12 +101,12 @@ class Searcher(Plugin):
# Don't search for quality lower then already available.
if has_better_quality is 0:
log.info('Search for %s in %s' % (default_title, quality_type['quality']['label']))
log.info('Search for %s in %s', (default_title, quality_type['quality']['label']))
quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True)
results = fireEvent('yarr.search', movie, quality, merge = True)
sorted_results = sorted(results, key = lambda k: k['score'], reverse = True)
if len(sorted_results) == 0:
log.debug('Nothing found for %s in %s' % (default_title, quality_type['quality']['label']))
log.debug('Nothing found for %s in %s', (default_title, quality_type['quality']['label']))
# Check if movie isn't deleted while searching
if not db.query(Movie).filter_by(id = movie.get('id')).first():
@@ -141,10 +141,13 @@ class Searcher(Plugin):
rls.info.append(rls_info)
db.commit()
except InterfaceError:
log.debug('Couldn\'t add %s to ReleaseInfo: %s' % (info, traceback.format_exc()))
log.debug('Couldn\'t add %s to ReleaseInfo: %s', (info, traceback.format_exc()))
for nzb in sorted_results:
if nzb['score'] <= 0:
log.debug('No more releases with score higher than 0')
break
downloaded = self.download(data = nzb, movie = movie)
if downloaded is True:
ret = True
@@ -152,7 +155,7 @@ class Searcher(Plugin):
elif downloaded != 'try_next':
break
else:
log.info('Better quality (%s) already available or snatched for %s' % (quality_type['quality']['label'], default_title))
log.info('Better quality (%s) already available or snatched for %s', (quality_type['quality']['label'], default_title))
fireEvent('movie.restatus', movie['id'])
break
@@ -200,7 +203,7 @@ class Searcher(Plugin):
if movie['status_id'] == active_status.get('id'):
for profile_type in movie['profile']['types']:
if profile_type['quality_id'] == rls.quality.id and profile_type['finish']:
log.info('Renamer disabled, marking movie as finished: %s' % log_movie)
log.info('Renamer disabled, marking movie as finished: %s', log_movie)
# Mark release done
rls.status_id = done_status.get('id')
@@ -211,7 +214,7 @@ class Searcher(Plugin):
mvie.status_id = done_status.get('id')
db.commit()
except Exception, e:
log.error('Failed marking movie finished: %s %s' % (e, traceback.format_exc()))
log.error('Failed marking movie finished: %s %s', (e, traceback.format_exc()))
#db.close()
return True
@@ -226,7 +229,7 @@ class Searcher(Plugin):
retention = Env.setting('retention', section = 'nzb')
if nzb.get('seeds') is None and retention < nzb.get('age', 0):
log.info('Wrong: Outside retention, age is %s, needs %s or lower: %s' % (nzb['age'], retention, nzb['name']))
log.info('Wrong: Outside retention, age is %s, needs %s or lower: %s', (nzb['age'], retention, nzb['name']))
return False
movie_name = getTitle(movie['library'])
@@ -245,10 +248,10 @@ class Searcher(Plugin):
log.info("Wrong: '%s' blacklisted words: %s" % (nzb['name'], ", ".join(blacklisted)))
return False
pron_tags = ['xxx', 'sex', 'anal', 'tits', 'fuck', 'porn', 'orgy', 'milf', 'boobs']
pron_tags = ['xxx', 'sex', 'anal', 'tits', 'fuck', 'porn', 'orgy', 'milf', 'boobs', 'erotica', 'erotic']
for p_tag in pron_tags:
if p_tag in nzb_words and p_tag not in movie_words:
log.info('Wrong: %s, probably pr0n' % (nzb['name']))
log.info('Wrong: %s, probably pr0n', (nzb['name']))
return False
#qualities = fireEvent('quality.all', single = True)
@@ -256,18 +259,18 @@ class Searcher(Plugin):
# Contains lower quality string
if self.containsOtherQuality(nzb, movie_year = movie['library']['year'], preferred_quality = preferred_quality, single_category = single_category):
log.info('Wrong: %s, looking for %s' % (nzb['name'], quality['label']))
log.info('Wrong: %s, looking for %s', (nzb['name'], quality['label']))
return False
# File to small
if nzb['size'] and preferred_quality['size_min'] > nzb['size']:
log.info('"%s" is too small to be %s. %sMB instead of the minimal of %sMB.' % (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_min']))
log.info('"%s" is too small to be %s. %sMB instead of the minimal of %sMB.', (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_min']))
return False
# File to large
if nzb['size'] and preferred_quality.get('size_max') < nzb['size']:
log.info('"%s" is too large to be %s. %sMB instead of the maximum of %sMB.' % (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_max']))
log.info('"%s" is too large to be %s. %sMB instead of the maximum of %sMB.', (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_max']))
return False
@@ -406,17 +409,19 @@ class Searcher(Plugin):
if dates.get('theater') - 604800 < now:
return True
else:
# 6 weeks after theater release
if dates.get('theater') + 3628800 < now:
# 12 weeks after theater release
if dates.get('theater') > 0 and dates.get('theater') + 7257600 < now:
return True
# 6 weeks before dvd release
if dates.get('dvd') - 3628800 < now:
return True
if dates.get('dvd') > 0:
# Dvd should be released
if dates.get('dvd') > 0 and dates.get('dvd') < now:
return True
# 3 weeks before dvd release
if dates.get('dvd') - 1814400 < now:
return True
# Dvd should be released
if dates.get('dvd') < now:
return True
return False
couchpotato/core/plugins/status/main.py
+1 -1
View File
@@ -92,7 +92,7 @@ class StatusPlugin(Plugin):
for identifier, label in self.statuses.iteritems():
s = db.query(Status).filter_by(identifier = identifier).first()
if not s:
log.info('Creating status: %s' % label)
log.info('Creating status: %s', label)
s = Status(
identifier = identifier,
label = toUnicode(label)
couchpotato/core/plugins/trailer/main.py
+2 -2
View File
@@ -18,7 +18,7 @@ class Trailer(Plugin):
trailers = fireEvent('trailer.search', group = group, merge = True)
if not trailers or trailers == []:
log.info('No trailers found for: %s' % getTitle(group['library']))
log.info('No trailers found for: %s', getTitle(group['library']))
return
for trailer in trailers.get(self.conf('quality'), []):
@@ -26,7 +26,7 @@ class Trailer(Plugin):
if not os.path.isfile(destination):
fireEvent('file.download', url = trailer, dest = destination, urlopen_kwargs = {'headers': {'User-Agent': 'Quicktime'}}, single = True)
else:
log.debug('Trailer already exists: %s' % destination)
log.debug('Trailer already exists: %s', destination)
# Download first and break
break
couchpotato/core/plugins/userscript/main.py
+1 -1
View File
@@ -73,7 +73,7 @@ class Userscript(Plugin):
'movie': fireEvent('userscript.get_movie_via_url', url = url, single = True)
}
if not isDict(params['movie']):
log.error('Failed adding movie via url: %s' % url)
log.error('Failed adding movie via url: %s', url)
params['error'] = params['movie'] if params['movie'] else 'Failed getting movie info'
return jsonified(params)
couchpotato/core/providers/automation/base.py
+2 -2
View File
@@ -20,7 +20,7 @@ class Automation(Plugin):
def _getMovies(self):
if not self.canCheck():
log.debug('Just checked, skipping %s' % self.getName())
log.debug('Just checked, skipping %s', self.getName())
return []
self.last_checked = time.time()
@@ -43,7 +43,7 @@ class Automation(Plugin):
type_value = movie.get(minimal_type, 0)
type_min = self.getMinimal(minimal_type)
if type_value < type_min:
log.info('%s to low for %s, need %s has %s' % (minimal_type, identifier, type_min, type_value))
log.info('%s to low for %s, need %s has %s', (minimal_type, identifier, type_min, type_value))
return False
return True
couchpotato/core/providers/automation/imdb/main.py
+8 -4
View File
@@ -31,12 +31,13 @@ class IMDB(Automation, RSS):
if not enablers[index]:
continue
elif 'rss.imdb' not in rss_url:
log.error('This isn\'t the correct url.: %s' % rss_url)
log.error('This isn\'t the correct url.: %s', rss_url)
continue
prop_name = 'automation.imdb.last_update.%s' % md5(rss_url)
last_update = float(Env.prop(prop_name, default = 0))
last_movie_added = 0
try:
cache_key = 'imdb.rss.%s' % md5(rss_url)
@@ -48,14 +49,17 @@ class IMDB(Automation, RSS):
created = int(time.mktime(parse(self.getTextElement(movie, "pubDate")).timetuple()))
imdb = getImdb(self.getTextElement(movie, "link"))
if not imdb or created < last_update:
if created > last_movie_added:
last_movie_added = created
if not imdb or created <= last_update:
continue
movies.append(imdb)
except:
log.error('Failed loading IMDB watchlist: %s %s' % (rss_url, traceback.format_exc()))
log.error('Failed loading IMDB watchlist: %s %s', (rss_url, traceback.format_exc()))
Env.prop(prop_name, time.time())
Env.prop(prop_name, last_movie_added)
return movies
couchpotato/core/providers/automation/trakt/main.py
+15 -9
View File
@@ -41,13 +41,19 @@ class Trakt(Automation):
def call(self, method_url):
if self.conf('automation_password'):
headers = {
'Authorization': "Basic %s" % base64.encodestring('%s:%s' % (self.conf('automation_username'), self.conf('automation_password')))[:-1]
}
else:
headers = {}
try:
if self.conf('automation_password'):
headers = {
'Authorization': 'Basic %s' % base64.encodestring('%s:%s' % (self.conf('automation_username'), self.conf('automation_password')))[:-1]
}
else:
headers = {}
cache_key = 'trakt.%s' % md5(method_url)
json_string = self.getCache(cache_key, self.urls['base'] + method_url, headers = headers)
return json.loads(json_string)
cache_key = 'trakt.%s' % md5(method_url)
json_string = self.getCache(cache_key, self.urls['base'] + method_url, headers = headers)
if json_string:
return json.loads(json_string)
except:
log.error('Failed to get data from trakt, check your login.')
return []
couchpotato/core/providers/base.py
+3 -3
View File
@@ -31,7 +31,7 @@ class Provider(Plugin):
self.urlopen(test_url, 30)
self.is_available[host] = True
except:
log.error('"%s" unavailable, trying again in an 15 minutes.' % host)
log.error('"%s" unavailable, trying again in an 15 minutes.', host)
self.is_available[host] = False
return self.is_available.get(host, False)
@@ -73,7 +73,7 @@ class YarrProvider(Provider):
if hostname in download_url:
return self
except:
log.debug('Url % s doesn\'t belong to %s' % (url, self.getName()))
log.debug('Url % s doesn\'t belong to %s', (url, self.getName()))
return
@@ -106,4 +106,4 @@ class YarrProvider(Provider):
return [self.cat_backup_id]
def found(self, new):
log.info('Found: score(%(score)s) on %(provider)s: %(name)s' % new)
log.info('Found: score(%(score)s) on %(provider)s: %(name)s', new)
couchpotato/core/providers/metadata/base.py
+4 -4
View File
@@ -19,14 +19,14 @@ class MetaDataBase(Plugin):
def create(self, release):
if self.isDisabled(): return
log.info('Creating %s metadata.' % self.getName())
log.info('Creating %s metadata.', self.getName())
# Update library to get latest info
try:
updated_library = fireEvent('library.update', release['library']['identifier'], force = True, single = True)
release['library'] = mergeDicts(release['library'], updated_library)
except:
log.error('Failed to update movie, before creating metadata: %s' % traceback.format_exc())
log.error('Failed to update movie, before creating metadata: %s', traceback.format_exc())
root_name = self.getRootName(release)
meta_name = os.path.basename(root_name)
@@ -44,14 +44,14 @@ class MetaDataBase(Plugin):
# Get file content
content = getattr(self, 'get' + file_type.capitalize())(movie_info = movie_info, data = release)
if content:
log.debug('Creating %s file: %s' % (file_type, name))
log.debug('Creating %s file: %s', (file_type, name))
if os.path.isfile(content):
shutil.copy2(content, name)
else:
self.createFile(name, content)
except:
log.error('Unable to create %s file: %s' % (file_type, traceback.format_exc()))
log.error('Unable to create %s file: %s', (file_type, traceback.format_exc()))
def getRootName(self, data):
return
couchpotato/core/providers/metadata/xbmc/main.py
+1 -1
View File
@@ -77,7 +77,7 @@ class XBMC(MetaDataBase):
votes.text = str(v)
break
except:
log.debug('Failed adding rating info from %s: %s' % (rating_type, traceback.format_exc()))
log.debug('Failed adding rating info from %s: %s', (rating_type, traceback.format_exc()))
# Genre
for genre in movie_info.get('genres', []):
couchpotato/core/providers/movie/_modifier/main.py
+1 -1
View File
@@ -68,7 +68,7 @@ class MovieResultModifier(Plugin):
if release.status_id == done_status['id']:
temp['in_library'] = fireEvent('movie.get', movie.id, single = True)
except:
log.error('Tried getting more info on searched movies: %s' % traceback.format_exc())
log.error('Tried getting more info on searched movies: %s', traceback.format_exc())
#db.close()
return temp
couchpotato/core/providers/movie/couchpotatoapi/main.py
+4 -4
View File
@@ -32,10 +32,10 @@ class CouchPotatoApi(MovieProvider):
headers = {'X-CP-Version': fireEvent('app.version', single = True)}
data = self.urlopen((self.api_url % ('eta')) + (identifier + '/'), headers = headers)
dates = json.loads(data)
log.debug('Found ETA for %s: %s' % (identifier, dates))
log.debug('Found ETA for %s: %s', (identifier, dates))
return dates
except Exception, e:
log.error('Error getting ETA for %s: %s' % (identifier, e))
log.error('Error getting ETA for %s: %s', (identifier, e))
return {}
@@ -43,9 +43,9 @@ class CouchPotatoApi(MovieProvider):
try:
data = self.urlopen((self.api_url % ('suggest')) + ','.join(movies) + '/' + ','.join(ignore) + '/')
suggestions = json.loads(data)
log.info('Found Suggestions for %s' % (suggestions))
log.info('Found Suggestions for %s', (suggestions))
except Exception, e:
log.error('Error getting suggestions for %s: %s' % (movies, e))
log.error('Error getting suggestions for %s: %s', (movies, e))
return suggestions
couchpotato/core/providers/movie/imdbapi/main.py
+3 -3
View File
@@ -36,7 +36,7 @@ class IMDBAPI(MovieProvider):
if cached:
result = self.parseMovie(cached)
if result.get('titles') and len(result.get('titles')) > 0:
log.info('Found: %s' % result['titles'][0] + ' (' + str(result['year']) + ')')
log.info('Found: %s', result['titles'][0] + ' (' + str(result['year']) + ')')
return [result]
return []
@@ -54,7 +54,7 @@ class IMDBAPI(MovieProvider):
if cached:
result = self.parseMovie(cached)
if result.get('titles') and len(result.get('titles')) > 0:
log.info('Found: %s' % result['titles'][0] + ' (' + str(result['year']) + ')')
log.info('Found: %s', result['titles'][0] + ' (' + str(result['year']) + ')')
return result
return {}
@@ -103,7 +103,7 @@ class IMDBAPI(MovieProvider):
'actors': movie.get('Actors', '').split(','),
}
except:
log.error('Failed parsing IMDB API json: %s' % traceback.format_exc())
log.error('Failed parsing IMDB API json: %s', traceback.format_exc())
return movie_data
couchpotato/core/providers/movie/themoviedb/main.py
+9 -9
View File
@@ -28,7 +28,7 @@ class TheMovieDb(MovieProvider):
results = self.getCache(cache_key)
if not results:
log.debug('Searching for movie by hash: %s' % file)
log.debug('Searching for movie by hash: %s', file)
try:
raw = tmdb.searchByHashingFile(file)
@@ -36,15 +36,15 @@ class TheMovieDb(MovieProvider):
if raw:
try:
results = self.parseMovie(raw)
log.info('Found: %s' % results['titles'][0] + ' (' + str(results['year']) + ')')
log.info('Found: %s', results['titles'][0] + ' (' + str(results['year']) + ')')
self.setCache(cache_key, results)
return results
except SyntaxError, e:
log.error('Failed to parse XML response: %s' % e)
log.error('Failed to parse XML response: %s', e)
return False
except:
log.debug('No movies known by hash for: %s' % file)
log.debug('No movies known by hash for: %s', file)
pass
return results
@@ -60,7 +60,7 @@ class TheMovieDb(MovieProvider):
results = self.getCache(cache_key)
if not results:
log.debug('Searching for movie: %s' % q)
log.debug('Searching for movie: %s', q)
raw = tmdb.search(search_string)
results = []
@@ -75,12 +75,12 @@ class TheMovieDb(MovieProvider):
if nr == limit:
break
log.info('Found: %s' % [result['titles'][0] + ' (' + str(result['year']) + ')' for result in results])
log.info('Found: %s', [result['titles'][0] + ' (' + str(result['year']) + ')' for result in results])
self.setCache(cache_key, results)
return results
except SyntaxError, e:
log.error('Failed to parse XML response: %s' % e)
log.error('Failed to parse XML response: %s', e)
return False
return results
@@ -98,7 +98,7 @@ class TheMovieDb(MovieProvider):
movie = None
try:
log.debug('Getting info: %s' % cache_key)
log.debug('Getting info: %s', cache_key)
movie = tmdb.imdbLookup(id = identifier)
except:
pass
@@ -119,7 +119,7 @@ class TheMovieDb(MovieProvider):
movie = None
try:
log.debug('Getting info: %s' % cache_key)
log.debug('Getting info: %s', cache_key)
movie = tmdb.getMovieInfo(id = id)
except:
pass
couchpotato/core/providers/nzb/mysterbin/main.py
+5 -5
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString
@@ -49,21 +49,21 @@ class Mysterbin(NZBProvider):
try:
html = BeautifulSoup(data)
resultable = html.find('table', attrs = {'class':'t'})
for result in resultable.findAll('tr'):
for result in resultable.find_all('tr'):
try:
myster_id = result.find('input', attrs = {'class': 'check4nzb'})['value']
# Age
age = ''
for temp in result.find('td', attrs = {'class': 'cdetail'}).findAll(text = True):
for temp in result.find('td', attrs = {'class': 'cdetail'}).find_all(text = True):
if 'days' in temp:
age = tryInt(temp.split(' ')[0])
break
# size
size = None
for temp in result.find('div', attrs = {'class': 'cdetail'}).findAll(text = True):
for temp in result.find('div', attrs = {'class': 'cdetail'}).find_all(text = True):
if 'gb' in temp.lower() or 'mb' in temp.lower() or 'kb' in temp.lower():
size = self.parseSize(temp)
break
@@ -74,7 +74,7 @@ class Mysterbin(NZBProvider):
new = {
'id': myster_id,
'name': ''.join(result.find('span', attrs = {'class': 'cname'}).findAll(text = True)),
'name': ''.join(result.find('span', attrs = {'class': 'cname'}).find_all(text = True)),
'type': 'nzb',
'provider': self.getName(),
'age': age,
couchpotato/core/providers/nzb/newzbin/main.py
+5 -3
View File
@@ -57,6 +57,8 @@ class Newzbin(NZBProvider, RSS):
'category': '6',
'ps_rb_video_format': str(cat_id),
'ps_rb_source': str(format_id),
'u_post_larger_than': quality.get('size_min'),
'u_post_smaller_than': quality.get('size_max'),
})
url = "%s?%s" % (self.urls['search'], arguments)
@@ -80,7 +82,7 @@ class Newzbin(NZBProvider, RSS):
data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item')
except Exception, e:
log.debug('%s, %s' % (self.getName(), e))
log.debug('%s, %s', (self.getName(), e))
return results
for nzb in nzbs:
@@ -131,7 +133,7 @@ class Newzbin(NZBProvider, RSS):
def download(self, url = '', nzb_id = ''):
try:
log.info('Download nzb from newzbin, report id: %s ' % nzb_id)
log.info('Download nzb from newzbin, report id: %s ', nzb_id)
return self.urlopen(self.urls['download'], params = {
'username' : self.conf('username'),
@@ -139,7 +141,7 @@ class Newzbin(NZBProvider, RSS):
'reportid' : nzb_id
}, show_error = False)
except Exception, e:
log.error('Failed downloading from newzbin, check credit: %s' % e)
log.error('Failed downloading from newzbin, check credit: %s', e)
return False
def getFormatId(self, format):
couchpotato/core/providers/nzb/newznab/main.py
+6 -6
View File
@@ -112,7 +112,7 @@ class Newznab(NZBProvider, RSS):
data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item')
except Exception, e:
log.debug('%s, %s' % (self.getName(), e))
log.debug('%s, %s', (self.getName(), e))
return results
results = []
@@ -126,8 +126,8 @@ class Newznab(NZBProvider, RSS):
elif item.attrib.get('name') == 'usenetdate':
date = item.attrib.get('value')
if date is '': log.debug('Date not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title")))
if size is 0: log.debug('Size not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title")))
if date is '': log.debug('Date not parsed properly or not available for %s: %s', (host['host'], self.getTextElement(nzb, "title")))
if size is 0: log.debug('Size not parsed properly or not available for %s: %s', (host['host'], self.getTextElement(nzb, "title")))
id = self.getTextElement(nzb, "guid").split('/')[-1:].pop()
new = {
@@ -157,7 +157,7 @@ class Newznab(NZBProvider, RSS):
return results
except SyntaxError:
log.error('Failed to parse XML response from Newznab: %s' % host)
log.error('Failed to parse XML response from Newznab: %s', host)
return results
def getHosts(self):
@@ -216,9 +216,9 @@ class Newznab(NZBProvider, RSS):
response = e.read().lower()
if 'maximum api' in response or 'download limit' in response:
if not self.limits_reached.get(host):
log.error('Limit reached for newznab provider: %s' % host)
log.error('Limit reached for newznab provider: %s', host)
self.limits_reached[host] = time.time()
return 'try_next'
log.error('Failed download from %s' % (host, traceback.format_exc()))
log.error('Failed download from %s', (host, traceback.format_exc()))
raise
couchpotato/core/providers/nzb/nzbclub/main.py
+8 -7
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString
@@ -28,7 +28,7 @@ class NZBClub(NZBProvider, RSS):
if self.isDisabled():
return results
q = '"%s" %s %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
for ignored in Env.setting('ignored_words', 'searcher').split(','):
q = '%s -%s' % (q, ignored.strip())
@@ -49,7 +49,7 @@ class NZBClub(NZBProvider, RSS):
data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item')
except Exception, e:
log.debug('%s, %s' % (self.getName(), e))
log.debug('%s, %s', (self.getName(), e))
return results
for nzb in nzbs:
@@ -62,9 +62,10 @@ class NZBClub(NZBProvider, RSS):
def extra_check(item):
full_description = self.getCache('nzbclub.%s' % nzbclub_id, item['detail_url'], cache_timeout = 25920000)
if 'ARCHIVE inside ARCHIVE' in full_description:
log.info('Wrong: Seems to be passworded files: %s' % new['name'])
return False
for ignored in ['ARCHIVE inside ARCHIVE', 'Incomplete', 'repair impossible']:
if ignored in full_description:
log.info('Wrong: Seems to be passworded or corrupted files: %s', new['name'])
return False
return True
@@ -111,7 +112,7 @@ class NZBClub(NZBProvider, RSS):
full_description = self.getCache('nzbclub.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
if 'ARCHIVE inside ARCHIVE' in full_description:
log.info('Wrong: Seems to be passworded files: %s' % item['name'])
log.info('Wrong: Seems to be passworded files: %s', item['name'])
return False
return True
couchpotato/core/providers/nzb/nzbindex/main.py
+11 -3
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \
simplifyString
@@ -30,7 +30,7 @@ class NzbIndex(NZBProvider, RSS):
if self.isDisabled():
return results
q = '%s %s %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
arguments = tryUrlencode({
'q': q,
'age': Env.setting('retention', 'nzb'),
@@ -53,7 +53,7 @@ class NzbIndex(NZBProvider, RSS):
data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item')
except Exception, e:
log.debug('%s, %s' % (self.getName(), e))
log.debug('%s, %s', (self.getName(), e))
return results
for nzb in nzbs:
@@ -67,6 +67,13 @@ class NzbIndex(NZBProvider, RSS):
except:
description = ''
def extra_check(new):
if '#c20000' in new['description'].lower():
log.info('Wrong: Seems to be passworded: %s', new['name'])
return False
return True
new = {
'id': nzbindex_id,
'type': 'nzb',
@@ -79,6 +86,7 @@ class NzbIndex(NZBProvider, RSS):
'detail_url': enclosure['url'].replace('/download/', '/release/'),
'description': description,
'get_more_info': self.getMoreInfo,
'extra_check': extra_check,
'check_nzb': True,
}
couchpotato/core/providers/nzb/nzbmatrix/main.py
+4 -1
View File
@@ -58,7 +58,7 @@ class NZBMatrix(NZBProvider, RSS):
data = XMLTree.fromstring(data)
nzbs = self.getElements(data, 'channel/item')
except Exception, e:
log.debug('%s, %s' % (self.getName(), e))
log.debug('%s, %s', (self.getName(), e))
return results
for nzb in nzbs:
@@ -99,6 +99,9 @@ class NZBMatrix(NZBProvider, RSS):
return results
def download(self, url = '', nzb_id = ''):
return self.urlopen(url, headers = {'User-Agent': Env.getIdentifier()})
def getApiExt(self):
return '&username=%s&apikey=%s' % (self.conf('username'), self.conf('api_key'))
couchpotato/core/providers/torrent/kickasstorrents/main.py
+7 -7
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.variable import tryInt, getTitle
from couchpotato.core.logger import CPLog
@@ -47,14 +47,14 @@ class KickAssTorrents(TorrentProvider):
try:
html = BeautifulSoup(data)
resultdiv = html.find('div', attrs = {'class':'tabs'})
for result in resultdiv.findAll('div', recursive = False):
for result in resultdiv.find_all('div', recursive = False):
if result.get('id').lower() not in cat_ids:
continue
try:
try:
for temp in result.findAll('tr'):
for temp in result.find_all('tr'):
if temp['class'] is 'firstr' or not temp.get('id'):
continue
@@ -68,15 +68,15 @@ class KickAssTorrents(TorrentProvider):
}
nr = 0
for td in temp.findAll('td'):
for td in temp.find_all('td'):
column_name = table_order[nr]
if column_name:
if column_name is 'name':
link = td.find('div', {'class': 'torrentname'}).findAll('a')[1]
link = td.find('div', {'class': 'torrentname'}).find_all('a')[1]
new['id'] = temp.get('id')[-8:]
new['name'] = link.text
new['url'] = td.findAll('a', 'idownload')[1]['href']
new['url'] = td.find_all('a', 'idownload')[1]['href']
if new['url'][:2] == '//':
new['url'] = 'http:%s' % new['url']
new['score'] = 20 if td.find('a', 'iverif') else 0
@@ -99,7 +99,7 @@ class KickAssTorrents(TorrentProvider):
results.append(new)
self.found(new)
except:
log.error('Failed parsing KickAssTorrents: %s' % traceback.format_exc())
log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc())
except:
pass
couchpotato/core/providers/torrent/thepiratebay/main.py
+5 -5
View File
@@ -36,11 +36,11 @@ class ThePirateBay(TorrentProvider):
url = self.apiUrl % (quote_plus(self.toSearchString(movie.name + ' ' + quality) + self.makeIgnoreString(type)), self.getCatId(type))
log.info('Searching: %s' % url)
log.info('Searching: %s', url)
data = self.urlopen(url)
if not data:
log.error('Failed to get data from %s.' % url)
log.error('Failed to get data from %s.', url)
return results
try:
@@ -104,7 +104,7 @@ class ThePirateBay(TorrentProvider):
new.content = self.getInfo(new.detailUrl)
if self.isCorrectMovie(new, movie, type):
results.append(new)
log.info('Found: %s' % new.name)
log.info('Found: %s', new.name)
return results
@@ -127,11 +127,11 @@ class ThePirateBay(TorrentProvider):
def getInfo(self, url):
log.debug('Getting info: %s' % url)
log.debug('Getting info: %s', url)
data = self.urlopen(url)
if not data:
log.error('Failed to get data from %s.' % url)
log.error('Failed to get data from %s.', url)
return ''
div = SoupStrainer('div')
couchpotato/core/providers/trailer/hdtrailers/main.py
+8 -8
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import SoupStrainer, BeautifulSoup
from bs4 import SoupStrainer, BeautifulSoup
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import mergeDicts, getTitle
from couchpotato.core.logger import CPLog
@@ -51,13 +51,13 @@ class HDTrailers(TrailerProvider):
try:
tables = SoupStrainer('div')
html = BeautifulSoup(data, parseOnlyThese = tables)
result_table = html.findAll('h2', text = re.compile(movie_name))
html = BeautifulSoup(data, parse_only = tables)
result_table = html.find_all('h2', text = re.compile(movie_name))
for h2 in result_table:
if 'trailer' in h2.lower():
parent = h2.parent.parent.parent
trailerLinks = parent.findAll('a', text = re.compile('480p|720p|1080p'))
trailerLinks = parent.find_all('a', text = re.compile('480p|720p|1080p'))
try:
for trailer in trailerLinks:
results[trailer].insert(0, trailer.parent['href'])
@@ -74,11 +74,11 @@ class HDTrailers(TrailerProvider):
results = {'480p':[], '720p':[], '1080p':[]}
try:
tables = SoupStrainer('table')
html = BeautifulSoup(data, parseOnlyThese = tables)
html = BeautifulSoup(data, parse_only = tables)
result_table = html.find('table', attrs = {'class':'bottomTable'})
for tr in result_table.findAll('tr'):
for tr in result_table.find_all('tr'):
trtext = str(tr).lower()
if 'clips' in trtext:
break
@@ -86,7 +86,7 @@ class HDTrailers(TrailerProvider):
nr = 0
if 'trailer' not in tr.find('span', 'standardTrailerName').text.lower():
continue
resolutions = tr.findAll('td', attrs = {'class':'bottomTableResolution'})
resolutions = tr.find_all('td', attrs = {'class':'bottomTableResolution'})
for res in resolutions:
results[str(res.a.contents[0])].insert(0, res.a['href'])
nr += 1
@@ -94,7 +94,7 @@ class HDTrailers(TrailerProvider):
return results
except AttributeError:
log.debug('No trailers found in provider %s.' % provider)
log.debug('No trailers found in provider %s.', provider)
results['404'] = True
return results
couchpotato/core/providers/userscript/allocine/main.py
+1 -1
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup
from couchpotato.core.providers.userscript.base import UserscriptBase
class AlloCine(UserscriptBase):
couchpotato/core/providers/userscript/rottentomatoes/main.py
+1 -1
View File
@@ -1,4 +1,4 @@
from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.providers.userscript.base import UserscriptBase
couchpotato/core/providers/userscript/youteather/main.py
+3 -3
View File
@@ -5,8 +5,8 @@ class YouTheater(UserscriptBase):
id_re = re.compile("view\.php\?id=(\d+)")
includes = ['http://www.youtheater.com/view.php?id=*', 'http://youtheater.com/view.php?id=*',
'http://www.sratim.co.il/view.php?id=*', 'http://sratim.co.il/view.php?id=*']
def getMovie(self, url):
id = self.id_re.findall(url)[0]
url = "http://www.youtheater.com/view.php?id=%s" % id
return super(YouTheater, self).getMovie(url)
url = 'http://www.youtheater.com/view.php?id=%s' % id
return super(YouTheater, self).getMovie(url)
couchpotato/runner.py
+5 -5
View File
@@ -155,10 +155,10 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
import color_logs
from couchpotato.core.logger import CPLog
log = CPLog(__name__)
log.debug('Started with options %s' % options)
log.debug('Started with options %s', options)
def customwarn(message, category, filename, lineno, file = None, line = None):
log.warning('%s %s %s line:%s' % (category, message, filename, lineno))
log.warning('%s %s %s line:%s', (category, message, filename, lineno))
warnings.showwarning = customwarn
@@ -185,7 +185,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
current_db_version = db_version(db, repo)
if current_db_version < latest_db_version and not debug:
log.info('Doing database upgrade. From %d to %d' % (current_db_version, latest_db_version))
log.info('Doing database upgrade. From %d to %d', (current_db_version, latest_db_version))
upgrade(db, repo)
# Configure Database
@@ -220,7 +220,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
app.register_blueprint(api, url_prefix = '%s/api/%s/' % (url_base, api_key))
# Some logging and fire load event
try: log.info('Starting server on port %(port)s' % config)
try: log.info('Starting server on port %(port)s', config)
except: pass
fireEventAsync('app.load')
@@ -248,7 +248,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
try:
nr, msg = e
if nr == 48:
log.info('Already in use, try %s more time after few seconds' % restart_tries)
log.info('Already in use, try %s more time after few seconds', restart_tries)
time.sleep(1)
restart_tries -= 1
couchpotato/static/images/icon.trailer.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 596 B

couchpotato/static/scripts/page/manage.js
+1 -1
View File
@@ -41,7 +41,7 @@ Page.Manage = new Class({
Api.request('manage.update', {
'data': {
'full': full ? 1 : null
'full': +full
}
})
couchpotato/static/scripts/page/settings.js
+12 -7
View File
@@ -926,14 +926,19 @@ Option.Choice = new Class({
var mtches = []
if(matches)
matches.each(function(match, mnr){
var msplit = value.split(match);
msplit.each(function(matchsplit, snr){
if(msplit.length-1 == snr)
value = matchsplit;
mtches.append([value == matchsplit ? match : matchsplit]);
var pos = value.indexOf(match),
msplit = [value.substr(0, pos), value.substr(pos, match.length), value.substr(pos+match.length)];
if(matches.length*2 == mtches.length)
mtches.append([value]);
msplit.each(function(matchsplit, snr){
if(msplit.length-1 == snr){
value = matchsplit;
if(matches.length-1 == mnr)
mtches.append([value]);
return;
}
mtches.append([value == matchsplit ? match : matchsplit]);
});
});
couchpotato/static/scripts/page/wanted.js
+5 -4
View File
@@ -28,8 +28,9 @@ var MovieActions = {};
window.addEvent('domready', function(){
MovieActions.Wanted = {
'IMBD': IMDBAction
,'releases': ReleaseAction
'IMDB': IMDBAction
,'Trailer': TrailerAction
,'Releases': ReleaseAction
,'Edit': new Class({
@@ -236,12 +237,12 @@ window.addEvent('domready', function(){
};
MovieActions.Snatched = {
'IMBD': IMDBAction
'IMDB': IMDBAction
,'Delete': MovieActions.Wanted.Delete
};
MovieActions.Done = {
'IMBD': IMDBAction
'IMDB': IMDBAction
,'Edit': MovieActions.Wanted.Edit
,'Files': new Class({
couchpotato/static/style/main.css
+5 -1
View File
@@ -154,6 +154,7 @@ body > .spinner, .mask{
.icon.rating { background-image: url('../images/icon.rating.png'); }
.icon.files { background-image: url('../images/icon.files.png'); }
.icon.info { background-image: url('../images/icon.info.png'); }
.icon.trailer { background-image: url('../images/icon.trailer.png'); }
/*** Navigation ***/
.header {
@@ -166,6 +167,7 @@ body > .spinner, .mask{
z-index: 5;
box-shadow: 0 20px 30px -30px rgba(0,0,0,0.05);
transition: box-shadow .4s cubic-bezier(0.9,0,0.1,1);
transform: translateZ(0);
}
.header.with_shadow {
box-shadow: 0 20px 30px -30px rgba(0,0,0,0.3);
@@ -215,6 +217,7 @@ body > .spinner, .mask{
outline: none;
box-shadow: inset 0 1px 8px rgba(0,0,0,0.05), 0 1px 0px rgba(255,255,255,0.15);
transition: all .4s cubic-bezier(0.9,0,0.1,1);
transform: translateZ(0);
}
.header .navigation li:hover a:after { background-color: #047792; }
@@ -311,7 +314,7 @@ body > .spinner, .mask{
.header .message.update {
text-align: center;
position: relative;
top: -70px;
top: -100px;
padding: 2px 0;
background: #ff6134;
font-size: 12px;
@@ -522,6 +525,7 @@ body > .spinner, .mask{
width: 25px;
border: 1px solid rgba(0,0,0,0.3);
transition: all 0.3s ease-in-out;
transform: translateZ(0);
}
.more_menu.show > a:not(:active), .more_menu > a:hover:not(:active) {
background-color: #406db8;
couchpotato/static/style/page/settings.css
+2
View File
@@ -35,6 +35,7 @@
padding: 11px 15px;
font-weight: normal;
transition: all 0.1s ease-in-out;
transform: translateZ(0);
color: rgba(255, 255, 255, 0.8);
}
.page.settings .tabs a:hover, .page.settings .tabs .active a {
@@ -49,6 +50,7 @@
padding: 0;
overflow: hidden;
transition: all 1s ease-in-out;
transform: translateZ(0);
max-height: 0;
}
.page.settings .tabs > .active .subtabs {
couchpotato/templates/_desktop.html
+2
View File
@@ -45,6 +45,8 @@
<link href="{{ url_for('web.static', filename='images/favicon.ico') }}" rel="icon" type="image/x-icon" />
<link rel="apple-touch-icon" href="{{ url_for('web.static', filename='images/homescreen.png') }}" />
<script type="text/javascript" src="https://www.youtube.com/player_api" defer="defer"></script>
<script type="text/javascript">
window.addEvent('domready', function() {
libs/BeautifulSoup.py
-2015
View File
File diff suppressed because it is too large Load Diff
libs/bs4/__init__.py
+355
View File
@@ -0,0 +1,355 @@
"""Beautiful Soup
Elixir and Tonic
"The Screen-Scraper's Friend"
http://www.crummy.com/software/BeautifulSoup/
Beautiful Soup uses a pluggable XML or HTML parser to parse a
(possibly invalid) document into a tree representation. Beautiful Soup
provides provides methods and Pythonic idioms that make it easy to
navigate, search, and modify the parse tree.
Beautiful Soup works with Python 2.6 and up. It works better if lxml
and/or html5lib is installed.
For more than you ever wanted to know about Beautiful Soup, see the
documentation:
http://www.crummy.com/software/BeautifulSoup/bs4/doc/
"""
__author__ = "Leonard Richardson (leonardr@segfault.org)"
__version__ = "4.1.0"
__copyright__ = "Copyright (c) 2004-2012 Leonard Richardson"
__license__ = "MIT"
__all__ = ['BeautifulSoup']
import re
import warnings
from .builder import builder_registry
from .dammit import UnicodeDammit
from .element import (
CData,
Comment,
DEFAULT_OUTPUT_ENCODING,
Declaration,
Doctype,
NavigableString,
PageElement,
ProcessingInstruction,
ResultSet,
SoupStrainer,
Tag,
)
# The very first thing we do is give a useful error if someone is
# running this code under Python 3 without converting it.
syntax_error = u'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work. You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
class BeautifulSoup(Tag):
"""
This class defines the basic interface called by the tree builders.
These methods will be called by the parser:
reset()
feed(markup)
The tree builder may call these methods from its feed() implementation:
handle_starttag(name, attrs) # See note about return value
handle_endtag(name)
handle_data(data) # Appends to the current data node
endData(containerClass=NavigableString) # Ends the current data node
No matter how complicated the underlying parser is, you should be
able to build a tree using 'start tag' events, 'end tag' events,
'data' events, and "done with data" events.
If you encounter an empty-element tag (aka a self-closing tag,
like HTML's <br> tag), call handle_starttag and then
handle_endtag.
"""
ROOT_TAG_NAME = u'[document]'
# If the end-user gives no indication which tree builder they
# want, look for one with these features.
DEFAULT_BUILDER_FEATURES = ['html', 'fast']
# Used when determining whether a text node is all whitespace and
# can be replaced with a single space. A text node that contains
# fancy Unicode spaces (usually non-breaking) should be left
# alone.
STRIP_ASCII_SPACES = {9: None, 10: None, 12: None, 13: None, 32: None, }
def __init__(self, markup="", features=None, builder=None,
parse_only=None, from_encoding=None, **kwargs):
"""The Soup object is initialized as the 'root tag', and the
provided markup (which can be a string or a file-like object)
is fed into the underlying parser."""
if 'convertEntities' in kwargs:
warnings.warn(
"BS4 does not respect the convertEntities argument to the "
"BeautifulSoup constructor. Entities are always converted "
"to Unicode characters.")
if 'markupMassage' in kwargs:
del kwargs['markupMassage']
warnings.warn(
"BS4 does not respect the markupMassage argument to the "
"BeautifulSoup constructor. The tree builder is responsible "
"for any necessary markup massage.")
if 'smartQuotesTo' in kwargs:
del kwargs['smartQuotesTo']
warnings.warn(
"BS4 does not respect the smartQuotesTo argument to the "
"BeautifulSoup constructor. Smart quotes are always converted "
"to Unicode characters.")
if 'selfClosingTags' in kwargs:
del kwargs['selfClosingTags']
warnings.warn(
"BS4 does not respect the selfClosingTags argument to the "
"BeautifulSoup constructor. The tree builder is responsible "
"for understanding self-closing tags.")
if 'isHTML' in kwargs:
del kwargs['isHTML']
warnings.warn(
"BS4 does not respect the isHTML argument to the "
"BeautifulSoup constructor. You can pass in features='html' "
"or features='xml' to get a builder capable of handling "
"one or the other.")
def deprecated_argument(old_name, new_name):
if old_name in kwargs:
warnings.warn(
'The "%s" argument to the BeautifulSoup constructor '
'has been renamed to "%s."' % (old_name, new_name))
value = kwargs[old_name]
del kwargs[old_name]
return value
return None
parse_only = parse_only or deprecated_argument(
"parseOnlyThese", "parse_only")
from_encoding = from_encoding or deprecated_argument(
"fromEncoding", "from_encoding")
if len(kwargs) > 0:
arg = kwargs.keys().pop()
raise TypeError(
"__init__() got an unexpected keyword argument '%s'" % arg)
if builder is None:
if isinstance(features, basestring):
features = [features]
if features is None or len(features) == 0:
features = self.DEFAULT_BUILDER_FEATURES
builder_class = builder_registry.lookup(*features)
if builder_class is None:
raise ValueError(
"Couldn't find a tree builder with the features you "
"requested: %s. Do you need to install a parser library?"
% ",".join(features))
builder = builder_class()
self.builder = builder
self.is_xml = builder.is_xml
self.builder.soup = self
self.parse_only = parse_only
self.reset()
if hasattr(markup, 'read'): # It's a file-type object.
markup = markup.read()
(self.markup, self.original_encoding, self.declared_html_encoding,
self.contains_replacement_characters) = (
self.builder.prepare_markup(markup, from_encoding))
try:
self._feed()
except StopParsing:
pass
# Clear out the markup and remove the builder's circular
# reference to this object.
self.markup = None
self.builder.soup = None
def _feed(self):
# Convert the document to Unicode.
self.builder.reset()
self.builder.feed(self.markup)
# Close out any unfinished strings and close all the open tags.
self.endData()
while self.currentTag.name != self.ROOT_TAG_NAME:
self.popTag()
def reset(self):
Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
self.hidden = 1
self.builder.reset()
self.currentData = []
self.currentTag = None
self.tagStack = []
self.pushTag(self)
def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
"""Create a new tag associated with this soup."""
return Tag(None, self.builder, name, namespace, nsprefix, attrs)
def new_string(self, s):
"""Create a new NavigableString associated with this soup."""
navigable = NavigableString(s)
navigable.setup()
return navigable
def insert_before(self, successor):
raise ValueError("BeautifulSoup objects don't support insert_before().")
def insert_after(self, successor):
raise ValueError("BeautifulSoup objects don't support insert_after().")
def popTag(self):
tag = self.tagStack.pop()
#print "Pop", tag.name
if self.tagStack:
self.currentTag = self.tagStack[-1]
return self.currentTag
def pushTag(self, tag):
#print "Push", tag.name
if self.currentTag:
self.currentTag.contents.append(tag)
self.tagStack.append(tag)
self.currentTag = self.tagStack[-1]
def endData(self, containerClass=NavigableString):
if self.currentData:
currentData = u''.join(self.currentData)
if (currentData.translate(self.STRIP_ASCII_SPACES) == '' and
not set([tag.name for tag in self.tagStack]).intersection(
self.builder.preserve_whitespace_tags)):
if '\n' in currentData:
currentData = '\n'
else:
currentData = ' '
self.currentData = []
if self.parse_only and len(self.tagStack) <= 1 and \
(not self.parse_only.text or \
not self.parse_only.search(currentData)):
return
o = containerClass(currentData)
self.object_was_parsed(o)
def object_was_parsed(self, o):
"""Add an object to the parse tree."""
o.setup(self.currentTag, self.previous_element)
if self.previous_element:
self.previous_element.next_element = o
self.previous_element = o
self.currentTag.contents.append(o)
def _popToTag(self, name, nsprefix=None, inclusivePop=True):
"""Pops the tag stack up to and including the most recent
instance of the given tag. If inclusivePop is false, pops the tag
stack up to but *not* including the most recent instqance of
the given tag."""
#print "Popping to %s" % name
if name == self.ROOT_TAG_NAME:
return
numPops = 0
mostRecentTag = None
for i in range(len(self.tagStack) - 1, 0, -1):
if (name == self.tagStack[i].name
and nsprefix == self.tagStack[i].nsprefix == nsprefix):
numPops = len(self.tagStack) - i
break
if not inclusivePop:
numPops = numPops - 1
for i in range(0, numPops):
mostRecentTag = self.popTag()
return mostRecentTag
def handle_starttag(self, name, namespace, nsprefix, attrs):
"""Push a start tag on to the stack.
If this method returns None, the tag was rejected by the
SoupStrainer. You should proceed as if the tag had not occured
in the document. For instance, if this was a self-closing tag,
don't call handle_endtag.
"""
# print "Start tag %s: %s" % (name, attrs)
self.endData()
if (self.parse_only and len(self.tagStack) <= 1
and (self.parse_only.text
or not self.parse_only.search_tag(name, attrs))):
return None
tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
self.currentTag, self.previous_element)
if tag is None:
return tag
if self.previous_element:
self.previous_element.next_element = tag
self.previous_element = tag
self.pushTag(tag)
return tag
def handle_endtag(self, name, nsprefix=None):
#print "End tag: " + name
self.endData()
self._popToTag(name, nsprefix)
def handle_data(self, data):
self.currentData.append(data)
def decode(self, pretty_print=False,
eventual_encoding=DEFAULT_OUTPUT_ENCODING,
formatter="minimal"):
"""Returns a string or Unicode representation of this document.
To get Unicode, pass None for encoding."""
if self.is_xml:
# Print the XML declaration
encoding_part = ''
if eventual_encoding != None:
encoding_part = ' encoding="%s"' % eventual_encoding
prefix = u'<?xml version="1.0"%s?>\n' % encoding_part
else:
prefix = u''
if not pretty_print:
indent_level = None
else:
indent_level = 0
return prefix + super(BeautifulSoup, self).decode(
indent_level, eventual_encoding, formatter)
class BeautifulStoneSoup(BeautifulSoup):
"""Deprecated interface to an XML parser."""
def __init__(self, *args, **kwargs):
kwargs['features'] = 'xml'
warnings.warn(
'The BeautifulStoneSoup class is deprecated. Instead of using '
'it, pass features="xml" into the BeautifulSoup constructor.')
super(BeautifulStoneSoup, self).__init__(*args, **kwargs)
class StopParsing(Exception):
pass
#By default, act as an HTML pretty-printer.
if __name__ == '__main__':
import sys
soup = BeautifulSoup(sys.stdin)
print soup.prettify()
libs/bs4/builder/__init__.py
+307
View File
@@ -0,0 +1,307 @@
from collections import defaultdict
import itertools
import sys
from bs4.element import (
CharsetMetaAttributeValue,
ContentMetaAttributeValue,
whitespace_re
)
__all__ = [
'HTMLTreeBuilder',
'SAXTreeBuilder',
'TreeBuilder',
'TreeBuilderRegistry',
]
# Some useful features for a TreeBuilder to have.
FAST = 'fast'
PERMISSIVE = 'permissive'
STRICT = 'strict'
XML = 'xml'
HTML = 'html'
HTML_5 = 'html5'
class TreeBuilderRegistry(object):
def __init__(self):
self.builders_for_feature = defaultdict(list)
self.builders = []
def register(self, treebuilder_class):
"""Register a treebuilder based on its advertised features."""
for feature in treebuilder_class.features:
self.builders_for_feature[feature].insert(0, treebuilder_class)
self.builders.insert(0, treebuilder_class)
def lookup(self, *features):
if len(self.builders) == 0:
# There are no builders at all.
return None
if len(features) == 0:
# They didn't ask for any features. Give them the most
# recently registered builder.
return self.builders[0]
# Go down the list of features in order, and eliminate any builders
# that don't match every feature.
features = list(features)
features.reverse()
candidates = None
candidate_set = None
while len(features) > 0:
feature = features.pop()
we_have_the_feature = self.builders_for_feature.get(feature, [])
if len(we_have_the_feature) > 0:
if candidates is None:
candidates = we_have_the_feature
candidate_set = set(candidates)
else:
# Eliminate any candidates that don't have this feature.
candidate_set = candidate_set.intersection(
set(we_have_the_feature))
# The only valid candidates are the ones in candidate_set.
# Go through the original list of candidates and pick the first one
# that's in candidate_set.
if candidate_set is None:
return None
for candidate in candidates:
if candidate in candidate_set:
return candidate
return None
# The BeautifulSoup class will take feature lists from developers and use them
# to look up builders in this registry.
builder_registry = TreeBuilderRegistry()
class TreeBuilder(object):
"""Turn a document into a Beautiful Soup object tree."""
features = []
is_xml = False
preserve_whitespace_tags = set()
empty_element_tags = None # A tag will be considered an empty-element
# tag when and only when it has no contents.
# A value for these tag/attribute combinations is a space- or
# comma-separated list of CDATA, rather than a single CDATA.
cdata_list_attributes = {}
def __init__(self):
self.soup = None
def reset(self):
pass
def can_be_empty_element(self, tag_name):
"""Might a tag with this name be an empty-element tag?
The final markup may or may not actually present this tag as
self-closing.
For instance: an HTMLBuilder does not consider a <p> tag to be
an empty-element tag (it's not in
HTMLBuilder.empty_element_tags). This means an empty <p> tag
will be presented as "<p></p>", not "<p />".
The default implementation has no opinion about which tags are
empty-element tags, so a tag will be presented as an
empty-element tag if and only if it has no contents.
"<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will
be left alone.
"""
if self.empty_element_tags is None:
return True
return tag_name in self.empty_element_tags
def feed(self, markup):
raise NotImplementedError()
def prepare_markup(self, markup, user_specified_encoding=None,
document_declared_encoding=None):
return markup, None, None, False
def test_fragment_to_document(self, fragment):
"""Wrap an HTML fragment to make it look like a document.
Different parsers do this differently. For instance, lxml
introduces an empty <head> tag, and html5lib
doesn't. Abstracting this away lets us write simple tests
which run HTML fragments through the parser and compare the
results against other HTML fragments.
This method should not be used outside of tests.
"""
return fragment
def set_up_substitutions(self, tag):
return False
def _replace_cdata_list_attribute_values(self, tag_name, attrs):
"""Replaces class="foo bar" with class=["foo", "bar"]
Modifies its input in place.
"""
if self.cdata_list_attributes:
universal = self.cdata_list_attributes.get('*', [])
tag_specific = self.cdata_list_attributes.get(
tag_name.lower(), [])
for cdata_list_attr in itertools.chain(universal, tag_specific):
if cdata_list_attr in dict(attrs):
# Basically, we have a "class" attribute whose
# value is a whitespace-separated list of CSS
# classes. Split it into a list.
value = attrs[cdata_list_attr]
values = whitespace_re.split(value)
attrs[cdata_list_attr] = values
return attrs
class SAXTreeBuilder(TreeBuilder):
"""A Beautiful Soup treebuilder that listens for SAX events."""
def feed(self, markup):
raise NotImplementedError()
def close(self):
pass
def startElement(self, name, attrs):
attrs = dict((key[1], value) for key, value in list(attrs.items()))
#print "Start %s, %r" % (name, attrs)
self.soup.handle_starttag(name, attrs)
def endElement(self, name):
#print "End %s" % name
self.soup.handle_endtag(name)
def startElementNS(self, nsTuple, nodeName, attrs):
# Throw away (ns, nodeName) for now.
self.startElement(nodeName, attrs)
def endElementNS(self, nsTuple, nodeName):
# Throw away (ns, nodeName) for now.
self.endElement(nodeName)
#handler.endElementNS((ns, node.nodeName), node.nodeName)
def startPrefixMapping(self, prefix, nodeValue):
# Ignore the prefix for now.
pass
def endPrefixMapping(self, prefix):
# Ignore the prefix for now.
# handler.endPrefixMapping(prefix)
pass
def characters(self, content):
self.soup.handle_data(content)
def startDocument(self):
pass
def endDocument(self):
pass
class HTMLTreeBuilder(TreeBuilder):
"""This TreeBuilder knows facts about HTML.
Such as which tags are empty-element tags.
"""
preserve_whitespace_tags = set(['pre', 'textarea'])
empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
'spacer', 'link', 'frame', 'base'])
# The HTML standard defines these attributes as containing a
# space-separated list of values, not a single value. That is,
# class="foo bar" means that the 'class' attribute has two values,
# 'foo' and 'bar', not the single value 'foo bar'. When we
# encounter one of these attributes, we will parse its value into
# a list of values if possible. Upon output, the list will be
# converted back into a string.
cdata_list_attributes = {
"*" : ['class', 'accesskey', 'dropzone'],
"a" : ['rel', 'rev'],
"link" : ['rel', 'rev'],
"td" : ["headers"],
"th" : ["headers"],
"td" : ["headers"],
"form" : ["accept-charset"],
"object" : ["archive"],
# These are HTML5 specific, as are *.accesskey and *.dropzone above.
"area" : ["rel"],
"icon" : ["sizes"],
"iframe" : ["sandbox"],
"output" : ["for"],
}
def set_up_substitutions(self, tag):
# We are only interested in <meta> tags
if tag.name != 'meta':
return False
http_equiv = tag.get('http-equiv')
content = tag.get('content')
charset = tag.get('charset')
# We are interested in <meta> tags that say what encoding the
# document was originally in. This means HTML 5-style <meta>
# tags that provide the "charset" attribute. It also means
# HTML 4-style <meta> tags that provide the "content"
# attribute and have "http-equiv" set to "content-type".
#
# In both cases we will replace the value of the appropriate
# attribute with a standin object that can take on any
# encoding.
meta_encoding = None
if charset is not None:
# HTML 5 style:
# <meta charset="utf8">
meta_encoding = charset
tag['charset'] = CharsetMetaAttributeValue(charset)
elif (content is not None and http_equiv is not None
and http_equiv.lower() == 'content-type'):
# HTML 4 style:
# <meta http-equiv="content-type" content="text/html; charset=utf8">
tag['content'] = ContentMetaAttributeValue(content)
return (meta_encoding is not None)
def register_treebuilders_from(module):
"""Copy TreeBuilders from the given module into this module."""
# I'm fairly sure this is not the best way to do this.
this_module = sys.modules['bs4.builder']
for name in module.__all__:
obj = getattr(module, name)
if issubclass(obj, TreeBuilder):
setattr(this_module, name, obj)
this_module.__all__.append(name)
# Register the builder while we're at it.
this_module.builder_registry.register(obj)
# Builders are registered in reverse order of priority, so that custom
# builder registrations will take precedence. In general, we want lxml
# to take precedence over html5lib, because it's faster. And we only
# want to use HTMLParser as a last result.
from . import _htmlparser
register_treebuilders_from(_htmlparser)
try:
from . import _html5lib
register_treebuilders_from(_html5lib)
except ImportError:
# They don't have html5lib installed.
pass
try:
from . import _lxml
register_treebuilders_from(_lxml)
except ImportError:
# They don't have lxml installed.
pass
libs/bs4/builder/_html5lib.py
+222
View File
@@ -0,0 +1,222 @@
__all__ = [
'HTML5TreeBuilder',
]
import warnings
from bs4.builder import (
PERMISSIVE,
HTML,
HTML_5,
HTMLTreeBuilder,
)
from bs4.element import NamespacedAttribute
import html5lib
from html5lib.constants import namespaces
from bs4.element import (
Comment,
Doctype,
NavigableString,
Tag,
)
class HTML5TreeBuilder(HTMLTreeBuilder):
"""Use html5lib to build a tree."""
features = ['html5lib', PERMISSIVE, HTML_5, HTML]
def prepare_markup(self, markup, user_specified_encoding):
# Store the user-specified encoding for use later on.
self.user_specified_encoding = user_specified_encoding
return markup, None, None, False
# These methods are defined by Beautiful Soup.
def feed(self, markup):
if self.soup.parse_only is not None:
warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
parser = html5lib.HTMLParser(tree=self.create_treebuilder)
doc = parser.parse(markup, encoding=self.user_specified_encoding)
# Set the character encoding detected by the tokenizer.
if isinstance(markup, unicode):
# We need to special-case this because html5lib sets
# charEncoding to UTF-8 if it gets Unicode input.
doc.original_encoding = None
else:
doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
def create_treebuilder(self, namespaceHTMLElements):
self.underlying_builder = TreeBuilderForHtml5lib(
self.soup, namespaceHTMLElements)
return self.underlying_builder
def test_fragment_to_document(self, fragment):
"""See `TreeBuilder`."""
return u'<html><head></head><body>%s</body></html>' % fragment
class TreeBuilderForHtml5lib(html5lib.treebuilders._base.TreeBuilder):
def __init__(self, soup, namespaceHTMLElements):
self.soup = soup
super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
def documentClass(self):
self.soup.reset()
return Element(self.soup, self.soup, None)
def insertDoctype(self, token):
name = token["name"]
publicId = token["publicId"]
systemId = token["systemId"]
doctype = Doctype.for_name_and_ids(name, publicId, systemId)
self.soup.object_was_parsed(doctype)
def elementClass(self, name, namespace):
tag = self.soup.new_tag(name, namespace)
return Element(tag, self.soup, namespace)
def commentClass(self, data):
return TextNode(Comment(data), self.soup)
def fragmentClass(self):
self.soup = BeautifulSoup("")
self.soup.name = "[document_fragment]"
return Element(self.soup, self.soup, None)
def appendChild(self, node):
# XXX This code is not covered by the BS4 tests.
self.soup.append(node.element)
def getDocument(self):
return self.soup
def getFragment(self):
return html5lib.treebuilders._base.TreeBuilder.getFragment(self).element
class AttrList(object):
def __init__(self, element):
self.element = element
self.attrs = dict(self.element.attrs)
def __iter__(self):
return list(self.attrs.items()).__iter__()
def __setitem__(self, name, value):
"set attr", name, value
self.element[name] = value
def items(self):
return list(self.attrs.items())
def keys(self):
return list(self.attrs.keys())
def __len__(self):
return len(self.attrs)
def __getitem__(self, name):
return self.attrs[name]
def __contains__(self, name):
return name in list(self.attrs.keys())
class Element(html5lib.treebuilders._base.Node):
def __init__(self, element, soup, namespace):
html5lib.treebuilders._base.Node.__init__(self, element.name)
self.element = element
self.soup = soup
self.namespace = namespace
def appendChild(self, node):
if (node.element.__class__ == NavigableString and self.element.contents
and self.element.contents[-1].__class__ == NavigableString):
# Concatenate new text onto old text node
# XXX This has O(n^2) performance, for input like
# "a</a>a</a>a</a>..."
old_element = self.element.contents[-1]
new_element = self.soup.new_string(old_element + node.element)
old_element.replace_with(new_element)
else:
self.element.append(node.element)
node.parent = self
def getAttributes(self):
return AttrList(self.element)
def setAttributes(self, attributes):
if attributes is not None and len(attributes) > 0:
converted_attributes = []
for name, value in list(attributes.items()):
if isinstance(name, tuple):
new_name = NamespacedAttribute(*name)
del attributes[name]
attributes[new_name] = value
self.soup.builder._replace_cdata_list_attribute_values(
self.name, attributes)
for name, value in attributes.items():
self.element[name] = value
# The attributes may contain variables that need substitution.
# Call set_up_substitutions manually.
#
# The Tag constructor called this method when the Tag was created,
# but we just set/changed the attributes, so call it again.
self.soup.builder.set_up_substitutions(self.element)
attributes = property(getAttributes, setAttributes)
def insertText(self, data, insertBefore=None):
text = TextNode(self.soup.new_string(data), self.soup)
if insertBefore:
self.insertBefore(text, insertBefore)
else:
self.appendChild(text)
def insertBefore(self, node, refNode):
index = self.element.index(refNode.element)
if (node.element.__class__ == NavigableString and self.element.contents
and self.element.contents[index-1].__class__ == NavigableString):
# (See comments in appendChild)
old_node = self.element.contents[index-1]
new_str = self.soup.new_string(old_node + node.element)
old_node.replace_with(new_str)
else:
self.element.insert(index, node.element)
node.parent = self
def removeChild(self, node):
node.element.extract()
def reparentChildren(self, newParent):
while self.element.contents:
child = self.element.contents[0]
child.extract()
if isinstance(child, Tag):
newParent.appendChild(
Element(child, self.soup, namespaces["html"]))
else:
newParent.appendChild(
TextNode(child, self.soup))
def cloneNode(self):
tag = self.soup.new_tag(self.element.name, self.namespace)
node = Element(tag, self.soup, self.namespace)
for key,value in self.attributes:
node.attributes[key] = value
return node
def hasContent(self):
return self.element.contents
def getNameTuple(self):
if self.namespace == None:
return namespaces["html"], self.name
else:
return self.namespace, self.name
nameTuple = property(getNameTuple)
class TextNode(Element):
def __init__(self, element, soup):
html5lib.treebuilders._base.Node.__init__(self, None)
self.element = element
self.soup = soup
def cloneNode(self):
raise NotImplementedError
libs/bs4/builder/_htmlparser.py
+244
View File
@@ -0,0 +1,244 @@
"""Use the HTMLParser library to parse HTML files that aren't too bad."""
__all__ = [
'HTMLParserTreeBuilder',
]
from HTMLParser import (
HTMLParser,
HTMLParseError,
)
import sys
import warnings
# Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
# argument, which we'd like to set to False. Unfortunately,
# http://bugs.python.org/issue13273 makes strict=True a better bet
# before Python 3.2.3.
#
# At the end of this file, we monkeypatch HTMLParser so that
# strict=True works well on Python 3.2.2.
major, minor, release = sys.version_info[:3]
CONSTRUCTOR_TAKES_STRICT = (
major > 3
or (major == 3 and minor > 2)
or (major == 3 and minor == 2 and release >= 3))
from bs4.element import (
CData,
Comment,
Declaration,
Doctype,
ProcessingInstruction,
)
from bs4.dammit import EntitySubstitution, UnicodeDammit
from bs4.builder import (
HTML,
HTMLTreeBuilder,
STRICT,
)
HTMLPARSER = 'html.parser'
class BeautifulSoupHTMLParser(HTMLParser):
def handle_starttag(self, name, attrs):
# XXX namespace
self.soup.handle_starttag(name, None, None, dict(attrs))
def handle_endtag(self, name):
self.soup.handle_endtag(name)
def handle_data(self, data):
self.soup.handle_data(data)
def handle_charref(self, name):
# XXX workaround for a bug in HTMLParser. Remove this once
# it's fixed.
if name.startswith('x'):
real_name = int(name.lstrip('x'), 16)
else:
real_name = int(name)
try:
data = unichr(real_name)
except (ValueError, OverflowError), e:
data = u"\N{REPLACEMENT CHARACTER}"
self.handle_data(data)
def handle_entityref(self, name):
character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
if character is not None:
data = character
else:
data = "&%s;" % name
self.handle_data(data)
def handle_comment(self, data):
self.soup.endData()
self.soup.handle_data(data)
self.soup.endData(Comment)
def handle_decl(self, data):
self.soup.endData()
if data.startswith("DOCTYPE "):
data = data[len("DOCTYPE "):]
self.soup.handle_data(data)
self.soup.endData(Doctype)
def unknown_decl(self, data):
if data.upper().startswith('CDATA['):
cls = CData
data = data[len('CDATA['):]
else:
cls = Declaration
self.soup.endData()
self.soup.handle_data(data)
self.soup.endData(cls)
def handle_pi(self, data):
self.soup.endData()
if data.endswith("?") and data.lower().startswith("xml"):
# "An XHTML processing instruction using the trailing '?'
# will cause the '?' to be included in data." - HTMLParser
# docs.
#
# Strip the question mark so we don't end up with two
# question marks.
data = data[:-1]
self.soup.handle_data(data)
self.soup.endData(ProcessingInstruction)
class HTMLParserTreeBuilder(HTMLTreeBuilder):
is_xml = False
features = [HTML, STRICT, HTMLPARSER]
def __init__(self, *args, **kwargs):
if CONSTRUCTOR_TAKES_STRICT:
kwargs['strict'] = False
self.parser_args = (args, kwargs)
def prepare_markup(self, markup, user_specified_encoding=None,
document_declared_encoding=None):
"""
:return: A 4-tuple (markup, original encoding, encoding
declared within markup, whether any characters had to be
replaced with REPLACEMENT CHARACTER).
"""
if isinstance(markup, unicode):
return markup, None, None, False
try_encodings = [user_specified_encoding, document_declared_encoding]
dammit = UnicodeDammit(markup, try_encodings, is_html=True)
return (dammit.markup, dammit.original_encoding,
dammit.declared_html_encoding,
dammit.contains_replacement_characters)
def feed(self, markup):
args, kwargs = self.parser_args
parser = BeautifulSoupHTMLParser(*args, **kwargs)
parser.soup = self.soup
try:
parser.feed(markup)
except HTMLParseError, e:
warnings.warn(RuntimeWarning(
"Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
raise e
# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
# 3.2.3 code. This ensures they don't treat markup like <p></p> as a
# string.
#
# XXX This code can be removed once most Python 3 users are on 3.2.3.
if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
import re
attrfind_tolerant = re.compile(
r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant
locatestarttagend = re.compile(r"""
<[a-zA-Z][-.a-zA-Z0-9:_]* # tag name
(?:\s+ # whitespace before attribute name
(?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name
(?:\s*=\s* # value indicator
(?:'[^']*' # LITA-enclosed value
|\"[^\"]*\" # LIT-enclosed value
|[^'\">\s]+ # bare value
)
)?
)
)*
\s* # trailing whitespace
""", re.VERBOSE)
BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend
from html.parser import tagfind, attrfind
def parse_starttag(self, i):
self.__starttag_text = None
endpos = self.check_for_whole_start_tag(i)
if endpos < 0:
return endpos
rawdata = self.rawdata
self.__starttag_text = rawdata[i:endpos]
# Now parse the data between i+1 and j into a tag and attrs
attrs = []
match = tagfind.match(rawdata, i+1)
assert match, 'unexpected call to parse_starttag()'
k = match.end()
self.lasttag = tag = rawdata[i+1:k].lower()
while k < endpos:
if self.strict:
m = attrfind.match(rawdata, k)
else:
m = attrfind_tolerant.match(rawdata, k)
if not m:
break
attrname, rest, attrvalue = m.group(1, 2, 3)
if not rest:
attrvalue = None
elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
attrvalue[:1] == '"' == attrvalue[-1:]:
attrvalue = attrvalue[1:-1]
if attrvalue:
attrvalue = self.unescape(attrvalue)
attrs.append((attrname.lower(), attrvalue))
k = m.end()
end = rawdata[k:endpos].strip()
if end not in (">", "/>"):
lineno, offset = self.getpos()
if "\n" in self.__starttag_text:
lineno = lineno + self.__starttag_text.count("\n")
offset = len(self.__starttag_text) \
- self.__starttag_text.rfind("\n")
else:
offset = offset + len(self.__starttag_text)
if self.strict:
self.error("junk characters in start tag: %r"
% (rawdata[k:endpos][:20],))
self.handle_data(rawdata[i:endpos])
return endpos
if end.endswith('/>'):
# XHTML-style empty tag: <span attr="value" />
self.handle_startendtag(tag, attrs)
else:
self.handle_starttag(tag, attrs)
if tag in self.CDATA_CONTENT_ELEMENTS:
self.set_cdata_mode(tag)
return endpos
def set_cdata_mode(self, elem):
self.cdata_elem = elem.lower()
self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
BeautifulSoupHTMLParser.parse_starttag = parse_starttag
BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode
CONSTRUCTOR_TAKES_STRICT = True
libs/bs4/builder/_lxml.py
+179
View File
@@ -0,0 +1,179 @@
__all__ = [
'LXMLTreeBuilderForXML',
'LXMLTreeBuilder',
]
from StringIO import StringIO
import collections
from lxml import etree
from bs4.element import Comment, Doctype, NamespacedAttribute
from bs4.builder import (
FAST,
HTML,
HTMLTreeBuilder,
PERMISSIVE,
TreeBuilder,
XML)
from bs4.dammit import UnicodeDammit
LXML = 'lxml'
class LXMLTreeBuilderForXML(TreeBuilder):
DEFAULT_PARSER_CLASS = etree.XMLParser
is_xml = True
# Well, it's permissive by XML parser standards.
features = [LXML, XML, FAST, PERMISSIVE]
CHUNK_SIZE = 512
@property
def default_parser(self):
# This can either return a parser object or a class, which
# will be instantiated with default arguments.
return etree.XMLParser(target=self, strip_cdata=False, recover=True)
def __init__(self, parser=None, empty_element_tags=None):
if empty_element_tags is not None:
self.empty_element_tags = set(empty_element_tags)
if parser is None:
# Use the default parser.
parser = self.default_parser
if isinstance(parser, collections.Callable):
# Instantiate the parser with default arguments
parser = parser(target=self, strip_cdata=False)
self.parser = parser
self.soup = None
self.nsmaps = None
def _getNsTag(self, tag):
# Split the namespace URL out of a fully-qualified lxml tag
# name. Copied from lxml's src/lxml/sax.py.
if tag[0] == '{':
return tuple(tag[1:].split('}', 1))
else:
return (None, tag)
def prepare_markup(self, markup, user_specified_encoding=None,
document_declared_encoding=None):
"""
:return: A 3-tuple (markup, original encoding, encoding
declared within markup).
"""
if isinstance(markup, unicode):
return markup, None, None, False
try_encodings = [user_specified_encoding, document_declared_encoding]
dammit = UnicodeDammit(markup, try_encodings, is_html=True)
return (dammit.markup, dammit.original_encoding,
dammit.declared_html_encoding,
dammit.contains_replacement_characters)
def feed(self, markup):
if isinstance(markup, basestring):
markup = StringIO(markup)
# Call feed() at least once, even if the markup is empty,
# or the parser won't be initialized.
data = markup.read(self.CHUNK_SIZE)
self.parser.feed(data)
while data != '':
# Now call feed() on the rest of the data, chunk by chunk.
data = markup.read(self.CHUNK_SIZE)
if data != '':
self.parser.feed(data)
self.parser.close()
def close(self):
self.nsmaps = None
def start(self, name, attrs, nsmap={}):
# Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
attrs = dict(attrs)
nsprefix = None
# Invert each namespace map as it comes in.
if len(nsmap) == 0 and self.nsmaps != None:
# There are no new namespaces for this tag, but namespaces
# are in play, so we need a separate tag stack to know
# when they end.
self.nsmaps.append(None)
elif len(nsmap) > 0:
# A new namespace mapping has come into play.
if self.nsmaps is None:
self.nsmaps = []
inverted_nsmap = dict((value, key) for key, value in nsmap.items())
self.nsmaps.append(inverted_nsmap)
# Also treat the namespace mapping as a set of attributes on the
# tag, so we can recreate it later.
attrs = attrs.copy()
for prefix, namespace in nsmap.items():
attribute = NamespacedAttribute(
"xmlns", prefix, "http://www.w3.org/2000/xmlns/")
attrs[attribute] = namespace
namespace, name = self._getNsTag(name)
if namespace is not None:
for inverted_nsmap in reversed(self.nsmaps):
if inverted_nsmap is not None and namespace in inverted_nsmap:
nsprefix = inverted_nsmap[namespace]
break
self.soup.handle_starttag(name, namespace, nsprefix, attrs)
def end(self, name):
self.soup.endData()
completed_tag = self.soup.tagStack[-1]
namespace, name = self._getNsTag(name)
nsprefix = None
if namespace is not None:
for inverted_nsmap in reversed(self.nsmaps):
if inverted_nsmap is not None and namespace in inverted_nsmap:
nsprefix = inverted_nsmap[namespace]
break
self.soup.handle_endtag(name, nsprefix)
if self.nsmaps != None:
# This tag, or one of its parents, introduced a namespace
# mapping, so pop it off the stack.
self.nsmaps.pop()
if len(self.nsmaps) == 0:
# Namespaces are no longer in play, so don't bother keeping
# track of the namespace stack.
self.nsmaps = None
def pi(self, target, data):
pass
def data(self, content):
self.soup.handle_data(content)
def doctype(self, name, pubid, system):
self.soup.endData()
doctype = Doctype.for_name_and_ids(name, pubid, system)
self.soup.object_was_parsed(doctype)
def comment(self, content):
"Handle comments as Comment objects."
self.soup.endData()
self.soup.handle_data(content)
self.soup.endData(Comment)
def test_fragment_to_document(self, fragment):
"""See `TreeBuilder`."""
return u'<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
features = [LXML, HTML, FAST, PERMISSIVE]
is_xml = False
@property
def default_parser(self):
return etree.HTMLParser
def feed(self, markup):
self.parser.feed(markup)
self.parser.close()
def test_fragment_to_document(self, fragment):
"""See `TreeBuilder`."""
return u'<html><body>%s</body></html>' % fragment
libs/bs4/dammit.py
+792
View File
@@ -0,0 +1,792 @@
# -*- coding: utf-8 -*-
"""Beautiful Soup bonus library: Unicode, Dammit
This class forces XML data into a standard format (usually to UTF-8 or
Unicode). It is heavily based on code from Mark Pilgrim's Universal
Feed Parser. It does not rewrite the XML or HTML to reflect a new
encoding; that's the tree builder's job.
"""
import codecs
from htmlentitydefs import codepoint2name
import re
import warnings
# Autodetects character encodings. Very useful.
# Download from http://chardet.feedparser.org/
# or 'apt-get install python-chardet'
# or 'easy_install chardet'
try:
import chardet
#import chardet.constants
#chardet.constants._debug = 1
except ImportError:
chardet = None
# Available from http://cjkpython.i18n.org/.
try:
import iconv_codec
except ImportError:
pass
xml_encoding_re = re.compile(
'^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
html_meta_re = re.compile(
'<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
class EntitySubstitution(object):
"""Substitute XML or HTML entities for the corresponding characters."""
def _populate_class_variables():
lookup = {}
reverse_lookup = {}
characters_for_re = []
for codepoint, name in list(codepoint2name.items()):
character = unichr(codepoint)
if codepoint != 34:
# There's no point in turning the quotation mark into
# &quot;, unless it happens within an attribute value, which
# is handled elsewhere.
characters_for_re.append(character)
lookup[character] = name
# But we do want to turn &quot; into the quotation mark.
reverse_lookup[name] = character
re_definition = "[%s]" % "".join(characters_for_re)
return lookup, reverse_lookup, re.compile(re_definition)
(CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
CHARACTER_TO_XML_ENTITY = {
"'": "apos",
'"': "quot",
"&": "amp",
"<": "lt",
">": "gt",
}
BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
"&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
")")
@classmethod
def _substitute_html_entity(cls, matchobj):
entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
return "&%s;" % entity
@classmethod
def _substitute_xml_entity(cls, matchobj):
"""Used with a regular expression to substitute the
appropriate XML entity for an XML special character."""
entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
return "&%s;" % entity
@classmethod
def quoted_attribute_value(self, value):
"""Make a value into a quoted XML attribute, possibly escaping it.
Most strings will be quoted using double quotes.
Bob's Bar -> "Bob's Bar"
If a string contains double quotes, it will be quoted using
single quotes.
Welcome to "my bar" -> 'Welcome to "my bar"'
If a string contains both single and double quotes, the
double quotes will be escaped, and the string will be quoted
using double quotes.
Welcome to "Bob's Bar" -> "Welcome to &quot;Bob's bar&quot;
"""
quote_with = '"'
if '"' in value:
if "'" in value:
# The string contains both single and double
# quotes. Turn the double quotes into
# entities. We quote the double quotes rather than
# the single quotes because the entity name is
# "&quot;" whether this is HTML or XML. If we
# quoted the single quotes, we'd have to decide
# between &apos; and &squot;.
replace_with = "&quot;"
value = value.replace('"', replace_with)
else:
# There are double quotes but no single quotes.
# We can use single quotes to quote the attribute.
quote_with = "'"
return quote_with + value + quote_with
@classmethod
def substitute_xml(cls, value, make_quoted_attribute=False):
"""Substitute XML entities for special XML characters.
:param value: A string to be substituted. The less-than sign will
become &lt;, the greater-than sign will become &gt;, and any
ampersands that are not part of an entity defition will
become &amp;.
:param make_quoted_attribute: If True, then the string will be
quoted, as befits an attribute value.
"""
# Escape angle brackets, and ampersands that aren't part of
# entities.
value = cls.BARE_AMPERSAND_OR_BRACKET.sub(
cls._substitute_xml_entity, value)
if make_quoted_attribute:
value = cls.quoted_attribute_value(value)
return value
@classmethod
def substitute_html(cls, s):
"""Replace certain Unicode characters with named HTML entities.
This differs from data.encode(encoding, 'xmlcharrefreplace')
in that the goal is to make the result more readable (to those
with ASCII displays) rather than to recover from
errors. There's absolutely nothing wrong with a UTF-8 string
containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
character with "&eacute;" will make it more readable to some
people.
"""
return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
cls._substitute_html_entity, s)
class UnicodeDammit:
"""A class for detecting the encoding of a *ML document and
converting it to a Unicode string. If the source encoding is
windows-1252, can replace MS smart quotes with their HTML or XML
equivalents."""
# This dictionary maps commonly seen values for "charset" in HTML
# meta tags to the corresponding Python codec names. It only covers
# values that aren't in Python's aliases and can't be determined
# by the heuristics in find_codec.
CHARSET_ALIASES = {"macintosh": "mac-roman",
"x-sjis": "shift-jis"}
ENCODINGS_WITH_SMART_QUOTES = [
"windows-1252",
"iso-8859-1",
"iso-8859-2",
]
def __init__(self, markup, override_encodings=[],
smart_quotes_to=None, is_html=False):
self.declared_html_encoding = None
self.smart_quotes_to = smart_quotes_to
self.tried_encodings = []
self.contains_replacement_characters = False
if markup == '' or isinstance(markup, unicode):
self.markup = markup
self.unicode_markup = unicode(markup)
self.original_encoding = None
return
new_markup, document_encoding, sniffed_encoding = \
self._detectEncoding(markup, is_html)
self.markup = new_markup
u = None
if new_markup != markup:
# _detectEncoding modified the markup, then converted it to
# Unicode and then to UTF-8. So convert it from UTF-8.
u = self._convert_from("utf8")
self.original_encoding = sniffed_encoding
if not u:
for proposed_encoding in (
override_encodings + [document_encoding, sniffed_encoding]):
if proposed_encoding is not None:
u = self._convert_from(proposed_encoding)
if u:
break
# If no luck and we have auto-detection library, try that:
if not u and chardet and not isinstance(self.markup, unicode):
u = self._convert_from(chardet.detect(self.markup)['encoding'])
# As a last resort, try utf-8 and windows-1252:
if not u:
for proposed_encoding in ("utf-8", "windows-1252"):
u = self._convert_from(proposed_encoding)
if u:
break
# As an absolute last resort, try the encodings again with
# character replacement.
if not u:
for proposed_encoding in (
override_encodings + [
document_encoding, sniffed_encoding, "utf-8", "windows-1252"]):
if proposed_encoding != "ascii":
u = self._convert_from(proposed_encoding, "replace")
if u is not None:
warnings.warn(
UnicodeWarning(
"Some characters could not be decoded, and were "
"replaced with REPLACEMENT CHARACTER."))
self.contains_replacement_characters = True
break
# We could at this point force it to ASCII, but that would
# destroy so much data that I think giving up is better
self.unicode_markup = u
if not u:
self.original_encoding = None
def _sub_ms_char(self, match):
"""Changes a MS smart quote character to an XML or HTML
entity, or an ASCII character."""
orig = match.group(1)
if self.smart_quotes_to == 'ascii':
sub = self.MS_CHARS_TO_ASCII.get(orig).encode()
else:
sub = self.MS_CHARS.get(orig)
if type(sub) == tuple:
if self.smart_quotes_to == 'xml':
sub = '&#x'.encode() + sub[1].encode() + ';'.encode()
else:
sub = '&'.encode() + sub[0].encode() + ';'.encode()
else:
sub = sub.encode()
return sub
def _convert_from(self, proposed, errors="strict"):
proposed = self.find_codec(proposed)
if not proposed or (proposed, errors) in self.tried_encodings:
return None
self.tried_encodings.append((proposed, errors))
markup = self.markup
# Convert smart quotes to HTML if coming from an encoding
# that might have them.
if (self.smart_quotes_to is not None
and proposed.lower() in self.ENCODINGS_WITH_SMART_QUOTES):
smart_quotes_re = b"([\x80-\x9f])"
smart_quotes_compiled = re.compile(smart_quotes_re)
markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
try:
#print "Trying to convert document to %s (errors=%s)" % (
# proposed, errors)
u = self._to_unicode(markup, proposed, errors)
self.markup = u
self.original_encoding = proposed
except Exception as e:
#print "That didn't work!"
#print e
return None
#print "Correct encoding: %s" % proposed
return self.markup
def _to_unicode(self, data, encoding, errors="strict"):
'''Given a string and its encoding, decodes the string into Unicode.
%encoding is a string recognized by encodings.aliases'''
# strip Byte Order Mark (if present)
if (len(data) >= 4) and (data[:2] == '\xfe\xff') \
and (data[2:4] != '\x00\x00'):
encoding = 'utf-16be'
data = data[2:]
elif (len(data) >= 4) and (data[:2] == '\xff\xfe') \
and (data[2:4] != '\x00\x00'):
encoding = 'utf-16le'
data = data[2:]
elif data[:3] == '\xef\xbb\xbf':
encoding = 'utf-8'
data = data[3:]
elif data[:4] == '\x00\x00\xfe\xff':
encoding = 'utf-32be'
data = data[4:]
elif data[:4] == '\xff\xfe\x00\x00':
encoding = 'utf-32le'
data = data[4:]
newdata = unicode(data, encoding, errors)
return newdata
def _detectEncoding(self, xml_data, is_html=False):
"""Given a document, tries to detect its XML encoding."""
xml_encoding = sniffed_xml_encoding = None
try:
if xml_data[:4] == b'\x4c\x6f\xa7\x94':
# EBCDIC
xml_data = self._ebcdic_to_ascii(xml_data)
elif xml_data[:4] == b'\x00\x3c\x00\x3f':
# UTF-16BE
sniffed_xml_encoding = 'utf-16be'
xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xfe\xff') \
and (xml_data[2:4] != b'\x00\x00'):
# UTF-16BE with BOM
sniffed_xml_encoding = 'utf-16be'
xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
elif xml_data[:4] == b'\x3c\x00\x3f\x00':
# UTF-16LE
sniffed_xml_encoding = 'utf-16le'
xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
elif (len(xml_data) >= 4) and (xml_data[:2] == b'\xff\xfe') and \
(xml_data[2:4] != b'\x00\x00'):
# UTF-16LE with BOM
sniffed_xml_encoding = 'utf-16le'
xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
elif xml_data[:4] == b'\x00\x00\x00\x3c':
# UTF-32BE
sniffed_xml_encoding = 'utf-32be'
xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
elif xml_data[:4] == b'\x3c\x00\x00\x00':
# UTF-32LE
sniffed_xml_encoding = 'utf-32le'
xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
elif xml_data[:4] == b'\x00\x00\xfe\xff':
# UTF-32BE with BOM
sniffed_xml_encoding = 'utf-32be'
xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
elif xml_data[:4] == b'\xff\xfe\x00\x00':
# UTF-32LE with BOM
sniffed_xml_encoding = 'utf-32le'
xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
elif xml_data[:3] == b'\xef\xbb\xbf':
# UTF-8 with BOM
sniffed_xml_encoding = 'utf-8'
xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
else:
sniffed_xml_encoding = 'ascii'
pass
except:
xml_encoding_match = None
xml_encoding_match = xml_encoding_re.match(xml_data)
if not xml_encoding_match and is_html:
xml_encoding_match = html_meta_re.search(xml_data)
if xml_encoding_match is not None:
xml_encoding = xml_encoding_match.groups()[0].decode(
'ascii').lower()
if is_html:
self.declared_html_encoding = xml_encoding
if sniffed_xml_encoding and \
(xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
'iso-10646-ucs-4', 'ucs-4', 'csucs4',
'utf-16', 'utf-32', 'utf_16', 'utf_32',
'utf16', 'u16')):
xml_encoding = sniffed_xml_encoding
return xml_data, xml_encoding, sniffed_xml_encoding
def find_codec(self, charset):
return self._codec(self.CHARSET_ALIASES.get(charset, charset)) \
or (charset and self._codec(charset.replace("-", ""))) \
or (charset and self._codec(charset.replace("-", "_"))) \
or charset
def _codec(self, charset):
if not charset:
return charset
codec = None
try:
codecs.lookup(charset)
codec = charset
except (LookupError, ValueError):
pass
return codec
EBCDIC_TO_ASCII_MAP = None
def _ebcdic_to_ascii(self, s):
c = self.__class__
if not c.EBCDIC_TO_ASCII_MAP:
emap = (0,1,2,3,156,9,134,127,151,141,142,11,12,13,14,15,
16,17,18,19,157,133,8,135,24,25,146,143,28,29,30,31,
128,129,130,131,132,10,23,27,136,137,138,139,140,5,6,7,
144,145,22,147,148,149,150,4,152,153,154,155,20,21,158,26,
32,160,161,162,163,164,165,166,167,168,91,46,60,40,43,33,
38,169,170,171,172,173,174,175,176,177,93,36,42,41,59,94,
45,47,178,179,180,181,182,183,184,185,124,44,37,95,62,63,
186,187,188,189,190,191,192,193,194,96,58,35,64,39,61,34,
195,97,98,99,100,101,102,103,104,105,196,197,198,199,200,
201,202,106,107,108,109,110,111,112,113,114,203,204,205,
206,207,208,209,126,115,116,117,118,119,120,121,122,210,
211,212,213,214,215,216,217,218,219,220,221,222,223,224,
225,226,227,228,229,230,231,123,65,66,67,68,69,70,71,72,
73,232,233,234,235,236,237,125,74,75,76,77,78,79,80,81,
82,238,239,240,241,242,243,92,159,83,84,85,86,87,88,89,
90,244,245,246,247,248,249,48,49,50,51,52,53,54,55,56,57,
250,251,252,253,254,255)
import string
c.EBCDIC_TO_ASCII_MAP = string.maketrans(
''.join(map(chr, list(range(256)))), ''.join(map(chr, emap)))
return s.translate(c.EBCDIC_TO_ASCII_MAP)
# A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
MS_CHARS = {b'\x80': ('euro', '20AC'),
b'\x81': ' ',
b'\x82': ('sbquo', '201A'),
b'\x83': ('fnof', '192'),
b'\x84': ('bdquo', '201E'),
b'\x85': ('hellip', '2026'),
b'\x86': ('dagger', '2020'),
b'\x87': ('Dagger', '2021'),
b'\x88': ('circ', '2C6'),
b'\x89': ('permil', '2030'),
b'\x8A': ('Scaron', '160'),
b'\x8B': ('lsaquo', '2039'),
b'\x8C': ('OElig', '152'),
b'\x8D': '?',
b'\x8E': ('#x17D', '17D'),
b'\x8F': '?',
b'\x90': '?',
b'\x91': ('lsquo', '2018'),
b'\x92': ('rsquo', '2019'),
b'\x93': ('ldquo', '201C'),
b'\x94': ('rdquo', '201D'),
b'\x95': ('bull', '2022'),
b'\x96': ('ndash', '2013'),
b'\x97': ('mdash', '2014'),
b'\x98': ('tilde', '2DC'),
b'\x99': ('trade', '2122'),
b'\x9a': ('scaron', '161'),
b'\x9b': ('rsaquo', '203A'),
b'\x9c': ('oelig', '153'),
b'\x9d': '?',
b'\x9e': ('#x17E', '17E'),
b'\x9f': ('Yuml', ''),}
# A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
# horrors like stripping diacritical marks to turn á into a, but also
# contains non-horrors like turning “ into ".
MS_CHARS_TO_ASCII = {
b'\x80' : 'EUR',
b'\x81' : ' ',
b'\x82' : ',',
b'\x83' : 'f',
b'\x84' : ',,',
b'\x85' : '...',
b'\x86' : '+',
b'\x87' : '++',
b'\x88' : '^',
b'\x89' : '%',
b'\x8a' : 'S',
b'\x8b' : '<',
b'\x8c' : 'OE',
b'\x8d' : '?',
b'\x8e' : 'Z',
b'\x8f' : '?',
b'\x90' : '?',
b'\x91' : "'",
b'\x92' : "'",
b'\x93' : '"',
b'\x94' : '"',
b'\x95' : '*',
b'\x96' : '-',
b'\x97' : '--',
b'\x98' : '~',
b'\x99' : '(TM)',
b'\x9a' : 's',
b'\x9b' : '>',
b'\x9c' : 'oe',
b'\x9d' : '?',
b'\x9e' : 'z',
b'\x9f' : 'Y',
b'\xa0' : ' ',
b'\xa1' : '!',
b'\xa2' : 'c',
b'\xa3' : 'GBP',
b'\xa4' : '$', #This approximation is especially parochial--this is the
#generic currency symbol.
b'\xa5' : 'YEN',
b'\xa6' : '|',
b'\xa7' : 'S',
b'\xa8' : '..',
b'\xa9' : '',
b'\xaa' : '(th)',
b'\xab' : '<<',
b'\xac' : '!',
b'\xad' : ' ',
b'\xae' : '(R)',
b'\xaf' : '-',
b'\xb0' : 'o',
b'\xb1' : '+-',
b'\xb2' : '2',
b'\xb3' : '3',
b'\xb4' : ("'", 'acute'),
b'\xb5' : 'u',
b'\xb6' : 'P',
b'\xb7' : '*',
b'\xb8' : ',',
b'\xb9' : '1',
b'\xba' : '(th)',
b'\xbb' : '>>',
b'\xbc' : '1/4',
b'\xbd' : '1/2',
b'\xbe' : '3/4',
b'\xbf' : '?',
b'\xc0' : 'A',
b'\xc1' : 'A',
b'\xc2' : 'A',
b'\xc3' : 'A',
b'\xc4' : 'A',
b'\xc5' : 'A',
b'\xc6' : 'AE',
b'\xc7' : 'C',
b'\xc8' : 'E',
b'\xc9' : 'E',
b'\xca' : 'E',
b'\xcb' : 'E',
b'\xcc' : 'I',
b'\xcd' : 'I',
b'\xce' : 'I',
b'\xcf' : 'I',
b'\xd0' : 'D',
b'\xd1' : 'N',
b'\xd2' : 'O',
b'\xd3' : 'O',
b'\xd4' : 'O',
b'\xd5' : 'O',
b'\xd6' : 'O',
b'\xd7' : '*',
b'\xd8' : 'O',
b'\xd9' : 'U',
b'\xda' : 'U',
b'\xdb' : 'U',
b'\xdc' : 'U',
b'\xdd' : 'Y',
b'\xde' : 'b',
b'\xdf' : 'B',
b'\xe0' : 'a',
b'\xe1' : 'a',
b'\xe2' : 'a',
b'\xe3' : 'a',
b'\xe4' : 'a',
b'\xe5' : 'a',
b'\xe6' : 'ae',
b'\xe7' : 'c',
b'\xe8' : 'e',
b'\xe9' : 'e',
b'\xea' : 'e',
b'\xeb' : 'e',
b'\xec' : 'i',
b'\xed' : 'i',
b'\xee' : 'i',
b'\xef' : 'i',
b'\xf0' : 'o',
b'\xf1' : 'n',
b'\xf2' : 'o',
b'\xf3' : 'o',
b'\xf4' : 'o',
b'\xf5' : 'o',
b'\xf6' : 'o',
b'\xf7' : '/',
b'\xf8' : 'o',
b'\xf9' : 'u',
b'\xfa' : 'u',
b'\xfb' : 'u',
b'\xfc' : 'u',
b'\xfd' : 'y',
b'\xfe' : 'b',
b'\xff' : 'y',
}
# A map used when removing rogue Windows-1252/ISO-8859-1
# characters in otherwise UTF-8 documents.
#
# Note that \x81, \x8d, \x8f, \x90, and \x9d are undefined in
# Windows-1252.
WINDOWS_1252_TO_UTF8 = {
0x80 : b'\xe2\x82\xac', # €
0x82 : b'\xe2\x80\x9a', # ‚
0x83 : b'\xc6\x92', # ƒ
0x84 : b'\xe2\x80\x9e', # „
0x85 : b'\xe2\x80\xa6', # …
0x86 : b'\xe2\x80\xa0', # †
0x87 : b'\xe2\x80\xa1', # ‡
0x88 : b'\xcb\x86', # ˆ
0x89 : b'\xe2\x80\xb0', # ‰
0x8a : b'\xc5\xa0', # Š
0x8b : b'\xe2\x80\xb9', # ‹
0x8c : b'\xc5\x92', # Œ
0x8e : b'\xc5\xbd', # Ž
0x91 : b'\xe2\x80\x98', # ‘
0x92 : b'\xe2\x80\x99', # ’
0x93 : b'\xe2\x80\x9c', # “
0x94 : b'\xe2\x80\x9d', # ”
0x95 : b'\xe2\x80\xa2', # •
0x96 : b'\xe2\x80\x93', # –
0x97 : b'\xe2\x80\x94', # —
0x98 : b'\xcb\x9c', # ˜
0x99 : b'\xe2\x84\xa2', # ™
0x9a : b'\xc5\xa1', # š
0x9b : b'\xe2\x80\xba', # ›
0x9c : b'\xc5\x93', # œ
0x9e : b'\xc5\xbe', # ž
0x9f : b'\xc5\xb8', # Ÿ
0xa0 : b'\xc2\xa0', #  
0xa1 : b'\xc2\xa1', # ¡
0xa2 : b'\xc2\xa2', # ¢
0xa3 : b'\xc2\xa3', # £
0xa4 : b'\xc2\xa4', # ¤
0xa5 : b'\xc2\xa5', # ¥
0xa6 : b'\xc2\xa6', # ¦
0xa7 : b'\xc2\xa7', # §
0xa8 : b'\xc2\xa8', # ¨
0xa9 : b'\xc2\xa9', # ©
0xaa : b'\xc2\xaa', # ª
0xab : b'\xc2\xab', # «
0xac : b'\xc2\xac', # ¬
0xad : b'\xc2\xad', # ­
0xae : b'\xc2\xae', # ®
0xaf : b'\xc2\xaf', # ¯
0xb0 : b'\xc2\xb0', # °
0xb1 : b'\xc2\xb1', # ±
0xb2 : b'\xc2\xb2', # ²
0xb3 : b'\xc2\xb3', # ³
0xb4 : b'\xc2\xb4', # ´
0xb5 : b'\xc2\xb5', # µ
0xb6 : b'\xc2\xb6', # ¶
0xb7 : b'\xc2\xb7', # ·
0xb8 : b'\xc2\xb8', # ¸
0xb9 : b'\xc2\xb9', # ¹
0xba : b'\xc2\xba', # º
0xbb : b'\xc2\xbb', # »
0xbc : b'\xc2\xbc', # ¼
0xbd : b'\xc2\xbd', # ½
0xbe : b'\xc2\xbe', # ¾
0xbf : b'\xc2\xbf', # ¿
0xc0 : b'\xc3\x80', # À
0xc1 : b'\xc3\x81', # Á
0xc2 : b'\xc3\x82', # Â
0xc3 : b'\xc3\x83', # Ã
0xc4 : b'\xc3\x84', # Ä
0xc5 : b'\xc3\x85', # Å
0xc6 : b'\xc3\x86', # Æ
0xc7 : b'\xc3\x87', # Ç
0xc8 : b'\xc3\x88', # È
0xc9 : b'\xc3\x89', # É
0xca : b'\xc3\x8a', # Ê
0xcb : b'\xc3\x8b', # Ë
0xcc : b'\xc3\x8c', # Ì
0xcd : b'\xc3\x8d', # Í
0xce : b'\xc3\x8e', # Î
0xcf : b'\xc3\x8f', # Ï
0xd0 : b'\xc3\x90', # Ð
0xd1 : b'\xc3\x91', # Ñ
0xd2 : b'\xc3\x92', # Ò
0xd3 : b'\xc3\x93', # Ó
0xd4 : b'\xc3\x94', # Ô
0xd5 : b'\xc3\x95', # Õ
0xd6 : b'\xc3\x96', # Ö
0xd7 : b'\xc3\x97', # ×
0xd8 : b'\xc3\x98', # Ø
0xd9 : b'\xc3\x99', # Ù
0xda : b'\xc3\x9a', # Ú
0xdb : b'\xc3\x9b', # Û
0xdc : b'\xc3\x9c', # Ü
0xdd : b'\xc3\x9d', # Ý
0xde : b'\xc3\x9e', # Þ
0xdf : b'\xc3\x9f', # ß
0xe0 : b'\xc3\xa0', # à
0xe1 : b'\xa1', # á
0xe2 : b'\xc3\xa2', # â
0xe3 : b'\xc3\xa3', # ã
0xe4 : b'\xc3\xa4', # ä
0xe5 : b'\xc3\xa5', # å
0xe6 : b'\xc3\xa6', # æ
0xe7 : b'\xc3\xa7', # ç
0xe8 : b'\xc3\xa8', # è
0xe9 : b'\xc3\xa9', # é
0xea : b'\xc3\xaa', # ê
0xeb : b'\xc3\xab', # ë
0xec : b'\xc3\xac', # ì
0xed : b'\xc3\xad', # í
0xee : b'\xc3\xae', # î
0xef : b'\xc3\xaf', # ï
0xf0 : b'\xc3\xb0', # ð
0xf1 : b'\xc3\xb1', # ñ
0xf2 : b'\xc3\xb2', # ò
0xf3 : b'\xc3\xb3', # ó
0xf4 : b'\xc3\xb4', # ô
0xf5 : b'\xc3\xb5', # õ
0xf6 : b'\xc3\xb6', # ö
0xf7 : b'\xc3\xb7', # ÷
0xf8 : b'\xc3\xb8', # ø
0xf9 : b'\xc3\xb9', # ù
0xfa : b'\xc3\xba', # ú
0xfb : b'\xc3\xbb', # û
0xfc : b'\xc3\xbc', # ü
0xfd : b'\xc3\xbd', # ý
0xfe : b'\xc3\xbe', # þ
}
MULTIBYTE_MARKERS_AND_SIZES = [
(0xc2, 0xdf, 2), # 2-byte characters start with a byte C2-DF
(0xe0, 0xef, 3), # 3-byte characters start with E0-EF
(0xf0, 0xf4, 4), # 4-byte characters start with F0-F4
]
FIRST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[0][0]
LAST_MULTIBYTE_MARKER = MULTIBYTE_MARKERS_AND_SIZES[-1][1]
@classmethod
def detwingle(cls, in_bytes, main_encoding="utf8",
embedded_encoding="windows-1252"):
"""Fix characters from one encoding embedded in some other encoding.
Currently the only situation supported is Windows-1252 (or its
subset ISO-8859-1), embedded in UTF-8.
The input must be a bytestring. If you've already converted
the document to Unicode, you're too late.
The output is a bytestring in which `embedded_encoding`
characters have been converted to their `main_encoding`
equivalents.
"""
if embedded_encoding.replace('_', '-').lower() not in (
'windows-1252', 'windows_1252'):
raise NotImplementedError(
"Windows-1252 and ISO-8859-1 are the only currently supported "
"embedded encodings.")
if main_encoding.lower() not in ('utf8', 'utf-8'):
raise NotImplementedError(
"UTF-8 is the only currently supported main encoding.")
byte_chunks = []
chunk_start = 0
pos = 0
while pos < len(in_bytes):
byte = in_bytes[pos]
if not isinstance(byte, int):
# Python 2.x
byte = ord(byte)
if (byte >= cls.FIRST_MULTIBYTE_MARKER
and byte <= cls.LAST_MULTIBYTE_MARKER):
# This is the start of a UTF-8 multibyte character. Skip
# to the end.
for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
if byte >= start and byte <= end:
pos += size
break
elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
# We found a Windows-1252 character!
# Save the string up to this point as a chunk.
byte_chunks.append(in_bytes[chunk_start:pos])
# Now translate the Windows-1252 character into UTF-8
# and add it as another, one-byte chunk.
byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
pos += 1
chunk_start = pos
else:
# Go on to the next character.
pos += 1
if chunk_start == 0:
# The string is unchanged.
return in_bytes
else:
# Store the final chunk.
byte_chunks.append(in_bytes[chunk_start:])
return b''.join(byte_chunks)
libs/bs4/element.py
+1347
View File
File diff suppressed because it is too large Load Diff
libs/bs4/testing.py
+515
View File
@@ -0,0 +1,515 @@
"""Helper classes for tests."""
import copy
import functools
import unittest
from unittest import TestCase
from bs4 import BeautifulSoup
from bs4.element import (
CharsetMetaAttributeValue,
Comment,
ContentMetaAttributeValue,
Doctype,
SoupStrainer,
)
from bs4.builder import HTMLParserTreeBuilder
default_builder = HTMLParserTreeBuilder
class SoupTest(unittest.TestCase):
@property
def default_builder(self):
return default_builder()
def soup(self, markup, **kwargs):
"""Build a Beautiful Soup object from markup."""
builder = kwargs.pop('builder', self.default_builder)
return BeautifulSoup(markup, builder=builder, **kwargs)
def document_for(self, markup):
"""Turn an HTML fragment into a document.
The details depend on the builder.
"""
return self.default_builder.test_fragment_to_document(markup)
def assertSoupEquals(self, to_parse, compare_parsed_to=None):
builder = self.default_builder
obj = BeautifulSoup(to_parse, builder=builder)
if compare_parsed_to is None:
compare_parsed_to = to_parse
self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
class HTMLTreeBuilderSmokeTest(object):
"""A basic test of a treebuilder's competence.
Any HTML treebuilder, present or future, should be able to pass
these tests. With invalid markup, there's room for interpretation,
and different parsers can handle it differently. But with the
markup in these tests, there's not much room for interpretation.
"""
def assertDoctypeHandled(self, doctype_fragment):
"""Assert that a given doctype string is handled correctly."""
doctype_str, soup = self._document_with_doctype(doctype_fragment)
# Make sure a Doctype object was created.
doctype = soup.contents[0]
self.assertEqual(doctype.__class__, Doctype)
self.assertEqual(doctype, doctype_fragment)
self.assertEqual(str(soup)[:len(doctype_str)], doctype_str)
# Make sure that the doctype was correctly associated with the
# parse tree and that the rest of the document parsed.
self.assertEqual(soup.p.contents[0], 'foo')
def _document_with_doctype(self, doctype_fragment):
"""Generate and parse a document with the given doctype."""
doctype = '<!DOCTYPE %s>' % doctype_fragment
markup = doctype + '\n<p>foo</p>'
soup = self.soup(markup)
return doctype, soup
def test_normal_doctypes(self):
"""Make sure normal, everyday HTML doctypes are handled correctly."""
self.assertDoctypeHandled("html")
self.assertDoctypeHandled(
'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"')
def test_public_doctype_with_url(self):
doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
self.assertDoctypeHandled(doctype)
def test_system_doctype(self):
self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"')
def test_namespaced_system_doctype(self):
# We can handle a namespaced doctype with a system ID.
self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"')
def test_namespaced_public_doctype(self):
# Test a namespaced doctype with a public id.
self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"')
def test_real_xhtml_document(self):
"""A real XHTML document should come out more or less the same as it went in."""
markup = b"""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><title>Hello.</title></head>
<body>Goodbye.</body>
</html>"""
soup = self.soup(markup)
self.assertEqual(
soup.encode("utf-8").replace(b"\n", b""),
markup.replace(b"\n", b""))
def test_deepcopy(self):
"""Make sure you can copy the tree builder.
This is important because the builder is part of a
BeautifulSoup object, and we want to be able to copy that.
"""
copy.deepcopy(self.default_builder)
def test_p_tag_is_never_empty_element(self):
"""A <p> tag is never designated as an empty-element tag.
Even if the markup shows it as an empty-element tag, it
shouldn't be presented that way.
"""
soup = self.soup("<p/>")
self.assertFalse(soup.p.is_empty_element)
self.assertEqual(str(soup.p), "<p></p>")
def test_unclosed_tags_get_closed(self):
"""A tag that's not closed by the end of the document should be closed.
This applies to all tags except empty-element tags.
"""
self.assertSoupEquals("<p>", "<p></p>")
self.assertSoupEquals("<b>", "<b></b>")
self.assertSoupEquals("<br>", "<br/>")
def test_br_is_always_empty_element_tag(self):
"""A <br> tag is designated as an empty-element tag.
Some parsers treat <br></br> as one <br/> tag, some parsers as
two tags, but it should always be an empty-element tag.
"""
soup = self.soup("<br></br>")
self.assertTrue(soup.br.is_empty_element)
self.assertEqual(str(soup.br), "<br/>")
def test_nested_formatting_elements(self):
self.assertSoupEquals("<em><em></em></em>")
def test_comment(self):
# Comments are represented as Comment objects.
markup = "<p>foo<!--foobar-->baz</p>"
self.assertSoupEquals(markup)
soup = self.soup(markup)
comment = soup.find(text="foobar")
self.assertEqual(comment.__class__, Comment)
def test_preserved_whitespace_in_pre_and_textarea(self):
"""Whitespace must be preserved in <pre> and <textarea> tags."""
self.assertSoupEquals("<pre> </pre>")
self.assertSoupEquals("<textarea> woo </textarea>")
def test_nested_inline_elements(self):
"""Inline elements can be nested indefinitely."""
b_tag = "<b>Inside a B tag</b>"
self.assertSoupEquals(b_tag)
nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>"
self.assertSoupEquals(nested_b_tag)
double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>"
self.assertSoupEquals(nested_b_tag)
def test_nested_block_level_elements(self):
"""Block elements can be nested."""
soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>')
blockquote = soup.blockquote
self.assertEqual(blockquote.p.b.string, 'Foo')
self.assertEqual(blockquote.b.string, 'Foo')
def test_correctly_nested_tables(self):
"""One table can go inside another one."""
markup = ('<table id="1">'
'<tr>'
"<td>Here's another table:"
'<table id="2">'
'<tr><td>foo</td></tr>'
'</table></td>')
self.assertSoupEquals(
markup,
'<table id="1"><tr><td>Here\'s another table:'
'<table id="2"><tr><td>foo</td></tr></table>'
'</td></tr></table>')
self.assertSoupEquals(
"<table><thead><tr><td>Foo</td></tr></thead>"
"<tbody><tr><td>Bar</td></tr></tbody>"
"<tfoot><tr><td>Baz</td></tr></tfoot></table>")
def test_angle_brackets_in_attribute_values_are_escaped(self):
self.assertSoupEquals('<a b="<a>"></a>', '<a b="&lt;a&gt;"></a>')
def test_entities_in_attributes_converted_to_unicode(self):
expect = u'<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
self.assertSoupEquals('<p id="pi&#241;ata"></p>', expect)
self.assertSoupEquals('<p id="pi&#xf1;ata"></p>', expect)
self.assertSoupEquals('<p id="pi&ntilde;ata"></p>', expect)
def test_entities_in_text_converted_to_unicode(self):
expect = u'<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
self.assertSoupEquals("<p>pi&#241;ata</p>", expect)
self.assertSoupEquals("<p>pi&#xf1;ata</p>", expect)
self.assertSoupEquals("<p>pi&ntilde;ata</p>", expect)
def test_quot_entity_converted_to_quotation_mark(self):
self.assertSoupEquals("<p>I said &quot;good day!&quot;</p>",
'<p>I said "good day!"</p>')
def test_out_of_range_entity(self):
expect = u"\N{REPLACEMENT CHARACTER}"
self.assertSoupEquals("&#10000000000000;", expect)
self.assertSoupEquals("&#x10000000000000;", expect)
self.assertSoupEquals("&#1000000000;", expect)
def test_basic_namespaces(self):
"""Parsers don't need to *understand* namespaces, but at the
very least they should not choke on namespaces or lose
data."""
markup = b'<html xmlns="http://www.w3.org/1999/xhtml" xmlns:mathml="http://www.w3.org/1998/Math/MathML" xmlns:svg="http://www.w3.org/2000/svg"><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill="red"></b></body></html>'
soup = self.soup(markup)
self.assertEqual(markup, soup.encode())
html = soup.html
self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
self.assertEqual(
'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
self.assertEqual(
'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])
def test_multivalued_attribute_value_becomes_list(self):
markup = b'<a class="foo bar">'
soup = self.soup(markup)
self.assertEqual(['foo', 'bar'], soup.a['class'])
#
# Generally speaking, tests below this point are more tests of
# Beautiful Soup than tests of the tree builders. But parsers are
# weird, so we run these tests separately for every tree builder
# to detect any differences between them.
#
def test_soupstrainer(self):
"""Parsers should be able to work with SoupStrainers."""
strainer = SoupStrainer("b")
soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>",
parse_only=strainer)
self.assertEqual(soup.decode(), "<b>bold</b>")
def test_single_quote_attribute_values_become_double_quotes(self):
self.assertSoupEquals("<foo attr='bar'></foo>",
'<foo attr="bar"></foo>')
def test_attribute_values_with_nested_quotes_are_left_alone(self):
text = """<foo attr='bar "brawls" happen'>a</foo>"""
self.assertSoupEquals(text)
def test_attribute_values_with_double_nested_quotes_get_quoted(self):
text = """<foo attr='bar "brawls" happen'>a</foo>"""
soup = self.soup(text)
soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
self.assertSoupEquals(
soup.foo.decode(),
"""<foo attr="Brawls happen at &quot;Bob\'s Bar&quot;">a</foo>""")
def test_ampersand_in_attribute_value_gets_escaped(self):
self.assertSoupEquals('<this is="really messed up & stuff"></this>',
'<this is="really messed up &amp; stuff"></this>')
self.assertSoupEquals(
'<a href="http://example.org?a=1&b=2;3">foo</a>',
'<a href="http://example.org?a=1&amp;b=2;3">foo</a>')
def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
self.assertSoupEquals('<a href="http://example.org?a=1&amp;b=2;3"></a>')
def test_entities_in_strings_converted_during_parsing(self):
# Both XML and HTML entities are converted to Unicode characters
# during parsing.
text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
self.assertSoupEquals(text, expected)
def test_smart_quotes_converted_on_the_way_in(self):
# Microsoft smart quotes are converted to Unicode characters during
# parsing.
quote = b"<p>\x91Foo\x92</p>"
soup = self.soup(quote)
self.assertEqual(
soup.p.string,
u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
def test_non_breaking_spaces_converted_on_the_way_in(self):
soup = self.soup("<a>&nbsp;&nbsp;</a>")
self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)
def test_entities_converted_on_the_way_out(self):
text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
expected = u"<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>".encode("utf-8")
soup = self.soup(text)
self.assertEqual(soup.p.encode("utf-8"), expected)
def test_real_iso_latin_document(self):
# Smoke test of interrelated functionality, using an
# easy-to-understand document.
# Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
unicode_html = u'<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
# That's because we're going to encode it into ISO-Latin-1, and use
# that to test.
iso_latin_html = unicode_html.encode("iso-8859-1")
# Parse the ISO-Latin-1 HTML.
soup = self.soup(iso_latin_html)
# Encode it to UTF-8.
result = soup.encode("utf-8")
# What do we expect the result to look like? Well, it would
# look like unicode_html, except that the META tag would say
# UTF-8 instead of ISO-Latin-1.
expected = unicode_html.replace("ISO-Latin-1", "utf-8")
# And, of course, it would be in UTF-8, not Unicode.
expected = expected.encode("utf-8")
# Ta-da!
self.assertEqual(result, expected)
def test_real_shift_jis_document(self):
# Smoke test to make sure the parser can handle a document in
# Shift-JIS encoding, without choking.
shift_jis_html = (
b'<html><head></head><body><pre>'
b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
b'</pre></body></html>')
unicode_html = shift_jis_html.decode("shift-jis")
soup = self.soup(unicode_html)
# Make sure the parse tree is correctly encoded to various
# encodings.
self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
def test_real_hebrew_document(self):
# A real-world test to make sure we can convert ISO-8859-9 (a
# Hebrew encoding) to UTF-8.
hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
soup = self.soup(
hebrew_document, from_encoding="iso8859-8")
self.assertEqual(soup.original_encoding, 'iso8859-8')
self.assertEqual(
soup.encode('utf-8'),
hebrew_document.decode("iso8859-8").encode("utf-8"))
def test_meta_tag_reflects_current_encoding(self):
# Here's the <meta> tag saying that a document is
# encoded in Shift-JIS.
meta_tag = ('<meta content="text/html; charset=x-sjis" '
'http-equiv="Content-type"/>')
# Here's a document incorporating that meta tag.
shift_jis_html = (
'<html><head>\n%s\n'
'<meta http-equiv="Content-language" content="ja"/>'
'</head><body>Shift-JIS markup goes here.') % meta_tag
soup = self.soup(shift_jis_html)
# Parse the document, and the charset is seemingly unaffected.
parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
content = parsed_meta['content']
self.assertEqual('text/html; charset=x-sjis', content)
# But that value is actually a ContentMetaAttributeValue object.
self.assertTrue(isinstance(content, ContentMetaAttributeValue))
# And it will take on a value that reflects its current
# encoding.
self.assertEqual('text/html; charset=utf8', content.encode("utf8"))
# For the rest of the story, see TestSubstitutions in
# test_tree.py.
def test_html5_style_meta_tag_reflects_current_encoding(self):
# Here's the <meta> tag saying that a document is
# encoded in Shift-JIS.
meta_tag = ('<meta id="encoding" charset="x-sjis" />')
# Here's a document incorporating that meta tag.
shift_jis_html = (
'<html><head>\n%s\n'
'<meta http-equiv="Content-language" content="ja"/>'
'</head><body>Shift-JIS markup goes here.') % meta_tag
soup = self.soup(shift_jis_html)
# Parse the document, and the charset is seemingly unaffected.
parsed_meta = soup.find('meta', id="encoding")
charset = parsed_meta['charset']
self.assertEqual('x-sjis', charset)
# But that value is actually a CharsetMetaAttributeValue object.
self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))
# And it will take on a value that reflects its current
# encoding.
self.assertEqual('utf8', charset.encode("utf8"))
def test_tag_with_no_attributes_can_have_attributes_added(self):
data = self.soup("<a>text</a>")
data.a['foo'] = 'bar'
self.assertEqual('<a foo="bar">text</a>', data.a.decode())
class XMLTreeBuilderSmokeTest(object):
def test_docstring_generated(self):
soup = self.soup("<root/>")
self.assertEqual(
soup.encode(), b'<?xml version="1.0" encoding="utf-8"?>\n<root/>')
def test_real_xhtml_document(self):
"""A real XHTML document should come out *exactly* the same as it went in."""
markup = b"""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><title>Hello.</title></head>
<body>Goodbye.</body>
</html>"""
soup = self.soup(markup)
self.assertEqual(
soup.encode("utf-8"), markup)
def test_docstring_includes_correct_encoding(self):
soup = self.soup("<root/>")
self.assertEqual(
soup.encode("latin1"),
b'<?xml version="1.0" encoding="latin1"?>\n<root/>')
def test_large_xml_document(self):
"""A large XML document should come out the same as it went in."""
markup = (b'<?xml version="1.0" encoding="utf-8"?>\n<root>'
+ b'0' * (2**12)
+ b'</root>')
soup = self.soup(markup)
self.assertEqual(soup.encode("utf-8"), markup)
def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
self.assertSoupEquals("<p>", "<p/>")
self.assertSoupEquals("<p>foo</p>")
def test_namespaces_are_preserved(self):
markup = '<root xmlns:a="http://example.com/" xmlns:b="http://example.net/"><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root>'
soup = self.soup(markup)
root = soup.root
self.assertEqual("http://example.com/", root['xmlns:a'])
self.assertEqual("http://example.net/", root['xmlns:b'])
class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
"""Smoke test for a tree builder that supports HTML5."""
def test_real_xhtml_document(self):
# Since XHTML is not HTML5, HTML5 parsers are not tested to handle
# XHTML documents in any particular way.
pass
def test_html_tags_have_namespace(self):
markup = "<a>"
soup = self.soup(markup)
self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)
def test_svg_tags_have_namespace(self):
markup = '<svg><circle/></svg>'
soup = self.soup(markup)
namespace = "http://www.w3.org/2000/svg"
self.assertEqual(namespace, soup.svg.namespace)
self.assertEqual(namespace, soup.circle.namespace)
def test_mathml_tags_have_namespace(self):
markup = '<math><msqrt>5</msqrt></math>'
soup = self.soup(markup)
namespace = 'http://www.w3.org/1998/Math/MathML'
self.assertEqual(namespace, soup.math.namespace)
self.assertEqual(namespace, soup.msqrt.namespace)
def skipIf(condition, reason):
def nothing(test, *args, **kwargs):
return None
def decorator(test_item):
if condition:
return nothing
else:
return test_item
return decorator
libs/certifi/__init__.py
+1
View File
@@ -0,0 +1 @@
from .core import where
libs/certifi/cacert.pem
+3338
View File
File diff suppressed because it is too large Load Diff
libs/certifi/core.py
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
ceritfi.py
~~~~~~~~~~
This module returns the installation location of cacert.pem.
"""
import os
def where():
f = os.path.split(__file__)[0]
return os.path.join(f, 'cacert.pem')
if __name__ == '__main__':
print(where())
libs/guessit/ISO-3166-1_utf8.txt
Regular → Executable
View File
libs/guessit/ISO-639-2_utf-8.txt
Regular → Executable
View File
libs/guessit/__init__.py
Regular → Executable
+1 -1
View File
@@ -18,7 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__version__ = '0.4'
__version__ = '0.5-dev'
__all__ = ['Guess', 'Language',
'guess_file_info', 'guess_video_info',
'guess_movie_info', 'guess_episode_info']
libs/guessit/country.py
Regular → Executable
+3 -1
View File
@@ -20,6 +20,7 @@
from __future__ import unicode_literals
from guessit import fileutils
from guessit.textutils import to_unicode
import logging
log = logging.getLogger(__name__)
@@ -66,7 +67,8 @@ class Country(object):
"""
def __init__(self, country, strict=False):
self.alpha3 = country_to_alpha3.get(country.lower())
country = to_unicode(country.strip().lower())
self.alpha3 = country_to_alpha3.get(country)
if self.alpha3 is None and strict:
msg = 'The given string "%s" could not be identified as a country'
libs/guessit/date.py
Regular → Executable
View File
libs/guessit/fileutils.py
Regular → Executable
+1 -6
View File
@@ -29,6 +29,7 @@ def split_path(path):
If the given path was an absolute path, the first element will always be:
- the '/' root folder on Unix systems
- the drive letter on Windows systems (eg: r'C:\')
- the mount point '\\' on Windows systems (eg: r'\\host\share')
>>> split_path('/usr/bin/smewt')
['/', 'usr', 'bin', 'smewt']
@@ -36,12 +37,6 @@ def split_path(path):
>>> split_path('relative_path/to/my_folder/')
['relative_path', 'to', 'my_folder']
>>> split_path(r'C:\Program Files\Smewt\smewt.exe')
['C:\\', 'Program Files', 'Smewt', 'smewt.exe']
>>> split_path(r'Documents and Settings\User\config\\')
['Documents and Settings', 'User', 'config']
"""
result = []
while True:
libs/guessit/guess.py
Regular → Executable
View File
libs/guessit/hash_ed2k.py
Regular → Executable
View File
libs/guessit/hash_mpc.py
Regular → Executable
View File

Some files were not shown because too many files have changed in this diff Show More