From 41e97bbef5042b21a9afff03d7beeb0c538066bb Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 8 Apr 2012 16:16:02 +0200 Subject: [PATCH 001/105] Directory selection would not hide --- couchpotato/static/scripts/page/settings.js | 4 ++-- couchpotato/static/style/page/settings.css | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/static/scripts/page/settings.js b/couchpotato/static/scripts/page/settings.js index 94643cc7..92cdf36b 100644 --- a/couchpotato/static/scripts/page/settings.js +++ b/couchpotato/static/scripts/page/settings.js @@ -574,7 +574,7 @@ Option.Directory = new Class({ self.el.adopt( self.createLabel(), - new Element('span.directory.inlay', { + self.directory_inlay = new Element('span.directory.inlay', { 'events': { 'click': self.showBrowser.bind(self) } @@ -664,7 +664,7 @@ Option.Directory = new Class({ } }) ) - ).inject(self.input, 'before'); + ).inject(self.directory_inlay, 'before'); new Form.Check(self.show_hidden); } diff --git a/couchpotato/static/style/page/settings.css b/couchpotato/static/style/page/settings.css index 464be778..4d157f58 100644 --- a/couchpotato/static/style/page/settings.css +++ b/couchpotato/static/style/page/settings.css @@ -208,7 +208,7 @@ z-index: 2; position: absolute; width: 450px; - margin: 28px 0 20px -4px; + margin: 28px 0 20px 18%; background: #5c697b; border-radius: 3px; box-shadow: 0 0 50px rgba(0,0,0,0.55); @@ -221,7 +221,7 @@ display: block; position: absolute; width: 0px; - margin: -6px 0 0 38%; + margin: -6px 0 0 22%; } .page .directory_list ul { From 5aceb949151bf6fb33b1000ee1184365648cc037 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 8 Apr 2012 17:10:54 +0200 Subject: [PATCH 002/105] Proper automation for trakt watchlist --- couchpotato/core/plugins/automation/__init__.py | 12 +++++++++--- couchpotato/core/plugins/automation/main.py | 2 ++ couchpotato/core/plugins/movie/main.py | 3 ++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/automation/__init__.py b/couchpotato/core/plugins/automation/__init__.py index 8f7d80b7..2eac7dac 100644 --- a/couchpotato/core/plugins/automation/__init__.py +++ b/couchpotato/core/plugins/automation/__init__.py @@ -16,21 +16,27 @@ config = [{ { 'name': 'year', 'default': 2011, - 'Label': 'Year', 'type': 'int', }, { 'name': 'votes', 'default': 1000, - 'Label': 'Votes', 'type': 'int', }, { 'name': 'rating', 'default': 6.0, - 'Label': 'Rating', 'type': 'float', }, + { + 'name': 'hour', + 'advanced': True, + 'default': 12, + 'label': 'Check every', + 'type': 'int', + 'unit': 'hours', + 'description': 'hours', + }, ], }, ], diff --git a/couchpotato/core/plugins/automation/main.py b/couchpotato/core/plugins/automation/main.py index 713b97ab..6acf26d8 100644 --- a/couchpotato/core/plugins/automation/main.py +++ b/couchpotato/core/plugins/automation/main.py @@ -10,6 +10,8 @@ class Automation(Plugin): def __init__(self): + fireEvent('schedule.interval', 'updater.check', self.addMovies, hours = self.conf('hour', default = 12)) + if not Env.get('dev'): addEvent('app.load', self.addMovies) diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 433ecaa9..354e64a6 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -264,7 +264,8 @@ class MoviePlugin(Plugin): if not m: m = Movie( library_id = library.get('id'), - profile_id = params.get('profile_id', default_profile.get('id')) + profile_id = params.get('profile_id', default_profile.get('id')), + status_id = status_active.get('id'), ) db.add(m) fireEvent('library.update', params.get('identifier'), default_title = params.get('title', '')) From de96d4e1fbc73487d809aad595466ed669242a72 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 8 Apr 2012 17:24:46 +0200 Subject: [PATCH 003/105] Remove Kinopolis as automation source --- .../automation/kinepolis/__init__.py | 31 -------------- .../providers/automation/kinepolis/main.py | 41 ------------------- 2 files changed, 72 deletions(-) delete mode 100644 couchpotato/core/providers/automation/kinepolis/__init__.py delete mode 100644 couchpotato/core/providers/automation/kinepolis/main.py diff --git a/couchpotato/core/providers/automation/kinepolis/__init__.py b/couchpotato/core/providers/automation/kinepolis/__init__.py deleted file mode 100644 index 3d4a156c..00000000 --- a/couchpotato/core/providers/automation/kinepolis/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -from .main import Kinepolis - -def start(): - return Kinepolis() - -config = [{ - 'name': 'kinepolis', - 'groups': [ - { - 'tab': 'automation', - 'name': 'kinepolis_automation', - 'label': 'Kinepolis', - 'description': 'from Kinepolis', - 'options': [ - { - 'name': 'automation_enabled', - 'default': False, - 'type': 'enabler', - }, - { - 'name': 'automation_use_requirements', - 'label': 'Use requirements', - 'description': 'Use the minimal requirements set above', - 'default': True, - 'advanced': True, - 'type': 'bool', - }, - ], - }, - ], -}] diff --git a/couchpotato/core/providers/automation/kinepolis/main.py b/couchpotato/core/providers/automation/kinepolis/main.py deleted file mode 100644 index 08e0c692..00000000 --- a/couchpotato/core/providers/automation/kinepolis/main.py +++ /dev/null @@ -1,41 +0,0 @@ -from couchpotato.core.helpers.rss import RSS -from couchpotato.core.logger import CPLog -from couchpotato.core.providers.automation.base import Automation -import xml.etree.ElementTree as XMLTree - -log = CPLog(__name__) - - -class Kinepolis(Automation, RSS): - - urls = { - 'top10': 'http://kinepolis.be/nl/top10-box-office/feed', - } - - - def getIMDBids(self): - - if self.isDisabled(): - return - - movies = [] - for key in self.urls: - url = self.urls[key] - cache_key = 'kinepolis.%s' % key - - rss_data = self.getCache(cache_key, url) - try: - items = self.getElements(XMLTree.fromstring(rss_data), 'channel/item') - except Exception, e: - log.debug('%s, %s' % (self.getName(), e)) - continue - - for item in items: - title = self.getTextElement(item, "title").lower() - result = self.search(title) - - if result: - if not self.conf('automation_use_requirements') or self.isMinimal(result): - movies.append(result) - - return movies From 2435f73fc8235b93ea21ad1850460fa1654c2eb9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 8 Apr 2012 17:31:33 +0200 Subject: [PATCH 004/105] Better setting descriptions for automations --- couchpotato/core/providers/automation/imdb/__init__.py | 2 +- couchpotato/core/providers/automation/trakt/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/providers/automation/imdb/__init__.py b/couchpotato/core/providers/automation/imdb/__init__.py index 86561e28..338a8b62 100644 --- a/couchpotato/core/providers/automation/imdb/__init__.py +++ b/couchpotato/core/providers/automation/imdb/__init__.py @@ -10,7 +10,7 @@ config = [{ 'tab': 'automation', 'name': 'imdb_automation', 'label': 'IMDB', - 'description': 'From any public IMDB watchlists', + 'description': 'From any public IMDB watchlists', 'options': [ { 'name': 'automation_enabled', diff --git a/couchpotato/core/providers/automation/trakt/__init__.py b/couchpotato/core/providers/automation/trakt/__init__.py index f1c30dfc..88ab454a 100644 --- a/couchpotato/core/providers/automation/trakt/__init__.py +++ b/couchpotato/core/providers/automation/trakt/__init__.py @@ -10,7 +10,7 @@ config = [{ 'tab': 'automation', 'name': 'trakt_automation', 'label': 'Trakt', - 'description': 'from watchlist', + 'description': 'import movies from your own watchlist', 'options': [ { 'name': 'automation_enabled', From 5321037f0807e6afdc6fe425fece306b0cbf049a Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 8 Apr 2012 17:31:54 +0200 Subject: [PATCH 005/105] Disable non working metadata for now --- couchpotato/core/providers/metadata/mediabrowser/__init__.py | 2 ++ couchpotato/core/providers/metadata/sonyps3/__init__.py | 2 ++ couchpotato/core/providers/metadata/wdtv/__init__.py | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/providers/metadata/mediabrowser/__init__.py b/couchpotato/core/providers/metadata/mediabrowser/__init__.py index c061ce3c..2e3cc202 100644 --- a/couchpotato/core/providers/metadata/mediabrowser/__init__.py +++ b/couchpotato/core/providers/metadata/mediabrowser/__init__.py @@ -3,6 +3,7 @@ from .main import MediaBrowser def start(): return MediaBrowser() +""" config = [{ 'name': 'mediabrowser', 'groups': [ @@ -37,3 +38,4 @@ config = [{ }, ], }] +""" diff --git a/couchpotato/core/providers/metadata/sonyps3/__init__.py b/couchpotato/core/providers/metadata/sonyps3/__init__.py index 002b8487..c6a7b49e 100644 --- a/couchpotato/core/providers/metadata/sonyps3/__init__.py +++ b/couchpotato/core/providers/metadata/sonyps3/__init__.py @@ -3,6 +3,7 @@ from .main import SonyPS3 def start(): return SonyPS3() +""" config = [{ 'name': 'sonyps3', 'groups': [ @@ -27,3 +28,4 @@ config = [{ }, ], }] +""" diff --git a/couchpotato/core/providers/metadata/wdtv/__init__.py b/couchpotato/core/providers/metadata/wdtv/__init__.py index b3dab6e7..9be6c750 100644 --- a/couchpotato/core/providers/metadata/wdtv/__init__.py +++ b/couchpotato/core/providers/metadata/wdtv/__init__.py @@ -2,7 +2,7 @@ from .main import WDTV def start(): return WDTV() - +""" config = [{ 'name': 'wdtv', 'groups': [ @@ -27,3 +27,4 @@ config = [{ }, ], }] +""" From 148e1940ac127e23c4cc4c229d1885eb243d4552 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 8 Apr 2012 17:35:39 +0200 Subject: [PATCH 006/105] Subliminal update --- libs/enzyme/__init__.py | 23 +- libs/enzyme/asf.py | 126 ++- libs/enzyme/core.py | 59 +- libs/enzyme/exceptions.py | 7 +- libs/enzyme/flv.py | 35 +- libs/enzyme/fourcc.py | 9 +- libs/enzyme/infos.py | 13 +- libs/enzyme/language.py | 11 +- libs/enzyme/mkv.py | 67 +- libs/enzyme/mp4.py | 130 ++- libs/enzyme/mpeg.py | 180 ++-- libs/enzyme/ogm.py | 60 +- libs/enzyme/real.py | 80 +- libs/enzyme/riff.py | 172 ++-- libs/enzyme/strutils.py | 10 +- libs/guessit/__init__.py | 59 +- libs/guessit/autodetect.py | 75 -- libs/guessit/date.py | 59 +- libs/guessit/filetype.py | 76 -- libs/guessit/fileutils.py | 21 +- libs/guessit/guess.py | 105 ++- libs/guessit/hash_ed2k.py | 15 +- libs/guessit/hash_mpc.py | 36 +- libs/guessit/language.py | 97 +- libs/guessit/matcher.py | 568 ++--------- libs/guessit/matchtree.py | 305 ++++-- libs/guessit/patterns.py | 68 +- libs/guessit/slogging.py | 10 +- libs/guessit/textutils.py | 58 +- libs/guessit/transfo/__init__.py | 100 ++ libs/guessit/transfo/guess_bonus_features.py | 60 ++ libs/guessit/transfo/guess_date.py | 37 + .../guess_episode_info_from_position.py | 142 +++ libs/guessit/transfo/guess_episodes_rexps.py | 42 + libs/guessit/transfo/guess_filetype.py | 113 +++ libs/guessit/transfo/guess_language.py | 47 + .../guess_movie_title_from_position.py | 171 ++++ libs/guessit/transfo/guess_properties.py | 37 + libs/guessit/transfo/guess_release_group.py | 44 + libs/guessit/transfo/guess_video_rexps.py | 48 + .../transfo/guess_weak_episodes_rexps.py | 56 ++ libs/guessit/transfo/guess_website.py | 38 + libs/guessit/transfo/guess_year.py | 37 + libs/guessit/transfo/post_process.py | 69 ++ libs/guessit/transfo/split_explicit_groups.py | 42 + libs/guessit/transfo/split_on_dash.py | 51 + libs/guessit/transfo/split_path_components.py | 35 + libs/requests/__init__.py | 5 +- libs/requests/api.py | 17 +- libs/requests/async.py | 38 +- libs/requests/auth.py | 6 +- libs/requests/compat.py | 5 +- libs/requests/defaults.py | 11 + libs/requests/exceptions.py | 9 +- libs/requests/models.py | 291 +++--- libs/requests/packages/oreos/monkeys.py | 5 +- libs/requests/packages/oreos/structures.py | 10 +- libs/requests/packages/urllib3/__init__.py | 18 +- .../packages/urllib3/connectionpool.py | 169 +--- libs/requests/packages/urllib3/exceptions.py | 21 +- libs/requests/packages/urllib3/filepost.py | 24 +- libs/requests/packages/urllib3/poolmanager.py | 14 +- libs/requests/packages/urllib3/request.py | 21 +- libs/requests/packages/urllib3/response.py | 22 +- libs/requests/packages/urllib3/util.py | 136 +++ libs/requests/sessions.py | 59 +- libs/requests/utils.py | 148 ++- libs/subliminal/__init__.py | 37 +- libs/subliminal/api.py | 100 ++ libs/subliminal/async.py | 141 +++ libs/subliminal/core.py | 500 ++++------ libs/subliminal/exceptions.py | 54 +- libs/subliminal/infos.py | 26 +- libs/subliminal/languages.py | 42 +- libs/subliminal/plugins.py | 890 ------------------ libs/subliminal/services/__init__.py | 214 +++++ libs/subliminal/services/bierdopje.py | 122 +++ libs/subliminal/services/opensubtitles.py | 143 +++ libs/subliminal/services/subswiki.py | 99 ++ libs/subliminal/services/subtitulos.py | 83 ++ libs/subliminal/services/thesubdb.py | 65 ++ libs/subliminal/subtitles.py | 106 ++- libs/subliminal/tasks.py | 51 +- libs/subliminal/utils.py | 65 +- libs/subliminal/videos.py | 325 ++++--- 85 files changed, 4375 insertions(+), 3420 deletions(-) delete mode 100644 libs/guessit/autodetect.py delete mode 100644 libs/guessit/filetype.py mode change 100644 => 100755 libs/guessit/patterns.py create mode 100644 libs/guessit/transfo/__init__.py create mode 100644 libs/guessit/transfo/guess_bonus_features.py create mode 100644 libs/guessit/transfo/guess_date.py create mode 100644 libs/guessit/transfo/guess_episode_info_from_position.py create mode 100644 libs/guessit/transfo/guess_episodes_rexps.py create mode 100644 libs/guessit/transfo/guess_filetype.py create mode 100644 libs/guessit/transfo/guess_language.py create mode 100644 libs/guessit/transfo/guess_movie_title_from_position.py create mode 100644 libs/guessit/transfo/guess_properties.py create mode 100644 libs/guessit/transfo/guess_release_group.py create mode 100644 libs/guessit/transfo/guess_video_rexps.py create mode 100644 libs/guessit/transfo/guess_weak_episodes_rexps.py create mode 100644 libs/guessit/transfo/guess_website.py create mode 100644 libs/guessit/transfo/guess_year.py create mode 100644 libs/guessit/transfo/post_process.py create mode 100644 libs/guessit/transfo/split_explicit_groups.py create mode 100644 libs/guessit/transfo/split_on_dash.py create mode 100644 libs/guessit/transfo/split_path_components.py create mode 100644 libs/requests/packages/urllib3/util.py create mode 100755 libs/subliminal/api.py create mode 100755 libs/subliminal/async.py delete mode 100755 libs/subliminal/plugins.py create mode 100755 libs/subliminal/services/__init__.py create mode 100755 libs/subliminal/services/bierdopje.py create mode 100755 libs/subliminal/services/opensubtitles.py create mode 100755 libs/subliminal/services/subswiki.py create mode 100755 libs/subliminal/services/subtitulos.py create mode 100755 libs/subliminal/services/thesubdb.py diff --git a/libs/enzyme/__init__.py b/libs/enzyme/__init__.py index 7bf572e9..c283dd59 100644 --- a/libs/enzyme/__init__.py +++ b/libs/enzyme/__init__.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2003-2006 Thomas Schueppel -# Copyright (C) 2003-2006 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Thomas Schueppel +# Copyright 2003-2006 Dirk Meyer # # This file is part of enzyme. # @@ -17,14 +17,13 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - +# along with enzyme. If not, see . import mimetypes import os import sys from exceptions import * + PARSERS = [('asf', ['video/asf'], ['asf', 'wmv', 'wma']), ('flv', ['video/flv'], ['flv']), ('mkv', ['video/x-matroska', 'application/mkv'], ['mkv', 'mka', 'webm']), @@ -32,10 +31,18 @@ PARSERS = [('asf', ['video/asf'], ['asf', 'wmv', 'wma']), ('mpeg', ['video/mpeg'], ['mpeg', 'mpg', 'mp4', 'ts']), ('ogm', ['application/ogg'], ['ogm', 'ogg', 'ogv']), ('real', ['video/real'], ['rm', 'ra', 'ram']), - ('riff', ['video/avi'], ['wav', 'avi'])] + ('riff', ['video/avi'], ['wav', 'avi']) +] def parse(path): + """Parse metadata of the given video + + :param string path: path to the video file to parse + :return: a parser corresponding to the video's mimetype or extension + :rtype: :class:`~enzyme.core.AVContainer` + + """ if not os.path.isfile(path): raise ValueError('Invalid path') extension = os.path.splitext(path)[1][1:] @@ -47,7 +54,7 @@ def parse(path): parser_mime = parser_name if extension in parser_extensions: parser_ext = parser_name - parser = parser_mime or parser_ext + parser = parser_mime or parser_ext if not parser: raise NoParserError() mod = __import__(parser, globals=globals(), locals=locals(), fromlist=[], level=-1) diff --git a/libs/enzyme/asf.py b/libs/enzyme/asf.py index e7cca1b2..a623b591 100644 --- a/libs/enzyme/asf.py +++ b/libs/enzyme/asf.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2003-2006 Thomas Schueppel -# Copyright (C) 2003-2006 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Thomas Schueppel +# Copyright 2003-2006 Dirk Meyer # # This file is part of enzyme. # @@ -17,30 +17,29 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# +# along with enzyme. If not, see . +from exceptions import ParseError +import core +import logging +import string +import struct + __all__ = ['Parser'] -import struct -import string -import logging -from exceptions import * -import core # get logging object log = logging.getLogger(__name__) def _guid(input): # Remove any '-' - s = string.join(string.split(input,'-'), '') + s = string.join(string.split(input, '-'), '') r = '' if len(s) != 32: return '' - x = '' - for i in range(0,16): - r+=chr(int(s[2*i:2*i+2],16)) - guid = struct.unpack('>IHHBB6s',r) + for i in range(0, 16): + r += chr(int(s[2 * i:2 * i + 2], 16)) + guid = struct.unpack('>IHHBB6s', r) return guid GUIDS = { @@ -93,8 +92,7 @@ GUIDS = { 'ASF_Web_Stream_Format' : _guid('DA1E6B13-8359-4050-B398-388E965BF00C'), 'ASF_No_Error_Correction' : _guid('20FB5700-5B55-11CF-A8FD-00805F5C442B'), - 'ASF_Audio_Spread' : _guid('BFC3CD50-618F-11CF-8BB2-00AA00B4E220'), - } + 'ASF_Audio_Spread' : _guid('BFC3CD50-618F-11CF-8BB2-00AA00B4E220')} class Asf(core.AVContainer): @@ -114,7 +112,7 @@ class Asf(core.AVContainer): raise ParseError() (guidstr, objsize, objnum, reserved1, \ - reserved2) = struct.unpack('<16sQIBB',h) + reserved2) = struct.unpack('<16sQIBB', h) guid = self._parseguid(guidstr) if (guid != GUIDS['ASF_Header_Object']): @@ -122,9 +120,9 @@ class Asf(core.AVContainer): if reserved1 != 0x01 or reserved2 != 0x02: raise ParseError() - log.debug("asf header size: %d / %d objects" % (objsize,objnum)) - header = file.read(objsize-30) - for i in range(0,objnum): + log.debug(u'Header size: %d / %d objects' % (objsize, objnum)) + header = file.read(objsize - 30) + for _ in range(0, objnum): h = self._getnextheader(header) header = header[h[1]:] @@ -148,19 +146,19 @@ class Asf(core.AVContainer): stream._appendtable('ASFMETADATA', metadata) - def _parseguid(self,string): + def _parseguid(self, string): return struct.unpack(' -# Copyright (C) 2003-2006 Thomas Schueppel -# Copyright (C) 2003-2006 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Thomas Schueppel +# Copyright 2003-2006 Dirk Meyer # # This file is part of enzyme. # @@ -17,21 +17,14 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - - +# along with enzyme. If not, see . import re import logging import fourcc import language -from exceptions import * from strutils import str_to_unicode, unicode_to_str UNPRINTABLE_KEYS = ['thumbnail', 'url', 'codec_private'] -EXTENSION_DEVICE = 'device' -EXTENSION_DIRECTORY = 'directory' -EXTENSION_STREAM = 'stream' MEDIACORE = ['title', 'caption', 'comment', 'size', 'type', 'subtype', 'timestamp', 'keywords', 'country', 'language', 'langcode', 'url', 'artist', 'mime', 'datetime', 'tags', 'hash'] @@ -52,8 +45,6 @@ log = logging.getLogger(__name__) class Media(object): - media = None - """ Media is the base class to all Media Metadata Containers. It defines the basic structures that handle metadata. Media and its derivates @@ -61,10 +52,11 @@ class Media(object): Specific derivates contain additional keys to the dublin core set that is defined in Media. """ + media = None _keys = MEDIACORE table_mapping = {} - def __init__(self, hash = None): + def __init__(self, hash=None): if hash is not None: # create Media based on dict for key, value in hash.items(): @@ -139,20 +131,21 @@ class Media(object): result += '| ' + re.sub(r'\n(.)', r'\n| \1', unicode(item)) # print tables - if log.level >= 10: - for name, table in self.tables.items(): - result += '+-- Table %s\n' % str(name) - for key, value in table.items(): - try: - value = unicode(value) - if len(value) > 50: - value = u'' % len(value) - except (UnicodeDecodeError, TypeError), e: - try: - value = u'' % len(value) - except AttributeError: - value = u'' - result += u'| | %s: %s\n' % (unicode(key), value) + #FIXME: WTH? +# if log.level >= 10: +# for name, table in self.tables.items(): +# result += '+-- Table %s\n' % str(name) +# for key, value in table.items(): +# try: +# value = unicode(value) +# if len(value) > 50: +# value = u'' % len(value) +# except (UnicodeDecodeError, TypeError): +# try: +# value = u'' % len(value) +# except AttributeError: +# value = u'' +# result += u'| | %s: %s\n' % (unicode(key), value) return result def __str__(self): @@ -253,7 +246,7 @@ class Media(object): """ return hasattr(self, key) - def get(self, attr, default = None): + def get(self, attr, default=None): """ Returns the given attribute. If the attribute is not set by the parser return 'default'. @@ -315,7 +308,7 @@ class Tag(object): Tag values are strings (for binary data), unicode objects, or datetime objects for tags that represent dates or times. """ - def __init__(self, value = None, langcode = 'und', binary = False): + def __init__(self, value=None, langcode='und', binary=False): super(Tag, self).__init__() self.value = value self.langcode = langcode @@ -363,7 +356,7 @@ class Tags(dict, Tag): The attribute RATING has a value (PG), but it also has a child tag COUNTRY that specifies the country code the rating belongs to. """ - def __init__(self, value = None, langcode = 'und', binary = False): + def __init__(self, value=None, langcode='und', binary=False): super(Tags, self).__init__() self.value = value self.langcode = langcode @@ -410,7 +403,7 @@ class Chapter(Media): """ _keys = ['enabled', 'name', 'pos', 'id'] - def __init__(self, name = None, pos = 0): + def __init__(self, name=None, pos=0): Media.__init__(self) self.name = name self.pos = pos @@ -424,7 +417,7 @@ class Subtitle(Media): _keys = ['enabled', 'default', 'langcode', 'language', 'trackno', 'title', 'id', 'codec'] - def __init__(self, language = None): + def __init__(self, language=None): Media.__init__(self) self.language = language diff --git a/libs/enzyme/exceptions.py b/libs/enzyme/exceptions.py index 403aa947..5bc4f7ac 100644 --- a/libs/enzyme/exceptions.py +++ b/libs/enzyme/exceptions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin +# Copyright 2011-2012 Antoine Bertin # # This file is part of enzyme. # @@ -15,10 +15,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - - +# along with enzyme. If not, see . class Error(Exception): pass diff --git a/libs/enzyme/flv.py b/libs/enzyme/flv.py index c2dd4067..61f34a78 100644 --- a/libs/enzyme/flv.py +++ b/libs/enzyme/flv.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2003-2006 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Dirk Meyer # # This file is part of enzyme. # @@ -16,16 +16,15 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -__all__ = ['Parser'] - -from exceptions import * +# along with enzyme. If not, see . +from exceptions import ParseError import core import logging import struct +__all__ = ['Parser'] + + # get logging object log = logging.getLogger(__name__) @@ -44,7 +43,7 @@ FLV_AUDIO_CODECID = (0x0001, 0x0002, 0x0055, 0x0001) # video flags FLV_VIDEO_CODECID_MASK = 0x0f -FLV_VIDEO_CODECID = ( 'FLV1', 'MSS1', 'VP60') # wild guess +FLV_VIDEO_CODECID = ('FLV1', 'MSS1', 'VP60') # wild guess FLV_DATA_TYPE_NUMBER = 0x00 FLV_DATA_TYPE_BOOL = 0x01 @@ -70,7 +69,7 @@ class FlashVideo(core.AVContainer): """ table_mapping = { 'FLVINFO' : FLVINFO } - def __init__(self,file): + def __init__(self, file): core.AVContainer.__init__(self) self.mime = 'video/flv' self.type = 'Flash Video' @@ -78,7 +77,7 @@ class FlashVideo(core.AVContainer): if len(data) < 13 or struct.unpack('>3sBBII', data)[0] != 'FLV': raise ParseError() - for i in range(10): + for _ in range(10): if self.audio and self.video: break data = file.read(11) @@ -116,13 +115,13 @@ class FlashVideo(core.AVContainer): file.seek(size - 1, 1) elif chunk[0] == FLV_TAG_TYPE_META: - log.info('metadata %s', str(chunk)) + log.info(u'metadata %r', str(chunk)) metadata = file.read(size) try: while metadata: length, value = self._parse_value(metadata) if isinstance(value, dict): - log.info('metadata: %s', value) + log.info(u'metadata: %r', value) if value.get('creator'): self.copyright = value.get('creator') if value.get('width'): @@ -139,7 +138,7 @@ class FlashVideo(core.AVContainer): except (IndexError, struct.error, TypeError): pass else: - log.info('unkown %s', str(chunk)) + log.info(u'unkown %r', str(chunk)) file.seek(size, 1) file.seek(4, 1) @@ -157,16 +156,16 @@ class FlashVideo(core.AVContainer): if ord(data[0]) == FLV_DATA_TYPE_STRING: length = (ord(data[1]) << 8) + ord(data[2]) - return length + 3, data[3:length+3] + return length + 3, data[3:length + 3] if ord(data[0]) == FLV_DATA_TYPE_ECMARRAY: init_length = len(data) num = struct.unpack('>I', data[1:5])[0] data = data[5:] result = {} - for i in range(num): + for _ in range(num): length = (ord(data[0]) << 8) + ord(data[1]) - key = data[2:length+2] + key = data[2:length + 2] data = data[length + 2:] length, value = self._parse_value(data) if not length: @@ -175,7 +174,7 @@ class FlashVideo(core.AVContainer): data = data[length:] return init_length - len(data), result - log.info('unknown code: %x. Stop metadata parser', ord(data[0])) + log.info(u'unknown code: %x. Stop metadata parser', ord(data[0])) return 0, None diff --git a/libs/enzyme/fourcc.py b/libs/enzyme/fourcc.py index 60cbb3c8..ac15b0b2 100644 --- a/libs/enzyme/fourcc.py +++ b/libs/enzyme/fourcc.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2003-2006 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Dirk Meyer # # This file is part of enzyme. # @@ -16,10 +16,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - - +# along with enzyme. If not, see . import string import re import struct diff --git a/libs/enzyme/infos.py b/libs/enzyme/infos.py index d6196682..a6f0bcc5 100644 --- a/libs/enzyme/infos.py +++ b/libs/enzyme/infos.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin +# Copyright 2011-2012 Antoine Bertin # # This file is part of enzyme. # @@ -15,12 +15,5 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -__title__ = 'enzyme' -__description__ = 'Video metadata parser' -__version__ = '0.1' -__author__ = 'Antoine Bertin' -__email__ = 'diaoulael@gmail.com' -__license__ = 'GPLv3' +# along with enzyme. If not, see . +__version__ = '0.2' diff --git a/libs/enzyme/language.py b/libs/enzyme/language.py index 9a52692a..3957f9d9 100644 --- a/libs/enzyme/language.py +++ b/libs/enzyme/language.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2003-2006 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Dirk Meyer # # This file is part of enzyme. # @@ -16,10 +16,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - - +# along with enzyme. If not, see . import re __all__ = ['resolve'] @@ -44,7 +41,7 @@ def resolve(code): if code in spec[:-1]: return code, spec[-1] - return code, u'Unknown (%s)' % code + return code, u'Unknown (%r)' % code # Parsed from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt diff --git a/libs/enzyme/mkv.py b/libs/enzyme/mkv.py index eac0a380..aba5325e 100644 --- a/libs/enzyme/mkv.py +++ b/libs/enzyme/mkv.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2003-2006 Thomas Schueppel -# Copyright (C) 2003-2006 Dirk Meyer -# Copyright (C) 2003-2006 Jason Tackaberry +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Thomas Schueppel +# Copyright 2003-2006 Dirk Meyer +# Copyright 2003-2006 Jason Tackaberry # # This file is part of enzyme. # @@ -18,18 +18,17 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - -__all__ = ['Parser'] - +# along with enzyme. If not, see . from datetime import datetime -from exceptions import * +from exceptions import ParseError from struct import unpack import core import logging import re +__all__ = ['Parser'] + + # get logging object log = logging.getLogger(__name__) @@ -236,7 +235,7 @@ class EbmlEntity: self.compute_id(inbuf) if self.id_len == 0: - log.error("EBML entity not found, bad file format") + log.error(u'EBML entity not found, bad file format') raise ParseError() self.entity_len, self.len_size = self.compute_len(inbuf[self.id_len:]) @@ -247,7 +246,7 @@ class EbmlEntity: # if the data size is 8 or less, it could be a numeric value self.value = 0 if self.entity_len <= 8: - for pos, shift in zip(range(self.entity_len), range((self.entity_len-1)*8, -1, -8)): + for pos, shift in zip(range(self.entity_len), range((self.entity_len - 1) * 8, -1, -8)): self.value |= ord(self.entity_data[pos]) << shift @@ -271,19 +270,19 @@ class EbmlEntity: if len(inbuf) < 2: return 0 self.id_len = 2 - self.entity_id = ord(inbuf[0])<<8 | ord(inbuf[1]) + self.entity_id = ord(inbuf[0]) << 8 | ord(inbuf[1]) elif first & 0x20: if len(inbuf) < 3: return 0 self.id_len = 3 - self.entity_id = (ord(inbuf[0])<<16) | (ord(inbuf[1])<<8) | \ + self.entity_id = (ord(inbuf[0]) << 16) | (ord(inbuf[1]) << 8) | \ (ord(inbuf[2])) elif first & 0x10: if len(inbuf) < 4: return 0 self.id_len = 4 - self.entity_id = (ord(inbuf[0])<<24) | (ord(inbuf[1])<<16) | \ - (ord(inbuf[2])<<8) | (ord(inbuf[3])) + self.entity_id = (ord(inbuf[0]) << 24) | (ord(inbuf[1]) << 16) | \ + (ord(inbuf[2]) << 8) | (ord(inbuf[3])) self.entity_str = inbuf[0:self.id_len] @@ -381,7 +380,7 @@ class Matroska(core.AVContainer): if header.get_id() != MATROSKA_HEADER_ID: raise ParseError() - log.debug("HEADER ID found %08X" % header.get_id() ) + log.debug(u'HEADER ID found %08X' % header.get_id()) self.mime = 'video/x-matroska' self.type = 'Matroska' self.has_idx = False @@ -392,10 +391,10 @@ class Matroska(core.AVContainer): # Record file offset of segment data for seekheads self.segment.offset = header.get_total_len() + segment.get_header_len() if segment.get_id() != MATROSKA_SEGMENT_ID: - log.debug("SEGMENT ID not found %08X" % segment.get_id()) + log.debug(u'SEGMENT ID not found %08X' % segment.get_id()) return - log.debug("SEGMENT ID found %08X" % segment.get_id()) + log.debug(u'SEGMENT ID found %08X' % segment.get_id()) try: for elem in self.process_one_level(segment): if elem.get_id() == MATROSKA_SEEKHEAD_ID: @@ -404,12 +403,12 @@ class Matroska(core.AVContainer): pass if not self.has_idx: - log.warning('File has no index') + log.warning(u'File has no index') self._set('corrupt', True) def process_elem(self, elem): elem_id = elem.get_id() - log.debug('BEGIN: process element %s' % hex(elem_id)) + log.debug(u'BEGIN: process element %r' % hex(elem_id)) if elem_id == MATROSKA_SEGMENT_INFO_ID: duration = 0 scalecode = 1000000.0 @@ -423,7 +422,7 @@ class Matroska(core.AVContainer): elif ielem_id == MATROSKA_TITLE_ID: self.title = ielem.get_utf8() elif ielem_id == MATROSKA_DATE_UTC_ID: - timestamp = unpack('!q', ielem.get_data())[0] / 10.0**9 + timestamp = unpack('!q', ielem.get_data())[0] / 10.0 ** 9 # Date is offset 2001-01-01 00:00:00 (timestamp 978307200.0) self.timestamp = int(timestamp + 978307200) @@ -447,7 +446,7 @@ class Matroska(core.AVContainer): elif elem_id == MATROSKA_CUES_ID: self.has_idx = True - log.debug('END: process element %s' % hex(elem_id)) + log.debug(u'END: process element %r' % hex(elem_id)) return True @@ -479,7 +478,7 @@ class Matroska(core.AVContainer): index = 0 while index < tracks.get_len(): trackelem = EbmlEntity(tracksbuf[index:]) - log.debug ("ELEMENT %X found" % trackelem.get_id()) + log.debug (u'ELEMENT %X found' % trackelem.get_id()) self.process_track(trackelem) index += trackelem.get_total_len() + trackelem.get_crc_len() @@ -503,20 +502,20 @@ class Matroska(core.AVContainer): elements = [x for x in self.process_one_level(track)] track_type = [x.get_value() for x in elements if x.get_id() == MATROSKA_TRACK_TYPE_ID] if not track_type: - log.debug('Bad track: no type id found') + log.debug(u'Bad track: no type id found') return track_type = track_type[0] track = None if track_type == MATROSKA_VIDEO_TRACK: - log.debug("Video track found") + log.debug(u'Video track found') track = self.process_video_track(elements) elif track_type == MATROSKA_AUDIO_TRACK: - log.debug("Audio track found") + log.debug(u'Audio track found') track = self.process_audio_track(elements) elif track_type == MATROSKA_SUBTITLES_TRACK: - log.debug("Subtitle track found") + log.debug(u'Subtitle track found') track = core.Subtitle() self.set_track_defaults(track) track.id = len(self.subtitles) @@ -529,7 +528,7 @@ class Matroska(core.AVContainer): elem_id = elem.get_id() if elem_id == MATROSKA_TRACK_LANGUAGE_ID: track.language = elem.get_str() - log.debug("Track language found: %s" % track.language) + log.debug(u'Track language found: %r' % track.language) elif elem_id == MATROSKA_NAME_ID: track.title = elem.get_utf8() elif elem_id == MATROSKA_TRACK_NUMBER_ID: @@ -671,7 +670,7 @@ class Matroska(core.AVContainer): elif elem_id == MATROSKA_CHAPTER_UID_ID: self.objects_by_uid[elem.get_value()] = chap - log.debug('Chapter "%s" found', chap.name) + log.debug(u'Chapter %r found', chap.name) chap.id = len(self.chapters) self.chapters.append(chap) @@ -704,10 +703,10 @@ class Matroska(core.AVContainer): # Right now we only support attachments that could be cover images. # Make a guess to see if this attachment is a cover image. - if mimetype.startswith("image/") and u"cover" in (name+desc).lower() and data: + if mimetype.startswith("image/") and u"cover" in (name + desc).lower() and data: self.thumbnail = data - log.debug('Attachment "%s" found' % name) + log.debug(u'Attachment %r found' % name) def process_tags(self, tags): @@ -744,7 +743,7 @@ class Matroska(core.AVContainer): self.objects_by_uid[target].tags.update(tags_dict) self.tags_to_attributes(self.objects_by_uid[target], tags_dict) except KeyError: - log.warning('Tags assigned to unknown/unsupported target uid %d', target) + log.warning(u'Tags assigned to unknown/unsupported target uid %d', target) else: self.tags.update(tags_dict) self.tags_to_attributes(self, tags_dict) @@ -816,7 +815,7 @@ class Matroska(core.AVContainer): try: value = [filter(item) for item in value] if isinstance(value, list) else filter(value) except Exception, e: - log.warning('Failed to convert tag to core attribute: %s', e) + log.warning(u'Failed to convert tag to core attribute: %r', e) # Special handling for tv series recordings. The 'title' tag # can be used for both the series and the episode name. The # same is true for trackno which may refer to the season diff --git a/libs/enzyme/mp4.py b/libs/enzyme/mp4.py index 41bf43da..c53f30d3 100644 --- a/libs/enzyme/mp4.py +++ b/libs/enzyme/mp4.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2003-2007 Thomas Schueppel -# Copyright (C) 2003-2007 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2007 Thomas Schueppel +# Copyright 2003-2007 Dirk Meyer # # This file is part of enzyme. # @@ -17,16 +17,14 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - +# along with enzyme. If not, see . __all__ = ['Parser'] import zlib import logging import StringIO import struct -from exceptions import * +from exceptions import ParseError import core # get logging object @@ -169,7 +167,7 @@ class MPEG4(core.AVContainer): self.type = 'Quicktime Video' h = file.read(8) try: - (size, type) = struct.unpack('>I4s',h) + (size, type) = struct.unpack('>I4s', h) except struct.error: # EOF. raise ParseError() @@ -183,18 +181,18 @@ class MPEG4(core.AVContainer): self.mime = 'video/mp4' self.type = 'MPEG-4 Video' size -= 4 - file.seek(size-8, 1) + file.seek(size - 8, 1) h = file.read(8) - (size, type) = struct.unpack('>I4s',h) + (size, type) = struct.unpack('>I4s', h) while type in ['mdat', 'skip']: # movie data at the beginning, skip - file.seek(size-8, 1) + file.seek(size - 8, 1) h = file.read(8) - (size, type) = struct.unpack('>I4s',h) + (size, type) = struct.unpack('>I4s', h) if not type in ['moov', 'wide', 'free']: - log.debug('invalid header: %r' % type) + log.debug(u'invalid header: %r' % type) raise ParseError() # Extended size @@ -216,37 +214,37 @@ class MPEG4(core.AVContainer): if len(s) < 8: return 0 - atomsize,atomtype = struct.unpack('>I4s', s) + atomsize, atomtype = struct.unpack('>I4s', s) if not str(atomtype).decode('latin1').isalnum(): # stop at nonsense data return 0 - log.debug('%s [%X]' % (atomtype,atomsize)) + log.debug(u'%r [%X]' % (atomtype, atomsize)) if atomtype == 'udta': # Userdata (Metadata) pos = 0 tabl = {} i18ntabl = {} - atomdata = file.read(atomsize-8) - while pos < atomsize-12: - (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8]) + atomdata = file.read(atomsize - 8) + while pos < atomsize - 12: + (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8]) if ord(datatype[0]) == 169: # i18n Metadata... - mypos = 8+pos - while mypos + 4 < datasize+pos: + mypos = 8 + pos + while mypos + 4 < datasize + pos: # first 4 Bytes are i18n header - (tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos+4]) + (tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos + 4]) i18ntabl[lang] = i18ntabl.get(lang, {}) - l = atomdata[mypos+4:mypos+tlen+4] + l = atomdata[mypos + 4:mypos + tlen + 4] i18ntabl[lang][datatype[1:]] = l - mypos += tlen+4 + mypos += tlen + 4 elif datatype == 'WLOC': # Drop Window Location pass else: - if ord(atomdata[pos+8:pos+datasize][0]) > 1: - tabl[datatype] = atomdata[pos+8:pos+datasize] + if ord(atomdata[pos + 8:pos + datasize][0]) > 1: + tabl[datatype] = atomdata[pos + 8:pos + datasize] pos += datasize if len(i18ntabl.keys()) > 0: for k in i18ntabl.keys(): @@ -254,19 +252,19 @@ class MPEG4(core.AVContainer): self._appendtable('QTUDTA', i18ntabl[k]) self._appendtable('QTUDTA', tabl) else: - log.debug('NO i18') + log.debug(u'NO i18') self._appendtable('QTUDTA', tabl) elif atomtype == 'trak': - atomdata = file.read(atomsize-8) + atomdata = file.read(atomsize - 8) pos = 0 trackinfo = {} tracktype = None - while pos < atomsize-8: - (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8]) + while pos < atomsize - 8: + (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8]) if datatype == 'tkhd': - tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos+8:pos+datasize]) + tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos + 8:pos + datasize]) trackinfo['width'] = tkhd[10] >> 16 trackinfo['height'] = tkhd[11] >> 16 trackinfo['id'] = tkhd[3] @@ -277,23 +275,23 @@ class MPEG4(core.AVContainer): # XXX Apple time. FIXME to work on Apple, too self.timestamp = int(tkhd[1]) - 2082844800 except Exception, e: - log.exception('There was trouble extracting timestamp') + log.exception(u'There was trouble extracting timestamp') elif datatype == 'mdia': pos += 8 datasize -= 8 - log.debug('--> mdia information') + log.debug(u'--> mdia information') while datasize: - mdia = struct.unpack('>I4s', atomdata[pos:pos+8]) + mdia = struct.unpack('>I4s', atomdata[pos:pos + 8]) if mdia[1] == 'mdhd': # Parse based on version of mdhd header. See # http://wiki.multimedia.cx/index.php?title=QuickTime_container#mdhd ver = ord(atomdata[pos + 8]) if ver == 0: - mdhd = struct.unpack('>IIIIIhh', atomdata[pos+8:pos+8+24]) + mdhd = struct.unpack('>IIIIIhh', atomdata[pos + 8:pos + 8 + 24]) elif ver == 1: - mdhd = struct.unpack('>IQQIQhh', atomdata[pos+8:pos+8+36]) + mdhd = struct.unpack('>IQQIQhh', atomdata[pos + 8:pos + 8 + 36]) else: mdhd = None @@ -313,34 +311,34 @@ class MPEG4(core.AVContainer): pos -= (mdia[0] - 8) datasize += (mdia[0] - 8) elif mdia[1] == 'hdlr': - hdlr = struct.unpack('>I4s4s', atomdata[pos+8:pos+8+12]) + hdlr = struct.unpack('>I4s4s', atomdata[pos + 8:pos + 8 + 12]) if hdlr[1] == 'mhlr': if hdlr[2] == 'vide': tracktype = 'video' if hdlr[2] == 'soun': tracktype = 'audio' elif mdia[1] == 'stsd': - stsd = struct.unpack('>2I', atomdata[pos+8:pos+8+8]) + stsd = struct.unpack('>2I', atomdata[pos + 8:pos + 8 + 8]) if stsd[1] > 0: - codec = atomdata[pos+16:pos+16+8] + codec = atomdata[pos + 16:pos + 16 + 8] codec = struct.unpack('>I4s', codec) trackinfo['codec'] = codec[1] if codec[1] == 'jpeg': tracktype = 'image' elif mdia[1] == 'dinf': - dref = struct.unpack('>I4s', atomdata[pos+8:pos+8+8]) - log.debug(' --> %s, %s (useless)' % mdia) + dref = struct.unpack('>I4s', atomdata[pos + 8:pos + 8 + 8]) + log.debug(u' --> %r, %r (useless)' % mdia) if dref[1] == 'dref': - num = struct.unpack('>I', atomdata[pos+20:pos+20+4])[0] - rpos = pos+20+4 + num = struct.unpack('>I', atomdata[pos + 20:pos + 20 + 4])[0] + rpos = pos + 20 + 4 for ref in range(num): # FIXME: do somthing if this references - ref = struct.unpack('>I3s', atomdata[rpos:rpos+7]) - data = atomdata[rpos+7:rpos+ref[0]] + ref = struct.unpack('>I3s', atomdata[rpos:rpos + 7]) + data = atomdata[rpos + 7:rpos + ref[0]] rpos += ref[0] else: if mdia[1].startswith('st'): - log.debug(' --> %s, %s (sample)' % mdia) + log.debug(u' --> %r, %r (sample)' % mdia) elif mdia[1] == 'vmhd' and not tracktype: # indicates that this track is video tracktype = 'video' @@ -348,19 +346,19 @@ class MPEG4(core.AVContainer): # indicates that this track is audio tracktype = 'audio' else: - log.debug(' --> %s, %s (unknown)' % mdia) + log.debug(u' --> %r, %r (unknown)' % mdia) pos += mdia[0] datasize -= mdia[0] elif datatype == 'udta': - log.debug(struct.unpack('>I4s', atomdata[:8])) + log.debug(u'udta: %r' % struct.unpack('>I4s', atomdata[:8])) else: if datatype == 'edts': - log.debug('--> %s [%d] (edit list)' % \ + log.debug(u'--> %r [%d] (edit list)' % \ (datatype, datasize)) else: - log.debug('--> %s [%d] (unknown)' % \ + log.debug(u'--> %r [%d] (unknown)' % \ (datatype, datasize)) pos += datasize @@ -380,7 +378,7 @@ class MPEG4(core.AVContainer): mvhd = struct.unpack('>6I2h', file.read(28)) self.length = max(self.length, mvhd[4] / mvhd[3]) self.volume = mvhd[6] - file.seek(atomsize-8-28,1) + file.seek(atomsize - 8 - 28, 1) elif atomtype == 'cmov': @@ -389,21 +387,21 @@ class MPEG4(core.AVContainer): if not atomtype == 'dcom': return atomsize - method = struct.unpack('>4s', file.read(datasize-8))[0] + method = struct.unpack('>4s', file.read(datasize - 8))[0] datasize, atomtype = struct.unpack('>I4s', file.read(8)) if not atomtype == 'cmvd': return atomsize if method == 'zlib': - data = file.read(datasize-8) + data = file.read(datasize - 8) try: decompressed = zlib.decompress(data) except Exception, e: try: decompressed = zlib.decompress(data[4:]) except Exception, e: - log.exception('There was a proble decompressiong atom') + log.exception(u'There was a proble decompressiong atom') return atomsize decompressedIO = StringIO.StringIO(decompressed) @@ -411,9 +409,9 @@ class MPEG4(core.AVContainer): pass else: - log.info('unknown compression %s' % method) + log.info(u'unknown compression %r' % method) # unknown compression method - file.seek(datasize-8,1) + file.seek(datasize - 8, 1) elif atomtype == 'moov': # decompressed movie info @@ -423,10 +421,10 @@ class MPEG4(core.AVContainer): elif atomtype == 'mdat': pos = file.tell() + atomsize - 8 # maybe there is data inside the mdat - log.info('parsing mdat') + log.info(u'parsing mdat') while self._readatom(file): pass - log.info('end of mdat') + log.info(u'end of mdat') file.seek(pos, 0) @@ -437,24 +435,24 @@ class MPEG4(core.AVContainer): elif atomtype == 'rmda': # reference - atomdata = file.read(atomsize-8) + atomdata = file.read(atomsize - 8) pos = 0 url = '' quality = 0 datarate = 0 - while pos < atomsize-8: - (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8]) + while pos < atomsize - 8: + (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8]) if datatype == 'rdrf': - rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos+8:pos+20]) + rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos + 8:pos + 20]) if rtype == 'url ': - url = atomdata[pos+20:pos+20+rlen] + url = atomdata[pos + 20:pos + 20 + rlen] if url.find('\0') > 0: url = url[:url.find('\0')] elif datatype == 'rmqu': - quality = struct.unpack('>I', atomdata[pos+8:pos+12])[0] + quality = struct.unpack('>I', atomdata[pos + 8:pos + 12])[0] elif datatype == 'rmdr': - datarate = struct.unpack('>I', atomdata[pos+12:pos+16])[0] + datarate = struct.unpack('>I', atomdata[pos + 12:pos + 16])[0] pos += datasize if url: @@ -462,11 +460,11 @@ class MPEG4(core.AVContainer): else: if not atomtype in ['wide', 'free']: - log.info('unhandled base atom %s' % atomtype) + log.info(u'unhandled base atom %r' % atomtype) # Skip unknown atoms try: - file.seek(atomsize-8,1) + file.seek(atomsize - 8, 1) except IOError: return 0 diff --git a/libs/enzyme/mpeg.py b/libs/enzyme/mpeg.py index d7c44db2..3d43ba4f 100644 --- a/libs/enzyme/mpeg.py +++ b/libs/enzyme/mpeg.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2003-2006 Thomas Schueppel -# Copyright (C) 2003-2006 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Thomas Schueppel +# Copyright 2003-2006 Dirk Meyer # # This file is part of enzyme. # @@ -17,16 +17,14 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - +# along with enzyme. If not, see . __all__ = ['Parser'] import os import struct import logging import stat -from exceptions import * +from exceptions import ParseError import core # get logging object @@ -49,7 +47,7 @@ START_CODE = { 0xB7 : 'sequence end', 0xB8 : 'group of pictures', } -for i in range(0x01,0xAF): +for i in range(0x01, 0xAF): START_CODE[i] = 'slice_start_code' ##------------------------------------------------------------------------ @@ -96,14 +94,14 @@ TS_SYNC = 0x47 ##------------------------------------------------------------------------ FRAME_RATE = [ 0, - 24000.0/1001, ## 3-2 pulldown NTSC (CPB/Main Level) - 24, ## Film (CPB/Main Level) - 25, ## PAL/SECAM or 625/60 video - 30000.0/1001, ## NTSC (CPB/Main Level) - 30, ## drop-frame NTSC or component 525/60 (CPB/Main Level) - 50, ## double-rate PAL - 60000.0/1001, ## double-rate NTSC - 60, ## double-rate, drop-frame NTSC/component 525/60 video + 24000.0 / 1001, ## 3-2 pulldown NTSC (CPB/Main Level) + 24, ## Film (CPB/Main Level) + 25, ## PAL/SECAM or 625/60 video + 30000.0 / 1001, ## NTSC (CPB/Main Level) + 30, ## drop-frame NTSC or component 525/60 (CPB/Main Level) + 50, ## double-rate PAL + 60000.0 / 1001, ## double-rate NTSC + 60, ## double-rate, drop-frame NTSC/component 525/60 video ] ##------------------------------------------------------------------------ @@ -114,9 +112,9 @@ FRAME_RATE = [ ## it, a stream that doesn't adhere to one of these aspect ratios is ## technically considered non-compliant. ##------------------------------------------------------------------------ -ASPECT_RATIO = ( None, # Forbidden - 1.0, # 1/1 (VGA) - 4.0 / 3, # 4/3 (TV) +ASPECT_RATIO = (None, # Forbidden + 1.0, # 1/1 (VGA) + 4.0 / 3, # 4/3 (TV) 16.0 / 9, # 16/9 (Widescreen) 2.21 # (Cinema) ) @@ -131,7 +129,7 @@ class MPEG(core.AVContainer): no additional metadata like title, etc; only codecs, length and resolution is reported back. """ - def __init__(self,file): + def __init__(self, file): core.AVContainer.__init__(self) self.sequence_header_offset = 0 self.mpeg_version = 2 @@ -190,29 +188,29 @@ class MPEG(core.AVContainer): a.codec = ac - def dxy(self,file): + def dxy(self, file): """ get width and height of the video """ - file.seek(self.sequence_header_offset+4,0) + file.seek(self.sequence_header_offset + 4, 0) v = file.read(4) - x = struct.unpack('>H',v[:2])[0] >> 4 - y = struct.unpack('>H',v[1:3])[0] & 0x0FFF - return (x,y) + x = struct.unpack('>H', v[:2])[0] >> 4 + y = struct.unpack('>H', v[1:3])[0] & 0x0FFF + return (x, y) - def framerate_aspect(self,file): + def framerate_aspect(self, file): """ read framerate and aspect ratio """ - file.seek(self.sequence_header_offset+7,0) - v = struct.unpack( '>B', file.read(1) )[0] + file.seek(self.sequence_header_offset + 7, 0) + v = struct.unpack('>B', file.read(1))[0] try: - fps = FRAME_RATE[v&0xf] + fps = FRAME_RATE[v & 0xf] except IndexError: fps = None - if v>>4 < len(ASPECT_RATIO): - aspect = ASPECT_RATIO[v>>4] + if v >> 4 < len(ASPECT_RATIO): + aspect = ASPECT_RATIO[v >> 4] else: aspect = None return (fps, aspect) @@ -238,18 +236,18 @@ class MPEG(core.AVContainer): if pos == -1 or len(buffer) - pos < 5: buffer = buffer[-10:] continue - ext = (ord(buffer[pos+4]) >> 4) + ext = (ord(buffer[pos + 4]) >> 4) if ext == 8: pass elif ext == 1: - if (ord(buffer[pos+5]) >> 3) & 1: + if (ord(buffer[pos + 5]) >> 3) & 1: self._set('progressive', True) else: self._set('interlaced', True) return True else: - log.debug('ext', ext) - buffer = buffer[pos+4:] + log.debug(u'ext: %r' % ext) + buffer = buffer[pos + 4:] return False @@ -292,12 +290,12 @@ class MPEG(core.AVContainer): ## Some parts in the code are based on mpgtx (mpgtx.sf.net) - def bitrate(self,file): + def bitrate(self, file): """ read the bitrate (most of the time broken) """ - file.seek(self.sequence_header_offset+8,0) - t,b = struct.unpack( '>HB', file.read(3) ) + file.seek(self.sequence_header_offset + 8, 0) + t, b = struct.unpack('>HB', file.read(3)) vrate = t << 2 | b >> 6 return vrate * 400 @@ -309,9 +307,9 @@ class MPEG(core.AVContainer): if len(buffer) < 6: return None - highbit = (ord(buffer[0])&0x20)>>5 + highbit = (ord(buffer[0]) & 0x20) >> 5 - low4Bytes= ((long(ord(buffer[0])) & 0x18) >> 3) << 30 + low4Bytes = ((long(ord(buffer[0])) & 0x18) >> 3) << 30 low4Bytes |= (ord(buffer[0]) & 0x03) << 28 low4Bytes |= ord(buffer[1]) << 20 low4Bytes |= (ord(buffer[2]) & 0xF8) << 12 @@ -319,10 +317,10 @@ class MPEG(core.AVContainer): low4Bytes |= ord(buffer[3]) << 5 low4Bytes |= (ord(buffer[4])) >> 3 - sys_clock_ref=(ord(buffer[4]) & 0x3) << 7 - sys_clock_ref|=(ord(buffer[5]) >> 1) + sys_clock_ref = (ord(buffer[4]) & 0x3) << 7 + sys_clock_ref |= (ord(buffer[5]) >> 1) - return (long(highbit * (1<<16) * (1<<16)) + low4Bytes) / 90000 + return (long(highbit * (1 << 16) * (1 << 16)) + low4Bytes) / 90000 def ReadSCRMpeg1(self, buffer): @@ -340,7 +338,7 @@ class MPEG(core.AVContainer): low4Bytes |= ord(buffer[3]) << 7; low4Bytes |= ord(buffer[4]) >> 1; - return (long(highbit) * (1<<16) * (1<<16) + low4Bytes) / 90000; + return (long(highbit) * (1 << 16) * (1 << 16) + low4Bytes) / 90000; def ReadPTS(self, buffer): @@ -350,7 +348,7 @@ class MPEG(core.AVContainer): high = ((ord(buffer[0]) & 0xF) >> 1) med = (ord(buffer[1]) << 7) + (ord(buffer[2]) >> 1) low = (ord(buffer[3]) << 7) + (ord(buffer[4]) >> 1) - return ((long(high) << 30 ) + (med << 15) + low) / 90000 + return ((long(high) << 30) + (med << 15) + low) / 90000 def ReadHeader(self, buffer, offset): @@ -358,25 +356,25 @@ class MPEG(core.AVContainer): Handle MPEG header in buffer on position offset Return None on error, new offset or 0 if the new offset can't be scanned """ - if buffer[offset:offset+3] != '\x00\x00\x01': + if buffer[offset:offset + 3] != '\x00\x00\x01': return None - id = ord(buffer[offset+3]) + id = ord(buffer[offset + 3]) if id == PADDING_PKT: - return offset + (ord(buffer[offset+4]) << 8) + \ - ord(buffer[offset+5]) + 6 + return offset + (ord(buffer[offset + 4]) << 8) + \ + ord(buffer[offset + 5]) + 6 if id == PACK_PKT: - if ord(buffer[offset+4]) & 0xF0 == 0x20: + if ord(buffer[offset + 4]) & 0xF0 == 0x20: self.type = 'MPEG-1 Video' self.get_time = self.ReadSCRMpeg1 self.mpeg_version = 1 return offset + 12 - elif (ord(buffer[offset+4]) & 0xC0) == 0x40: + elif (ord(buffer[offset + 4]) & 0xC0) == 0x40: self.type = 'MPEG-2 Video' self.get_time = self.ReadSCRMpeg2 - return offset + (ord(buffer[offset+13]) & 0x07) + 14 + return offset + (ord(buffer[offset + 13]) & 0x07) + 14 else: # I have no idea what just happened, but for some DVB # recordings done with mencoder this points to a @@ -412,10 +410,10 @@ class MPEG(core.AVContainer): if id in [PRIVATE_STREAM1, PRIVATE_STREAM2]: # private stream. we don't know, but maybe we can guess later - add = ord(buffer[offset+8]) + add = ord(buffer[offset + 8]) # if (ord(buffer[offset+6]) & 4) or 1: # id = ord(buffer[offset+10+add]) - if buffer[offset+11+add:offset+15+add].find('\x0b\x77') != -1: + if buffer[offset + 11 + add:offset + 15 + add].find('\x0b\x77') != -1: # AC3 stream for a in self.audio: if a.id == id: @@ -442,18 +440,18 @@ class MPEG(core.AVContainer): This MPEG starts with a sequence of 0x00 followed by a PACK Header http://dvd.sourceforge.net/dvdinfo/packhdr.html """ - file.seek(0,0) + file.seek(0, 0) buffer = file.read(10000) offset = 0 # seek until the 0 byte stop - while offset < len(buffer)-100 and buffer[offset] == '\0': + while offset < len(buffer) - 100 and buffer[offset] == '\0': offset += 1 offset -= 2 # test for mpeg header 0x00 0x00 0x01 header = '\x00\x00\x01%s' % chr(PACK_PKT) - if offset < 0 or not buffer[offset:offset+4] == header: + if offset < 0 or not buffer[offset:offset + 4] == header: if not force: return 0 # brute force and try to find the pack header in the first @@ -470,9 +468,9 @@ class MPEG(core.AVContainer): self.ReadHeader(buffer, offset) # store first timestamp - self.start = self.get_time(buffer[offset+4:]) + self.start = self.get_time(buffer[offset + 4:]) while len(buffer) > offset + 1000 and \ - buffer[offset:offset+3] == '\x00\x00\x01': + buffer[offset:offset + 3] == '\x00\x00\x01': # read the mpeg header new_offset = self.ReadHeader(buffer, offset) @@ -486,12 +484,12 @@ class MPEG(core.AVContainer): # skip padding 0 before a new header while len(buffer) > offset + 10 and \ - not ord(buffer[offset+2]): + not ord(buffer[offset + 2]): offset += 1 else: # seek to new header by brute force - offset += buffer[offset+4:].find('\x00\x00\x01') + 4 + offset += buffer[offset + 4:].find('\x00\x00\x01') + 4 # fill in values for support functions: self.__seek_size__ = 1000000 @@ -555,15 +553,15 @@ class MPEG(core.AVContainer): self.video[-1]._set('id', id) # new mpeg starting - if buffer[header_length+9:header_length+13] == \ + if buffer[header_length + 9:header_length + 13] == \ '\x00\x00\x01\xB3' and not self.sequence_header_offset: # yes, remember offset for later use - self.sequence_header_offset = offset + header_length+9 + self.sequence_header_offset = offset + header_length + 9 elif ord(buffer[3]) == 189 or ord(buffer[3]) == 191: # private stream. we don't know, but maybe we can guess later id = id or ord(buffer[3]) & 0xF if align and \ - buffer[header_length+9:header_length+11] == '\x0b\x77': + buffer[header_length + 9:header_length + 11] == '\x0b\x77': # AC3 stream for a in self.audio: if a.id == id: @@ -581,7 +579,7 @@ class MPEG(core.AVContainer): if ptsdts and ptsdts == ord(buffer[9]) >> 4: if ord(buffer[9]) >> 4 != ptsdts: - log.warning('WARNING: bad PTS/DTS, please contact us') + log.warning(u'WARNING: bad PTS/DTS, please contact us') return packet_length, None # timestamp = self.ReadPTS(buffer[9:14]) @@ -595,8 +593,8 @@ class MPEG(core.AVContainer): def isPES(self, file): - log.info('trying mpeg-pes scan') - file.seek(0,0) + log.info(u'trying mpeg-pes scan') + file.seek(0, 0) buffer = file.read(3) # header (also valid for all mpegs) @@ -613,7 +611,7 @@ class MPEG(core.AVContainer): return 0 if timestamp != None and not hasattr(self, 'start'): self.get_time = self.ReadPTS - bpos = buffer[offset+timestamp:offset+timestamp+5] + bpos = buffer[offset + timestamp:offset + timestamp + 5] self.start = self.get_time(bpos) if self.sequence_header_offset and hasattr(self, 'start'): # we have all informations we need @@ -702,13 +700,13 @@ class MPEG(core.AVContainer): # Transport Stream =============================================== def isTS(self, file): - file.seek(0,0) + file.seek(0, 0) buffer = file.read(TS_PACKET_LENGTH * 2) c = 0 while c + TS_PACKET_LENGTH < len(buffer): - if ord(buffer[c]) == ord(buffer[c+TS_PACKET_LENGTH]) == TS_SYNC: + if ord(buffer[c]) == ord(buffer[c + TS_PACKET_LENGTH]) == TS_SYNC: break c += 1 else: @@ -718,7 +716,7 @@ class MPEG(core.AVContainer): self.type = 'MPEG-TS' while c + TS_PACKET_LENGTH < len(buffer): - start = ord(buffer[c+1]) & 0x40 + start = ord(buffer[c + 1]) & 0x40 # maybe load more into the buffer if c + 2 * TS_PACKET_LENGTH > len(buffer) and c < 500000: buffer += file.read(10000) @@ -728,30 +726,30 @@ class MPEG(core.AVContainer): c += TS_PACKET_LENGTH continue - tsid = ((ord(buffer[c+1]) & 0x3F) << 8) + ord(buffer[c+2]) - adapt = (ord(buffer[c+3]) & 0x30) >> 4 + tsid = ((ord(buffer[c + 1]) & 0x3F) << 8) + ord(buffer[c + 2]) + adapt = (ord(buffer[c + 3]) & 0x30) >> 4 offset = 4 if adapt & 0x02: # meta info present, skip it for now - adapt_len = ord(buffer[c+offset]) + adapt_len = ord(buffer[c + offset]) offset += adapt_len + 1 - if not ord(buffer[c+1]) & 0x40: + if not ord(buffer[c + 1]) & 0x40: # no new pes or psi in stream payload starting pass elif adapt & 0x01: # PES - timestamp = self.ReadPESHeader(c+offset, buffer[c+offset:], + timestamp = self.ReadPESHeader(c + offset, buffer[c + offset:], tsid)[1] if timestamp != None: if not hasattr(self, 'start'): self.get_time = self.ReadPTS timestamp = c + offset + timestamp - self.start = self.get_time(buffer[timestamp:timestamp+5]) + self.start = self.get_time(buffer[timestamp:timestamp + 5]) elif not hasattr(self, 'audio_ok'): timestamp = c + offset + timestamp - start = self.get_time(buffer[timestamp:timestamp+5]) + start = self.get_time(buffer[timestamp:timestamp + 5]) if start is not None and self.start is not None and \ abs(start - self.start) < 10: # looks ok @@ -759,7 +757,7 @@ class MPEG(core.AVContainer): else: # timestamp broken del self.start - log.warning('Timestamp error, correcting') + log.warning(u'Timestamp error, correcting') if hasattr(self, 'start') and self.start and \ self.sequence_header_offset and self.video and self.audio: @@ -786,31 +784,31 @@ class MPEG(core.AVContainer): c = 0 while c + TS_PACKET_LENGTH < len(buffer): - if ord(buffer[c]) == ord(buffer[c+TS_PACKET_LENGTH]) == TS_SYNC: + if ord(buffer[c]) == ord(buffer[c + TS_PACKET_LENGTH]) == TS_SYNC: break c += 1 else: return None while c + TS_PACKET_LENGTH < len(buffer): - start = ord(buffer[c+1]) & 0x40 + start = ord(buffer[c + 1]) & 0x40 if not start: c += TS_PACKET_LENGTH continue - tsid = ((ord(buffer[c+1]) & 0x3F) << 8) + ord(buffer[c+2]) - adapt = (ord(buffer[c+3]) & 0x30) >> 4 + tsid = ((ord(buffer[c + 1]) & 0x3F) << 8) + ord(buffer[c + 2]) + adapt = (ord(buffer[c + 3]) & 0x30) >> 4 offset = 4 if adapt & 0x02: # meta info present, skip it for now - offset += ord(buffer[c+offset]) + 1 + offset += ord(buffer[c + offset]) + 1 if adapt & 0x01: - timestamp = self.ReadPESHeader(c+offset, buffer[c+offset:], tsid)[1] + timestamp = self.ReadPESHeader(c + offset, buffer[c + offset:], tsid)[1] if timestamp is None: # this should not happen - log.error('bad TS') + log.error(u'bad TS') return None return c + offset + timestamp c += TS_PACKET_LENGTH @@ -841,7 +839,7 @@ class MPEG(core.AVContainer): if pos == None: break end = self.get_time(buffer[pos:]) or end - buffer = buffer[pos+100:] + buffer = buffer[pos + 100:] file.close() return end @@ -855,7 +853,7 @@ class MPEG(core.AVContainer): if end == None or self.start == None: return None if self.start > end: - return int(((long(1) << 33) - 1 ) / 90000) - self.start + end + return int(((long(1) << 33) - 1) / 90000) - self.start + end return end - self.start @@ -897,7 +895,7 @@ class MPEG(core.AVContainer): return 0 file = open(self.filename) - log.debug('scanning file...') + log.debug(u'scanning file...') while 1: file.seek(self.__seek_size__ * 10, 1) buffer = file.read(self.__sample_size__) @@ -906,10 +904,10 @@ class MPEG(core.AVContainer): pos = self.__search__(buffer) if pos == None: continue - log.debug('buffer position: %s' % self.get_time(buffer[pos:])) + log.debug(u'buffer position: %r' % self.get_time(buffer[pos:])) file.close() - log.debug('done scanning file') + log.debug(u'done scanning file') Parser = MPEG diff --git a/libs/enzyme/ogm.py b/libs/enzyme/ogm.py index 6857cb66..4198be24 100644 --- a/libs/enzyme/ogm.py +++ b/libs/enzyme/ogm.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2003-2006 Thomas Schueppel -# Copyright (C) 2003-2006 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Thomas Schueppel +# Copyright 2003-2006 Dirk Meyer # # This file is part of enzyme. # @@ -17,9 +17,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - +# along with enzyme. If not, see . __all__ = ['Parser'] import struct @@ -27,7 +25,7 @@ import re import stat import os import logging -from exceptions import * +from exceptions import ParseError import core # get logging object @@ -82,7 +80,7 @@ class Ogm(core.AVContainer): # seek to the end of the stream, to avoid scanning the whole file if (os.stat(file.name)[stat.ST_SIZE] > 50000): - file.seek(os.stat(file.name)[stat.ST_SIZE]-49000) + file.seek(os.stat(file.name)[stat.ST_SIZE] - 49000) # read the rest of the file into a buffer h = file.read() @@ -147,21 +145,21 @@ class Ogm(core.AVContainer): self._appendtable('VORBISCOMMENT', header) - def _parseOGGS(self,file): + def _parseOGGS(self, file): h = file.read(27) if len(h) == 0: # Regular File end return None, None elif len(h) < 27: - log.debug("%d Bytes of Garbage found after End." % len(h)) + log.debug(u'%d Bytes of Garbage found after End.' % len(h)) return None, None if h[:4] != "OggS": - log.debug("Invalid Ogg") + log.debug(u'Invalid Ogg') raise ParseError() version = ord(h[4]) if version != 0: - log.debug("Unsupported OGG/OGM Version %d." % version) + log.debug(u'Unsupported OGG/OGM Version %d' % version) return None, None head = struct.unpack(' serial: stream = self.all_streams[serial] if hasattr(stream, 'samplerate') and \ @@ -197,27 +195,27 @@ class Ogm(core.AVContainer): return granulepos, nextlen + 27 + pageSegCount - def _parseMeta(self,h): + def _parseMeta(self, h): flags = ord(h[0]) headerlen = len(h) if headerlen >= 7 and h[1:7] == 'vorbis': header = {} nextlen, self.encoder = self._extractHeaderString(h[7:]) - numItems = struct.unpack('= struct.calcsize(STREAM_HEADER_VIDEO)+1: + headerlen >= struct.calcsize(STREAM_HEADER_VIDEO) + 1: # New Directshow Format htype = header[1:9] if htype[:5] == 'video': - sh = header[9:struct.calcsize(STREAM_HEADER_VIDEO)+9] + sh = header[9:struct.calcsize(STREAM_HEADER_VIDEO) + 9] streamheader = struct.unpack(STREAM_HEADER_VIDEO, sh) vi = core.VideoStream() (type, ssize, timeunit, samplerate, vi.length, buffersize, \ @@ -270,13 +268,13 @@ class Ogm(core.AVContainer): self.all_streams.append(vi) elif htype[:5] == 'audio': - sha = header[9:struct.calcsize(STREAM_HEADER_AUDIO)+9] + sha = header[9:struct.calcsize(STREAM_HEADER_AUDIO) + 9] streamheader = struct.unpack(STREAM_HEADER_AUDIO, sha) ai = core.AudioStream() (type, ssize, timeunit, ai.samplerate, ai.length, buffersize, \ ai.bitrate, ai.channels, bloc, ai.bitrate) = streamheader self.samplerate = ai.samplerate - log.debug("Samplerate %d" % self.samplerate) + log.debug(u'Samplerate %d' % self.samplerate) self.audio.append(ai) self.all_streams.append(ai) @@ -287,15 +285,15 @@ class Ogm(core.AVContainer): self.all_streams.append(subtitle) else: - log.debug("Unknown Header") + log.debug(u'Unknown Header') - def _extractHeaderString(self,header): + def _extractHeaderString(self, header): len = struct.unpack(' -# Copyright (C) 2003-2006 Thomas Schueppel -# Copyright (C) 2003-2006 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Thomas Schueppel +# Copyright 2003-2006 Dirk Meyer # # This file is part of enzyme. # @@ -17,14 +17,12 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - +# along with enzyme. If not, see . __all__ = ['Parser'] import struct import logging -from exceptions import * +from exceptions import ParseError import core # http://www.pcisys.net/~melanson/codecs/rmff.htm @@ -34,13 +32,13 @@ import core log = logging.getLogger(__name__) class RealVideo(core.AVContainer): - def __init__(self,file): + def __init__(self, file): core.AVContainer.__init__(self) self.mime = 'video/real' self.type = 'Real Video' h = file.read(10) try: - (object_id,object_size,object_version) = struct.unpack('>4sIH',h) + (object_id, object_size, object_version) = struct.unpack('>4sIH', h) except struct.error: # EOF. raise ParseError() @@ -49,47 +47,47 @@ class RealVideo(core.AVContainer): raise ParseError() file_version, num_headers = struct.unpack('>II', file.read(8)) - log.debug("size: %d, ver: %d, headers: %d" % \ - (object_size, file_version,num_headers)) - for i in range(0,num_headers): + log.debug(u'size: %d, ver: %d, headers: %d' % \ + (object_size, file_version, num_headers)) + for _ in range(0, num_headers): try: - oi = struct.unpack('>4sIH',file.read(10)) + oi = struct.unpack('>4sIH', file.read(10)) except (struct.error, IOError): # Header data we expected wasn't there. File may be # only partially complete. break if object_id == 'DATA' and oi[0] != 'INDX': - log.debug('INDX chunk expected after DATA but not found -- file corrupt') + log.debug(u'INDX chunk expected after DATA but not found -- file corrupt') break - (object_id,object_size,object_version) = oi + (object_id, object_size, object_version) = oi if object_id == 'DATA': # Seek over the data chunk rather than reading it in. file.seek(object_size - 10, 1) else: - self._read_header(object_id, file.read(object_size-10)) - log.debug("%s [%d]" % (object_id,object_size-10)) + self._read_header(object_id, file.read(object_size - 10)) + log.debug(u'%r [%d]' % (object_id, object_size - 10)) # Read all the following headers - def _read_header(self,object_id,s): + def _read_header(self, object_id, s): if object_id == 'PROP': prop = struct.unpack('>9IHH', s) - log.debug(prop) + log.debug(u'PROP: %r' % prop) if object_id == 'MDPR': mdpr = struct.unpack('>H7I', s[:30]) - log.debug(mdpr) - self.length = mdpr[7]/1000.0 + log.debug(u'MDPR: %r' % mdpr) + self.length = mdpr[7] / 1000.0 (stream_name_size,) = struct.unpack('>B', s[30:31]) - stream_name = s[31:31+stream_name_size] - pos = 31+stream_name_size - (mime_type_size,) = struct.unpack('>B', s[pos:pos+1]) - mime = s[pos+1:pos+1+mime_type_size] - pos += mime_type_size+1 - (type_specific_len,) = struct.unpack('>I', s[pos:pos+4]) - type_specific = s[pos+4:pos+4+type_specific_len] - pos += 4+type_specific_len + stream_name = s[31:31 + stream_name_size] + pos = 31 + stream_name_size + (mime_type_size,) = struct.unpack('>B', s[pos:pos + 1]) + mime = s[pos + 1:pos + 1 + mime_type_size] + pos += mime_type_size + 1 + (type_specific_len,) = struct.unpack('>I', s[pos:pos + 4]) + type_specific = s[pos + 4:pos + 4 + type_specific_len] + pos += 4 + type_specific_len if mime[:5] == 'audio': ai = core.AudioStream() ai.id = mdpr[0] @@ -101,20 +99,20 @@ class RealVideo(core.AVContainer): vi.bitrate = mdpr[2] self.video.append(vi) else: - log.debug("Unknown: %s" % mime) + log.debug(u'Unknown: %r' % mime) if object_id == 'CONT': pos = 0 - (title_len,) = struct.unpack('>H', s[pos:pos+2]) - self.title = s[2:title_len+2] - pos += title_len+2 - (author_len,) = struct.unpack('>H', s[pos:pos+2]) - self.artist = s[pos+2:pos+author_len+2] - pos += author_len+2 - (copyright_len,) = struct.unpack('>H', s[pos:pos+2]) - self.copyright = s[pos+2:pos+copyright_len+2] - pos += copyright_len+2 - (comment_len,) = struct.unpack('>H', s[pos:pos+2]) - self.comment = s[pos+2:pos+comment_len+2] + (title_len,) = struct.unpack('>H', s[pos:pos + 2]) + self.title = s[2:title_len + 2] + pos += title_len + 2 + (author_len,) = struct.unpack('>H', s[pos:pos + 2]) + self.artist = s[pos + 2:pos + author_len + 2] + pos += author_len + 2 + (copyright_len,) = struct.unpack('>H', s[pos:pos + 2]) + self.copyright = s[pos + 2:pos + copyright_len + 2] + pos += copyright_len + 2 + (comment_len,) = struct.unpack('>H', s[pos:pos + 2]) + self.comment = s[pos + 2:pos + comment_len + 2] Parser = RealVideo diff --git a/libs/enzyme/riff.py b/libs/enzyme/riff.py index 221a5fc7..516c727b 100644 --- a/libs/enzyme/riff.py +++ b/libs/enzyme/riff.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2003-2006 Thomas Schueppel -# Copyright (C) 2003-2006 Dirk Meyer +# Copyright 2011-2012 Antoine Bertin +# Copyright 2003-2006 Thomas Schueppel +# Copyright 2003-2006 Dirk Meyer # # This file is part of enzyme. # @@ -17,9 +17,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - +# along with enzyme. If not, see . __all__ = ['Parser'] import os @@ -27,7 +25,7 @@ import struct import string import logging import time -from exceptions import * +from exceptions import ParseError import core # get logging object @@ -70,7 +68,7 @@ class Riff(core.AVContainer): """ table_mapping = { 'AVIINFO' : AVIINFO } - def __init__(self,file): + def __init__(self, file): core.AVContainer.__init__(self) # read the header h = file.read(12) @@ -90,12 +88,12 @@ class Riff(core.AVContainer): while self._parseRIFFChunk(file): pass except IOError: - log.exception('error in file, stop parsing') + log.exception(u'error in file, stop parsing') self._find_subtitles(file.name) if not self.has_idx and isinstance(self, core.AVContainer): - log.debug('WARNING: avi has no index') + log.debug(u'WARNING: avi has no index') self._set('corrupt', True) @@ -104,9 +102,9 @@ class Riff(core.AVContainer): Search for subtitle files. Right now only VobSub is supported """ base = os.path.splitext(filename)[0] - if os.path.isfile(base+'.idx') and \ - (os.path.isfile(base+'.sub') or os.path.isfile(base+'.rar')): - file = open(base+'.idx') + if os.path.isfile(base + '.idx') and \ + (os.path.isfile(base + '.sub') or os.path.isfile(base + '.rar')): + file = open(base + '.idx') if file.readline().find('VobSub index file') > 0: for line in file.readlines(): if line.find('id') == 0: @@ -117,10 +115,10 @@ class Riff(core.AVContainer): file.close() - def _parseAVIH(self,t): + def _parseAVIH(self, t): retval = {} - v = struct.unpack(' 1024*500: + if key[2:] != 'dc' or sz > 1024 * 500: # This chunk is not video or is unusually big (> 500KB); # skip it. file.seek(sz, 1) @@ -338,7 +336,7 @@ class Riff(core.AVContainer): startcode = 0xff def bits(v, o, n): # Returns n bits in v, offset o bits. - return (v & 2**n-1 << (64-n-o)) >> 64-n-o + return (v & 2 ** n - 1 << (64 - n - o)) >> 64 - n - o while pos < sz: startcode = ((startcode << 8) | ord(data[pos])) & 0xffffffff @@ -350,7 +348,7 @@ class Riff(core.AVContainer): if startcode >= 0x120 and startcode <= 0x12F: # We have the VOL startcode. Pull 64 bits of it and treat # as a bitstream - v = struct.unpack(">Q", data[pos : pos+8])[0] + v = struct.unpack(">Q", data[pos : pos + 8])[0] offset = 10 if bits(v, 9, 1): # is_ol_id, skip over vo_ver_id and vo_priority @@ -385,36 +383,36 @@ class Riff(core.AVContainer): if i < size: # Seek past whatever might be remaining of the movi list. - file.seek(size-i,1) + file.seek(size - i, 1) - def _parseLIST(self,t): + def _parseLIST(self, t): retval = {} i = 0 size = len(t) - while i < size-8: + while i < size - 8: # skip zero if ord(t[i]) == 0: i += 1 - key = t[i:i+4] + key = t[i:i + 4] sz = 0 if key == 'LIST': - sz = struct.unpack(' 0: @@ -466,17 +464,17 @@ class Riff(core.AVContainer): except ValueError: pass else: - log.debug('no support for time format %s', value) - i+=sz + log.debug(u'no support for time format %r', value) + i += sz return retval - def _parseRIFFChunk(self,file): + def _parseRIFFChunk(self, file): h = file.read(8) if len(h) < 8: return False name = h[:4] - size = struct.unpack(' 80000: - log.debug('RIFF LIST "%s" too long to parse: %s bytes' % (key, size)) - t = file.seek(size-4,1) + log.debug(u'RIFF LIST %r too long to parse: %r bytes' % (key, size)) + t = file.seek(size - 4, 1) return True elif size < 5: - log.debug('RIFF LIST "%s" too short: %s bytes' % (key, size)) + log.debug(u'RIFF LIST %r too short: %r bytes' % (key, size)) return True - t = file.read(size-4) - log.debug('parse RIFF LIST "%s": %d bytes' % (key, size)) + t = file.read(size - 4) + log.debug(u'parse RIFF LIST %r: %d bytes' % (key, size)) value = self._parseLIST(t) self.header[key] = value if key == 'INFO': @@ -511,7 +509,7 @@ class Riff(core.AVContainer): # no need to add this info to a table pass else: - log.debug('Skipping table info %s' % key) + log.debug(u'Skipping table info %r' % key) elif name == 'JUNK': self.junkStart = file.tell() - 8 @@ -519,15 +517,15 @@ class Riff(core.AVContainer): file.seek(size, 1) elif name == 'idx1': self.has_idx = True - log.debug('idx1: %s bytes' % size) + log.debug(u'idx1: %r bytes' % size) # no need to parse this - t = file.seek(size,1) + t = file.seek(size, 1) elif name == 'RIFF': - log.debug("New RIFF chunk, extended avi [%i]" % size) + log.debug(u'New RIFF chunk, extended avi [%i]' % size) type = file.read(4) if type != 'AVIX': - log.debug("Second RIFF chunk is %s, not AVIX, skipping", type) - file.seek(size-4, 1) + log.debug(u'Second RIFF chunk is %r, not AVIX, skipping', type) + file.seek(size - 4, 1) # that's it, no new informations should be in AVIX return False elif name == 'fmt ' and size <= 50: @@ -556,11 +554,11 @@ class Riff(core.AVContainer): elif not name.strip(string.printable + string.whitespace): # check if name is something usefull at all, maybe it is no # avi or broken - t = file.seek(size,1) - log.debug("Skipping %s [%i]" % (name,size)) + t = file.seek(size, 1) + log.debug(u'Skipping %r [%i]' % (name, size)) else: # bad avi - log.debug("Bad or broken avi") + log.debug(u'Bad or broken avi') return False return True diff --git a/libs/enzyme/strutils.py b/libs/enzyme/strutils.py index b1df1d91..8578aefa 100644 --- a/libs/enzyme/strutils.py +++ b/libs/enzyme/strutils.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # enzyme - Video metadata parser -# Copyright (C) 2011 Antoine Bertin -# Copyright (C) 2006-2009 Dirk Meyer -# Copyright (C) 2006-2009 Jason Tackaberry +# Copyright 2011-2012 Antoine Bertin +# Copyright 2006-2009 Dirk Meyer +# Copyright 2006-2009 Jason Tackaberry # # This file is part of enzyme. # @@ -17,9 +17,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# - +# along with enzyme. If not, see . __all__ = ['ENCODING', 'str_to_unicode', 'unicode_to_str'] import locale diff --git a/libs/guessit/__init__.py b/libs/guessit/__init__.py index cb62db02..a86f71bb 100644 --- a/libs/guessit/__init__.py +++ b/libs/guessit/__init__.py @@ -18,10 +18,10 @@ # along with this program. If not, see . # -__version__ = '0.3-dev' -__all__ = [ 'Guess', 'Language', - 'guess_file_info', 'guess_video_info', - 'guess_movie_info', 'guess_episode_info' ] +__version__ = '0.3.1' +__all__ = ['Guess', 'Language', + 'guess_file_info', 'guess_video_info', + 'guess_movie_info', 'guess_episode_info'] from guessit.guess import Guess, merge_all @@ -31,6 +31,7 @@ import logging log = logging.getLogger("guessit") + class NullHandler(logging.Handler): def emit(self, record): pass @@ -40,9 +41,7 @@ h = NullHandler() log.addHandler(h) - - -def guess_file_info(filename, filetype, info = [ 'filename' ]): +def guess_file_info(filename, filetype, info=None): """info can contain the names of the various plugins, such as 'filename' to detect filename info, or 'hash_md5' to get the md5 hash of the file. @@ -52,27 +51,30 @@ def guess_file_info(filename, filetype, info = [ 'filename' ]): result = [] hashers = [] + if info is None: + info = ['filename'] + if isinstance(info, basestring): - info = [ info ] + info = [info] for infotype in info: if infotype == 'filename': - m = IterativeMatcher(filename, filetype = filetype) + m = IterativeMatcher(filename, filetype=filetype) result.append(m.matched()) elif infotype == 'hash_mpc': - import hash_mpc + from guessit.hash_mpc import hash_file try: - result.append(Guess({ 'hash_mpc': hash_mpc.hash_file(filename) }, - confidence = 1.0)) + result.append(Guess({'hash_mpc': hash_file(filename)}, + confidence=1.0)) except Exception, e: log.warning('Could not compute MPC-style hash because: %s' % e) elif infotype == 'hash_ed2k': - import hash_ed2k + from guessit.hash_ed2k import hash_file try: - result.append(Guess({ 'hash_ed2k': hash_ed2k.hash_file(filename) }, - confidence = 1.0)) + result.append(Guess({'hash_ed2k': hash_file(filename)}, + confidence=1.0)) except Exception, e: log.warning('Could not compute ed2k hash because: %s' % e) @@ -88,18 +90,6 @@ def guess_file_info(filename, filetype, info = [ 'filename' ]): else: log.warning('Invalid infotype: %s' % infotype) - - """For plugins which depend on some optional library, import them like that: - - if infotype == 'plugin_name': - try: - import optional_lib - except ImportError: - raise Exception, 'The plugin module cannot be loaded because the optional_lib lib is missing' - - # do some stuff - """ - # do all the hashes now, but on a single pass if hashers: try: @@ -112,22 +102,21 @@ def guess_file_info(filename, filetype, info = [ 'filename' ]): hasher.update(chunk) for infotype, hasher in hashers: - result.append(Guess({ infotype: hasher.hexdigest() }, - confidence = 1.0)) + result.append(Guess({infotype: hasher.hexdigest()}, + confidence=1.0)) except Exception, e: log.warning('Could not compute hash because: %s' % e) - return merge_all(result) -def guess_video_info(filename, info = [ 'filename' ]): +def guess_video_info(filename, info=None): return guess_file_info(filename, 'autodetect', info) -def guess_movie_info(filename, info = [ 'filename' ]): + +def guess_movie_info(filename, info=None): return guess_file_info(filename, 'movie', info) -def guess_episode_info(filename, info = [ 'filename' ]): + +def guess_episode_info(filename, info=None): return guess_file_info(filename, 'episode', info) - - diff --git a/libs/guessit/autodetect.py b/libs/guessit/autodetect.py deleted file mode 100644 index fe05b68b..00000000 --- a/libs/guessit/autodetect.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# GuessIt - A library for guessing information from filenames -# Copyright (c) 2011 Nicolas Wack -# -# GuessIt is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# GuessIt is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. -# -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . -# - -#from guessit import movie, episode -import os, os.path -import logging - -log = logging.getLogger('guessit.autodetect') - -def within(x, nrange): - """Return whether a number is inside a given range, specified as a list or tuple - of the lower and upper bounds.""" - low, high = nrange - return low <= x <= high - -def guess_filename_info(filename): - log.debug('Trying to guess info for file: ' + filename) - - # try to guess info as if it were an episode - episode_info = episode.guess_episode_filename(filename) - - # 1- if we found either season/episodeNumber, then we're pretty sure it must - # be an episode - if 'season' in episode_info or 'episodeNumber' in episode_info: - log.debug('Likely an episode as it contains season and/or episodeNumber: ' + filename) - episode_info.update({ 'type': 'episode' }, confidence = 0.9) - return episode_info - - # try to guess info as if it were a movie - movie_info = movie.guess_movie_filename(filename) - - # 2- if the file exists, try to guess its type using its size - if os.path.exists(filename): - size = os.stat(filename).st_size / (1024 * 1024) - - # if size <= 1/2 of 1CD -> episode (very unlikely a movie so small) - if size < 400: - log.debug('Likely an episode due to its small size (%dMB): %s' % (size, filename)) - episode_info.update({ 'type': 'episode' }, confidence = 0.8) - return episode_info - - # if size > 2G -> movie (even fullHD eps aren't that big yet) - if size > 2048: - log.debug('Likely a movie due to its big size (%dMB): %s' % (size, filename)) - movie_info.update({ 'type': 'movie' }, confidence = 0.8) - return movie_info - - # if size == 1CD or 2CDs -> movie - if within(size, [690, 710]) or within(size, [1380, 1420]): - log.debug('Likely a movie due to its size close to a CD size (%dMB): %s' % (size, filename)) - movie_info.update({ 'type': 'movie' }, confidence = 0.8) - return movie_info - - - # 3- if all else fails, assume it's a movie - log.debug('Couldn\'t make an informed guess... Assuming file is a movie: %s' % filename) - movie_info.update({ 'type': 'movie' }, confidence = 0.5) - return movie_info diff --git a/libs/guessit/date.py b/libs/guessit/date.py index 9f79daec..4d0b510e 100644 --- a/libs/guessit/date.py +++ b/libs/guessit/date.py @@ -21,6 +21,7 @@ import datetime import re + def search_year(string): """Looks for year patterns, and if found return the year and group span. Assumes there are sentinels at the beginning and end of the string that @@ -62,34 +63,35 @@ def search_date(string): dsep = r'[-/ \.]' - date_rexps = [ # 20010823 - r'[^0-9]' + - r'(?P[0-9]{4})' + - r'(?P[0-9]{2})' + - r'(?P[0-9]{2})' + - r'[^0-9]', + date_rexps = [ + # 20010823 + r'[^0-9]' + + r'(?P[0-9]{4})' + + r'(?P[0-9]{2})' + + r'(?P[0-9]{2})' + + r'[^0-9]', - # 2001-08-23 - r'[^0-9]' + - r'(?P[0-9]{4})' + dsep + - r'(?P[0-9]{2})' + dsep + - r'(?P[0-9]{2})' + - r'[^0-9]', + # 2001-08-23 + r'[^0-9]' + + r'(?P[0-9]{4})' + dsep + + r'(?P[0-9]{2})' + dsep + + r'(?P[0-9]{2})' + + r'[^0-9]', - # 23-08-2001 - r'[^0-9]' + - r'(?P[0-9]{2})' + dsep + - r'(?P[0-9]{2})' + dsep + - r'(?P[0-9]{4})' + - r'[^0-9]', + # 23-08-2001 + r'[^0-9]' + + r'(?P[0-9]{2})' + dsep + + r'(?P[0-9]{2})' + dsep + + r'(?P[0-9]{4})' + + r'[^0-9]', - # 23-08-01 - r'[^0-9]' + - r'(?P[0-9]{2})' + dsep + - r'(?P[0-9]{2})' + dsep + - r'(?P[0-9]{2})' + - r'[^0-9]', - ] + # 23-08-01 + r'[^0-9]' + + r'(?P[0-9]{2})' + dsep + + r'(?P[0-9]{2})' + dsep + + r'(?P[0-9]{2})' + + r'[^0-9]', + ] for drexp in date_rexps: match = re.search(drexp, string) @@ -98,7 +100,7 @@ def search_date(string): year, month, day = int(d['year']), int(d['month']), int(d['day']) # years specified as 2 digits should be adjusted here if year < 100: - if year > (datetime.date.today().year % 100)+ 5: + if year > (datetime.date.today().year % 100) + 5: year = 1900 + year else: year = 2000 + year @@ -120,8 +122,9 @@ def search_date(string): continue # looks like we have a valid date - # note: span is [+1,-1] because we don't want to include the non-digit char + # note: span is [+1,-1] because we don't want to include the + # non-digit char start, end = match.span() - return (date, (start+1, end-1)) + return (date, (start + 1, end - 1)) return None, None diff --git a/libs/guessit/filetype.py b/libs/guessit/filetype.py deleted file mode 100644 index fdc9d695..00000000 --- a/libs/guessit/filetype.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# GuessIt - A library for guessing information from filenames -# Copyright (c) 2011 Nicolas Wack -# -# GuessIt is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# GuessIt is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. -# -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . -# - -from guessit.patterns import subtitle_exts, video_exts, episode_rexps, find_properties, canonical_form -import os.path -import re -import logging - -log = logging.getLogger("guessit.filetype") - - -def guess_filetype(filename, filetype = 'autodetect'): - other = {} - - # look at the extension first - fileext = os.path.splitext(filename)[1][1:].lower() - if fileext in subtitle_exts: - if 'movie' in filetype: - filetype = 'moviesubtitle' - elif 'episode' in filetype: - filetype = 'episodesubtitle' - else: - filetype = 'subtitle' - other = { 'container': fileext } - elif fileext in video_exts: - if filetype == 'autodetect': - filetype = 'video' - other = { 'container': fileext } - else: - if filetype == 'autodetect': - filetype = 'unknown' - other = { 'extension': fileext } - - # now look whether there are some specific hints for episode vs movie - if filetype in ('video', 'subtitle'): - for rexp, confidence, span_adjust in episode_rexps: - match = re.search(rexp, filename, re.IGNORECASE) - if match: - if filetype == 'video': - filetype = 'episode' - elif filetype == 'subtitle': - filetype = 'episodesubtitle' - break - - for prop, value, start, end in find_properties(filename): - if canonical_form(value) == 'DVB': - if filetype == 'video': - filetype = 'episode' - elif filetype == 'subtitle': - filetype = 'episodesubtitle' - break - - # if no episode info found, assume it's a movie - if filetype == 'video': - filetype = 'movie' - elif filetype == 'subtitle': - filetype = 'moviesubtitle' - - return filetype, other diff --git a/libs/guessit/fileutils.py b/libs/guessit/fileutils.py index ceefee1e..13d12c51 100644 --- a/libs/guessit/fileutils.py +++ b/libs/guessit/fileutils.py @@ -50,11 +50,11 @@ def split_path(path): # on Unix systems, the root folder is '/' if head == '/' and tail == '': - return [ '/' ] + result + return ['/'] + result # on Windows, the root folder is a drive letter (eg: 'C:\') if len(head) == 3 and head[1:] == ':\\' and tail == '': - return [ head ] + result + return [head] + result if head == '' and tail == '': return result @@ -64,17 +64,10 @@ def split_path(path): path = head continue - result = [ tail ] + result + result = [tail] + result path = head -def split_path_components(filename): - """Returns the filename split into [ dir*, basename, ext ].""" - result = split_path(filename) - basename = result.pop(-1) - return result + list(os.path.splitext(basename)) - - def file_in_same_dir(ref_file, desired_file): """Return the path for a file in the same dir as a given reference file. @@ -82,17 +75,17 @@ def file_in_same_dir(ref_file, desired_file): '~/smewt/smewt.settings' """ - return os.path.join(*(split_path(ref_file)[:-1] + [ desired_file ])) + return os.path.join(*(split_path(ref_file)[:-1] + [desired_file])) def load_file_in_same_dir(ref_file, filename): """Load a given file. Works even when the file is contained inside a zip.""" - path = split_path(ref_file)[:-1] + [ filename ] + path = split_path(ref_file)[:-1] + [filename] for i, p in enumerate(path): if p.endswith('.zip'): - zfilename = os.path.join(*path[:i+1]) + zfilename = os.path.join(*path[:i + 1]) zfile = zipfile.ZipFile(zfilename) - return zfile.read('/'.join(path[i+1:])) + return zfile.read('/'.join(path[i + 1:])) return open(os.path.join(*path)).read() diff --git a/libs/guessit/guess.py b/libs/guessit/guess.py index 92ec4bea..9950a12e 100644 --- a/libs/guessit/guess.py +++ b/libs/guessit/guess.py @@ -26,9 +26,12 @@ log = logging.getLogger("guessit.guess") class Guess(dict): - """A Guess is a dictionary which has an associated confidence for each of its values. + """A Guess is a dictionary which has an associated confidence for each of + its values. + + As it is a subclass of dict, you can use it everywhere you expect a + simple dict.""" - As it is a subclass of dict, you can use it everywhere you expect a simple dict""" def __init__(self, *args, **kwargs): try: confidence = kwargs.pop('confidence') @@ -52,20 +55,20 @@ class Guess(dict): elif isinstance(value, unicode): data[prop] = value.encode('utf-8') elif isinstance(value, list): - data[prop] = [ str(x) for x in value ] + data[prop] = [str(x) for x in value] return data def nice_string(self): data = self.to_utf8_dict() - parts = json.dumps(data, indent = 4).split('\n') + parts = json.dumps(data, indent=4).split('\n') for i, p in enumerate(parts): if p[:5] != ' "': continue prop = p.split('"')[1] - parts[i] = (' [%.2f] "' % (self._confidence.get(prop) or -1)) + p[5:] + parts[i] = (' [%.2f] "' % self.confidence(prop)) + p[5:] return '\n'.join(parts) @@ -73,9 +76,9 @@ class Guess(dict): return str(self.to_utf8_dict()) def confidence(self, prop): - return self._confidence[prop] + return self._confidence.get(prop, -1) - def set(self, prop, value, confidence = None): + def set(self, prop, value, confidence=None): self[prop] = value if confidence is not None: self._confidence[prop] = confidence @@ -83,7 +86,7 @@ class Guess(dict): def set_confidence(self, prop, value): self._confidence[prop] = value - def update(self, other, confidence = None): + def update(self, other, confidence=None): dict.update(self, other) if isinstance(other, Guess): for prop in other: @@ -94,36 +97,36 @@ class Guess(dict): self._confidence[prop] = confidence def update_highest_confidence(self, other): - """Update this guess with the values from the given one. In case there is - property present in both, only the one with the highest one is kept.""" + """Update this guess with the values from the given one. In case + there is property present in both, only the one with the highest one + is kept.""" if not isinstance(other, Guess): - raise ValueError, 'Can only call this function on Guess instances' + raise ValueError('Can only call this function on Guess instances') for prop in other: - if prop in self and self._confidence[prop] >= other._confidence[prop]: + if prop in self and self.confidence(prop) >= other.confidence(prop): continue self[prop] = other[prop] - self._confidence[prop] = other._confidence[prop] - - + self._confidence[prop] = other.confidence(prop) def choose_int(g1, g2): - """Function used by merge_similar_guesses to choose between 2 possible properties - when they are integers.""" + """Function used by merge_similar_guesses to choose between 2 possible + properties when they are integers.""" v1, c1 = g1 # value, confidence v2, c2 = g2 if (v1 == v2): - return (v1, 1 - (1-c1)*(1-c2)) + return (v1, 1 - (1 - c1) * (1 - c2)) else: if c1 > c2: return (v1, c1 - c2) else: return (v2, c2 - c1) + def choose_string(g1, g2): - """Function used by merge_similar_guesses to choose between 2 possible properties - when they are strings. + """Function used by merge_similar_guesses to choose between 2 possible + properties when they are strings. If the 2 strings are similar, or one is contained in the other, the latter is returned with an increased confidence. @@ -142,7 +145,7 @@ def choose_string(g1, g2): ('Hello', 0.75) >>> choose_string(('Hello', 0.4), ('Hello World', 0.4)) - ('Hello', 0.64000000000000001) + ('Hello', 0.64) >>> choose_string(('simpsons', 0.5), ('The Simpsons', 0.5)) ('The Simpsons', 0.75) @@ -159,7 +162,7 @@ def choose_string(g1, g2): v1, v2 = v1.strip(), v2.strip() v1l, v2l = v1.lower(), v2.lower() - combined_prob = 1 - (1-c1)*(1-c2) + combined_prob = 1 - (1 - c1) * (1 - c2) if v1l == v2l: return (v1, combined_prob) @@ -191,26 +194,31 @@ def _merge_similar_guesses_nocheck(guesses, prop, choose): This function assumes there are at least 2 valid guesses.""" - similar = [ guess for guess in guesses if prop in guess ] + similar = [guess for guess in guesses if prop in guess] g1, g2 = similar[0], similar[1] other_props = set(g1) & set(g2) - set([prop]) if other_props: + log.debug('guess 1: %s' % g1) + log.debug('guess 2: %s' % g2) for prop in other_props: if g1[prop] != g2[prop]: - log.warning('both guesses to be merged have more than one different property in common, bailing out...') + log.warning('both guesses to be merged have more than one ' + 'different property in common, bailing out...') return - # merge all props of s2 into s1, updating the confidence for the considered property + # merge all props of s2 into s1, updating the confidence for the + # considered property v1, v2 = g1[prop], g2[prop] c1, c2 = g1.confidence(prop), g2.confidence(prop) new_value, new_confidence = choose((v1, c1), (v2, c2)) if new_confidence >= c1: - log.debug("Updating matching property '%s' with confidence %.2f" % (prop, new_confidence)) + msg = "Updating matching property '%s' with confidence %.2f" else: - log.debug("Updating non-matching property '%s' with confidence %.2f" % (prop, new_confidence)) + msg = "Updating non-matching property '%s' with confidence %.2f" + log.debug(msg % (prop, new_confidence)) g2[prop] = new_value g2.set_confidence(prop, new_confidence) @@ -218,12 +226,13 @@ def _merge_similar_guesses_nocheck(guesses, prop, choose): g1.update(g2) guesses.remove(g2) + def merge_similar_guesses(guesses, prop, choose): """Take a list of guesses and merge those which have the same properties, increasing or decreasing the confidence depending on whether their values are similar.""" - similar = [ guess for guess in guesses if prop in guess ] + similar = [guess for guess in guesses if prop in guess] if len(similar) < 2: # nothing to merge return @@ -233,9 +242,13 @@ def merge_similar_guesses(guesses, prop, choose): if len(similar) > 2: log.debug('complex merge, trying our best...') + before = len(guesses) _merge_similar_guesses_nocheck(guesses, prop, choose) - merge_similar_guesses(guesses, prop, choose) - return + after = len(guesses) + if after < before: + # recurse only when the previous call actually did something, + # otherwise we end up in an infinite loop + merge_similar_guesses(guesses, prop, choose) def merge_append_guesses(guesses, prop): @@ -245,14 +258,12 @@ def merge_append_guesses(guesses, prop): DEPRECATED, remove with old guessers """ - - - similar = [ guess for guess in guesses if prop in guess ] + similar = [guess for guess in guesses if prop in guess] if not similar: return merged = similar[0] - merged[prop] = [ merged[prop] ] + merged[prop] = [merged[prop]] # TODO: what to do with global confidence? mean of them all? for m in similar[1:]: @@ -261,17 +272,18 @@ def merge_append_guesses(guesses, prop): merged[prop].append(m[prop]) else: if prop2 in m: - log.warning('overwriting property "%s" with value ' % (prop2, m[prop2])) + log.warning('overwriting property "%s" with value %s' % (prop2, m[prop2])) merged[prop2] = m[prop2] # TODO: confidence also guesses.remove(m) -def merge_all(guesses, append = []): - """Merges all the guesses in a single result, removes very unlikely values, and returns it. - You can specify a list of properties that should be appended into a list instead of being - merged. +def merge_all(guesses, append=None): + """Merge all the guesses in a single result, remove very unlikely values, + and return it. + You can specify a list of properties that should be appended into a list + instead of being merged. >>> merge_all([ Guess({ 'season': 2 }, confidence = 0.6), ... Guess({ 'episodeNumber': 13 }, confidence = 0.8) ]) @@ -286,20 +298,24 @@ def merge_all(guesses, append = []): return Guess() result = guesses[0] + if append is None: + append = [] for g in guesses[1:]: # first append our appendable properties for prop in append: if prop in g: - result.set(prop, result.get(prop, []) + [ g[prop] ], - # TODO: what to do with confidence here? maybe an arithmetic mean... - confidence = g.confidence(prop)) + result.set(prop, result.get(prop, []) + [g[prop]], + # TODO: what to do with confidence here? maybe an + # arithmetic mean... + confidence=g.confidence(prop)) del g[prop] # then merge the remaining ones - if set(result) & set(g): - log.warning('duplicate properties %s in merged result...' % (set(result) & set(g))) + dups = set(result) & set(g) + if dups: + log.warning('duplicate properties %s in merged result...' % dups) result.update_highest_confidence(g) @@ -314,4 +330,3 @@ def merge_all(guesses, append = []): result[prop] = list(set(result[prop])) return result - diff --git a/libs/guessit/hash_ed2k.py b/libs/guessit/hash_ed2k.py index eb1a4ea6..43303c58 100644 --- a/libs/guessit/hash_ed2k.py +++ b/libs/guessit/hash_ed2k.py @@ -18,8 +18,9 @@ # along with this program. If not, see . # -from guessit import Guess -import hashlib, os.path +import hashlib +import os.path + def hash_file(filename): """Returns the ed2k hash of a given file. @@ -31,6 +32,7 @@ def hash_file(filename): os.path.getsize(filename), hash_filehash(filename).upper()) + def hash_filehash(filename): """Returns the ed2k hash of a given file. @@ -42,8 +44,10 @@ def hash_filehash(filename): def gen(f): while True: x = f.read(9728000) - if x: yield x - else: return + if x: + yield x + else: + return def md4_hash(data): m = md4() @@ -55,4 +59,5 @@ def hash_filehash(filename): hashes = [md4_hash(data).digest() for data in a] if len(hashes) == 1: return hashes[0].encode("hex") - else: return md4_hash(reduce(lambda a,d: a + d, hashes, "")).hexd + else: + return md4_hash(reduce(lambda a, d: a + d, hashes, "")).hexd diff --git a/libs/guessit/hash_mpc.py b/libs/guessit/hash_mpc.py index 2023f3d5..7475940a 100644 --- a/libs/guessit/hash_mpc.py +++ b/libs/guessit/hash_mpc.py @@ -18,8 +18,9 @@ # along with this program. If not, see . # -from guessit import Guess -import struct, os +import struct +import os + def hash_file(filename): """This function is taken from: @@ -32,25 +33,24 @@ def hash_file(filename): f = open(filename, "rb") filesize = os.path.getsize(filename) - hash = filesize + hash_value = filesize if filesize < 65536 * 2: - raise Exception, "SizeError: size is %d, should be > 132K..." % filesize + raise Exception("SizeError: size is %d, should be > 132K..." % filesize) - for x in range(65536/bytesize): - buffer = f.read(bytesize) - (l_value,)= struct.unpack(longlongformat, buffer) - hash += l_value - hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number + for x in range(65536 / bytesize): + buf = f.read(bytesize) + (l_value,) = struct.unpack(longlongformat, buf) + hash_value += l_value + hash_value = hash_value & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number - - f.seek(max(0,filesize-65536),0) - for x in range(65536/bytesize): - buffer = f.read(bytesize) - (l_value,)= struct.unpack(longlongformat, buffer) - hash += l_value - hash = hash & 0xFFFFFFFFFFFFFFFF + f.seek(max(0, filesize - 65536), 0) + for x in range(65536 / bytesize): + buf = f.read(bytesize) + (l_value,) = struct.unpack(longlongformat, buf) + hash_value += l_value + hash_value = hash_value & 0xFFFFFFFFFFFFFFFF f.close() - returnedhash = "%016x" % hash - return returnedhash + + return "%016x" % hash_value diff --git a/libs/guessit/language.py b/libs/guessit/language.py index 51cd5461..777d0e25 100644 --- a/libs/guessit/language.py +++ b/libs/guessit/language.py @@ -19,28 +19,28 @@ # from guessit import fileutils -import os.path -import re import logging log = logging.getLogger('guessit.language') - - # downloaded from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt # # Description of the fields: # "An alpha-3 (bibliographic) code, an alpha-3 (terminologic) code (when given), # an alpha-2 code (when given), an English name, and a French name of a language # are all separated by pipe (|) characters." +_iso639_contents = fileutils.load_file_in_same_dir(__file__, + 'ISO-639-2_utf-8.txt') language_matrix = [ l.strip().decode('utf-8').split('|') - for l in fileutils.load_file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt').split('\n') ] + for l in _iso639_contents.split('\n') ] -lng3 = frozenset(filter(bool, (l[0] for l in language_matrix))) -lng3term = frozenset(filter(bool, (l[1] for l in language_matrix))) -lng2 = frozenset(filter(bool, (l[2] for l in language_matrix))) -lng_en_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[3].lower().split('; ')))) -lng_fr_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[4].lower().split('; ')))) +lng3 = frozenset(l[0] for l in language_matrix if l[0]) +lng3term = frozenset(l[1] for l in language_matrix if l[1]) +lng2 = frozenset(l[2] for l in language_matrix if l[2]) +lng_en_name = frozenset(lng for l in language_matrix + for lng in l[3].lower().split('; ') if lng) +lng_fr_name = frozenset(lng for l in language_matrix + for lng in l[4].lower().split('; ') if lng) lng_all_names = lng3 | lng3term | lng2 | lng_en_name | lng_fr_name lng3_to_lng3term = dict((l[0], l[1]) for l in language_matrix if l[1]) @@ -50,22 +50,29 @@ lng3_to_lng2 = dict((l[0], l[2]) for l in language_matrix if l[2]) lng2_to_lng3 = dict((l[2], l[0]) for l in language_matrix if l[2]) # we only return the first given english name, hoping it is the most used one -lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0]) for l in language_matrix if l[3]) -lng_en_name_to_lng3 = dict((en_name.lower(), l[0]) for l in language_matrix if l[3] for en_name in l[3].split('; ')) +lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0]) + for l in language_matrix if l[3]) +lng_en_name_to_lng3 = dict((en_name.lower(), l[0]) + for l in language_matrix if l[3] + for en_name in l[3].split('; ')) # we only return the first given french name, hoping it is the most used one -lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0]) for l in language_matrix if l[4]) -lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0]) for l in language_matrix if l[4] for fr_name in l[4].split('; ')) +lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0]) + for l in language_matrix if l[4]) +lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0]) + for l in language_matrix if l[4] + for fr_name in l[4].split('; ')) def is_language(language): return language.lower() in lng_all_names + class Language(object): """This class represents a human language. - You can initialize it with pretty much everything, as it knows conversion from - ISO-639 2-letter and 3-letter codes, English and French names. + You can initialize it with pretty much everything, as it knows conversion + from ISO-639 2-letter and 3-letter codes, English and French names. >>> Language('fr') Language(French) @@ -79,12 +86,16 @@ class Language(object): if len(language) == 2: lang = lng2_to_lng3.get(language) elif len(language) == 3: - lang = language if language in lng3 else lng3term_to_lng3.get(language) + lang = (language + if language in lng3 + else lng3term_to_lng3.get(language)) else: - lang = lng_en_name_to_lng3.get(language) or lng_fr_name_to_lng3.get(language) + lang = (lng_en_name_to_lng3.get(language) or + lng_fr_name_to_lng3.get(language)) if lang is None: - raise ValueError, 'The given string "%s" could not be identified as a language' % language + msg = 'The given string "%s" could not be identified as a language' + raise ValueError(msg % language) self.lang = lang @@ -103,7 +114,6 @@ class Language(object): def french_name(self): return lng3_to_lng_fr_name[self.lang] - def __hash__(self): return hash(self.lang) @@ -132,19 +142,15 @@ class Language(object): return 'Language(%s)' % self - -def search_language(string, lang_filter = None): +def search_language(string, lang_filter=None): """Looks for language patterns, and if found return the language object, its group span and an associated confidence. you can specify a list of allowed languages using the lang_filter argument, as in lang_filter = [ 'fr', 'eng', 'spanish' ] - Assumes there are sentinels at the beginning and end of the string that - always allow matching a non-letter delimiting the language. - >>> search_language('movie [en].avi') - (Language(English), (7, 9), 0.80000000000000004) + (Language(English), (7, 9), 0.8) >>> search_language('the zen fat cat and the gay mad men got a new fan', lang_filter = ['en', 'fr', 'es']) (None, None, None) @@ -153,25 +159,27 @@ def search_language(string, lang_filter = None): # list of common words which could be interpreted as languages, but which # are far too common to be able to say they represent a language in the # middle of a string (where they most likely carry their commmon meaning) - lng_common_words = frozenset([ # english words - 'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to', - 'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan', - 'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as', - 'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr', - # french words - 'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que', - 'mal', 'est', 'vol', 'or', 'mon', 'se', - # spanish words - 'la', 'el', 'del', 'por', 'mar', - # other - 'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san' - ]) + lng_common_words = frozenset([ + # english words + 'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to', + 'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan', + 'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as', + 'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr', + # french words + 'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que', + 'mal', 'est', 'vol', 'or', 'mon', 'se', + # spanish words + 'la', 'el', 'del', 'por', 'mar', + # other + 'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san', 'oss', 'iii', + 'vi' + ]) sep = r'[](){} \._-+' if lang_filter: lang_filter = set(Language(l) for l in lang_filter) - slow = string.lower() + slow = ' %s ' % string.lower() confidence = 1.0 # for all of them for lang in lng_all_names: @@ -183,7 +191,7 @@ def search_language(string, lang_filter = None): if pos != -1: end = pos + len(lang) # make sure our word is always surrounded by separators - if slow[pos-1] not in sep or slow[end] not in sep: + if slow[pos - 1] not in sep or slow[end] not in sep: continue language = Language(slow[pos:end]) @@ -201,10 +209,11 @@ def search_language(string, lang_filter = None): elif len(lang) == 3: confidence = 0.9 else: - # Note: we could either be really confident that we found a language - # or assume that full language names are too common words + # Note: we could either be really confident that we found a + # language or assume that full language names are too + # common words confidence = 0.3 # going with the low-confidence route here - return language, (pos, end), confidence + return language, (pos - 1, end - 1), confidence return None, None, None diff --git a/libs/guessit/matcher.py b/libs/guessit/matcher.py index 63e25987..cac172de 100644 --- a/libs/guessit/matcher.py +++ b/libs/guessit/matcher.py @@ -2,8 +2,7 @@ # -*- coding: utf-8 -*- # # GuessIt - A library for guessing information from filenames -# Copyright (c) 2011 Nicolas Wack -# Copyright (c) 2011 Ricard Marxer +# Copyright (c) 2012 Nicolas Wack # # GuessIt is free software; you can redistribute it and/or modify it under # the terms of the Lesser GNU General Public License as published by @@ -19,282 +18,18 @@ # along with this program. If not, see . # -from guessit import fileutils, textutils -from guessit.guess import Guess, merge_similar_guesses, merge_all, choose_int, choose_string -from guessit.date import search_date, search_year -from guessit.language import search_language -from guessit.filetype import guess_filetype -from guessit.patterns import video_exts, subtitle_exts, sep, deleted, video_rexps, websites, episode_rexps, weak_episode_rexps, non_episode_title, find_properties, canonical_form, unlikely_series -from guessit.matchtree import get_group, find_group, leftover_valid_groups, tree_to_string -from guessit.textutils import find_first_level_groups, split_on_groups, blank_region, clean_string, to_utf8 -from guessit.fileutils import split_path_components -import datetime -import os.path -import re +from guessit.matchtree import MatchTree +from guessit.textutils import to_utf8 +from guessit.guess import (merge_similar_guesses, merge_all, + choose_int, choose_string) import copy import logging -import mimetypes log = logging.getLogger("guessit.matcher") - -def split_explicit_groups(string): - """return the string split into explicit groups, that is, those either - between parenthese, square brackets or curly braces, and those separated - by a dash.""" - result = find_first_level_groups(string, '()') - result = reduce(lambda l, x: l + find_first_level_groups(x, '[]'), result, []) - result = reduce(lambda l, x: l + find_first_level_groups(x, '{}'), result, []) - # do not do this at this moment, it is not strong enough and can break other - # patterns, such as dates, etc... - #result = reduce(lambda l, x: l + x.split('-'), result, []) - - return result - - -def format_guess(guess): - """Format all the found values to their natural type. - For instance, a year would be stored as an int value, etc... - - Note that this modifies the dictionary given as input. - """ - for prop, value in guess.items(): - if prop in ('season', 'episodeNumber', 'year', 'cdNumber', 'cdNumberTotal'): - guess[prop] = int(guess[prop]) - elif isinstance(value, basestring): - if prop in ('edition',): - value = clean_string(value) - guess[prop] = canonical_form(value) - - return guess - - -def guess_groups(string, result, filetype): - # add sentinels so we can match a separator char at either end of - # our groups, even when they are at the beginning or end of the string - # we will adjust the span accordingly later - # - # filetype can either be movie, moviesubtitle, episode, episodesubtitle - current = ' ' + string + ' ' - - regions = [] # list of (start, end) of matched regions - - def guessed(match_dict, confidence): - guess = format_guess(Guess(match_dict, confidence = confidence)) - result.append(guess) - log.debug('Found with confidence %.2f: %s' % (confidence, guess)) - return guess - - def update_found(string, guess, span, span_adjust = (0,0)): - span = (span[0] + span_adjust[0], - span[1] + span_adjust[1]) - regions.append((span, guess)) - return blank_region(string, span) - - # try to find dates first, as they are very specific - date, span = search_date(current) - if date: - guess = guessed({ 'date': date }, confidence = 1.0) - current = update_found(current, guess, span) - - # for non episodes only, look for year information - if filetype not in ('episode', 'episodesubtitle'): - year, span = search_year(current) - if year: - guess = guessed({ 'year': year }, confidence = 1.0) - current = update_found(current, guess, span) - - # specific regexps (ie: cd number, season X episode, ...) - for rexp, confidence, span_adjust in video_rexps: - match = re.search(rexp, current, re.IGNORECASE) - if match: - metadata = match.groupdict() - # is this the better place to put it? (maybe, as it is at least the soonest that we can catch it) - if 'cdNumberTotal' in metadata and metadata['cdNumberTotal'] is None: - del metadata['cdNumberTotal'] - - guess = guessed(metadata, confidence = confidence) - current = update_found(current, guess, match.span(), span_adjust) - - if filetype in ('episode', 'episodesubtitle'): - for rexp, confidence, span_adjust in episode_rexps: - match = re.search(rexp, current, re.IGNORECASE) - if match: - metadata = match.groupdict() - guess = guessed(metadata, confidence = confidence) - current = update_found(current, guess, match.span(), span_adjust) - - - # Now websites, but as exact string instead of regexps - clow = current.lower() - for site in websites: - pos = clow.find(site.lower()) - if pos != -1: - guess = guessed({ 'website': site }, confidence = confidence) - current = update_found(current, guess, (pos, pos+len(site))) - clow = current.lower() - - - # release groups have certain constraints, cannot be included in the previous general regexps - group_names = [ r'\.(Xvid)-(?P.*?)[ \.]', - r'\.(DivX)-(?P.*?)[\. ]', - r'\.(DVDivX)-(?P.*?)[\. ]', - ] - for rexp in group_names: - match = re.search(rexp, current, re.IGNORECASE) - if match: - metadata = match.groupdict() - metadata.update({ 'videoCodec': match.group(1) }) - guess = guessed(metadata, confidence = 0.8) - current = update_found(current, guess, match.span(), span_adjust = (1, -1)) - - - # common well-defined words and regexps - confidence = 1.0 # for all of them - for prop, value, pos, end in find_properties(current): - guess = guessed({ prop: value }, confidence = confidence) - current = update_found(current, guess, (pos, end)) - - - # weak guesses for episode number, only run it if we don't have an estimate already - if filetype in ('episode', 'episodesubtitle'): - if not any('episodeNumber' in match for match in result): - for rexp, _, span_adjust in weak_episode_rexps: - match = re.search(rexp, current, re.IGNORECASE) - if match: - metadata = match.groupdict() - epnum = int(metadata['episodeNumber']) - if epnum > 100: - guess = guessed({ 'season': epnum // 100, - 'episodeNumber': epnum % 100 }, confidence = 0.6) - else: - guess = guessed(metadata, confidence = 0.3) - current = update_found(current, guess, match.span(), span_adjust) - - # try to find languages now - language, span, confidence = search_language(current) - while language: - # is it a subtitle language? - if 'sub' in clean_string(current[:span[0]]).lower().split(' '): - guess = guessed({ 'subtitleLanguage': language }, confidence = confidence) - else: - guess = guessed({ 'language': language }, confidence = confidence) - current = update_found(current, guess, span) - - language, span, confidence = search_language(current) - - - # remove our sentinels now and ajust spans accordingly - assert(current[0] == ' ' and current[-1] == ' ') - current = current[1:-1] - regions = [ ((start-1, end-1), guess) for (start, end), guess in regions ] - - # split into '-' separated subgroups (with required separator chars - # around the dash) - didx = current.find('-') - while didx > 0: - regions.append(((didx, didx), None)) - didx = current.find('-', didx+1) - - # cut our final groups, and rematch the guesses to the group that created - # id, None if it is a leftover group - region_spans = [ span for span, guess in regions ] - string_groups = split_on_groups(string, region_spans) - remaining_groups = split_on_groups(current, region_spans) - guesses = [] - - pos = 0 - for group in string_groups: - found = False - for span, guess in regions: - if span[0] == pos: - guesses.append(guess) - found = True - if not found: - guesses.append(None) - - pos += len(group) - - return zip(string_groups, - remaining_groups, - guesses) - - -def match_from_epnum_position(match_tree, epnum_pos, guessed, update_found): - """guessed is a callback function to call with the guessed group - update_found is a callback to update the match group and returns leftover groups.""" - pidx, eidx, gidx = epnum_pos - - # a few helper functions to be able to filter using high-level semantics - def same_pgroup_before(group): - _, (ppidx, eeidx, ggidx) = group - return ppidx == pidx and (eeidx, ggidx) < (eidx, gidx) - - def same_pgroup_after(group): - _, (ppidx, eeidx, ggidx) = group - return ppidx == pidx and (eeidx, ggidx) > (eidx, gidx) - - def same_egroup_before(group): - _, (ppidx, eeidx, ggidx) = group - return ppidx == pidx and eeidx == eidx and ggidx < gidx - - def same_egroup_after(group): - _, (ppidx, eeidx, ggidx) = group - return ppidx == pidx and eeidx == eidx and ggidx > gidx - - leftover = leftover_valid_groups(match_tree) - - # if we have at least 1 valid group before the episodeNumber, then it's probably - # the series name - series_candidates = filter(same_pgroup_before, leftover) - if len(series_candidates) >= 1: - guess = guessed({ 'series': series_candidates[0][0] }, confidence = 0.7) - leftover = update_found(leftover, series_candidates[0][1], guess) - - # only 1 group after (in the same path group) and it's probably the episode title - title_candidates = filter(lambda g:g[0].lower() not in non_episode_title, - filter(same_pgroup_after, leftover)) - if len(title_candidates) == 1: - guess = guessed({ 'title': title_candidates[0][0] }, confidence = 0.5) - leftover = update_found(leftover, title_candidates[0][1], guess) - else: - # try in the same explicit group, with lower confidence - title_candidates = filter(lambda g:g[0].lower() not in non_episode_title, - filter(same_egroup_after, leftover)) - if len(title_candidates) == 1: - guess = guessed({ 'title': title_candidates[0][0] }, confidence = 0.4) - leftover = update_found(leftover, title_candidates[0][1], guess) - - # epnumber is the first group and there are only 2 after it in same path group - # -> season title - episode title - already_has_title = (find_group(match_tree, 'title') != []) - - title_candidates = filter(lambda g:g[0].lower() not in non_episode_title, - filter(same_pgroup_after, leftover)) - if (not already_has_title and # no title - not filter(same_pgroup_before, leftover) and # no groups before - len(title_candidates) == 2): # only 2 groups after - - guess = guessed({ 'series': title_candidates[0][0] }, confidence = 0.4) - leftover = update_found(leftover, title_candidates[0][1], guess) - guess = guessed({ 'title': title_candidates[1][0] }, confidence = 0.4) - leftover = update_found(leftover, title_candidates[1][1], guess) - - - # if we only have 1 remaining valid group in the pathpart before the filename, - # then it's likely that it is the series name - series_candidates = [ group for group in leftover if group[1][0] == pidx-1 ] - if len(series_candidates) == 1: - guess = guessed({ 'series': series_candidates[0][0] }, confidence = 0.5) - leftover = update_found(leftover, series_candidates[0][1], guess) - - return match_tree - - - class IterativeMatcher(object): - def __init__(self, filename, filetype = 'autodetect'): + def __init__(self, filename, filetype='autodetect'): """An iterative matcher tries to match different patterns that appear in the filename. @@ -325,7 +60,7 @@ class IterativeMatcher(object): The first 3 lines indicates the group index in which a char in the filename is located. So for instance, x264 is the group (0, 4, 1), and it corresponds to a video codec, denoted by the letter'v' in the 4th line. - (for more info, see guess.matchtree.tree_to_string) + (for more info, see guess.matchtree.to_string) Second, it tries to merge all this information into a single object @@ -333,266 +68,89 @@ class IterativeMatcher(object): resolution when they arise. """ - if filetype not in ('autodetect', 'subtitle', 'video', + valid_filetypes = ('autodetect', 'subtitle', 'video', 'movie', 'moviesubtitle', - 'episode', 'episodesubtitle'): - raise ValueError, "filetype needs to be one of ('autodetect', 'subtitle', 'video', 'movie', 'moviesubtitle', 'episode', 'episodesubtitle')" + 'episode', 'episodesubtitle') + if filetype not in valid_filetypes: + raise ValueError("filetype needs to be one of %s" % valid_filetypes) if not isinstance(filename, unicode): log.debug('WARNING: given filename to matcher is not unicode...') - match_tree = [] - result = [] # list of found metadata - - def guessed(match_dict, confidence): - guess = format_guess(Guess(match_dict, confidence = confidence)) - result.append(guess) - log.debug('Found with confidence %.2f: %s' % (confidence, guess)) - return guess - - def update_found(leftover, group_pos, guess): - pidx, eidx, gidx = group_pos - group = match_tree[pidx][eidx][gidx] - match_tree[pidx][eidx][gidx] = (group[0], - deleted * len(group[0]), - guess) - return [ g for g in leftover if g[1] != group_pos ] + self.match_tree = MatchTree(filename) + mtree = self.match_tree + mtree.guess.set('type', filetype, confidence=1.0) + def apply_transfo(transfo_name, *args, **kwargs): + transfo = __import__('guessit.transfo.' + transfo_name, + globals=globals(), locals=locals(), + fromlist=['process'], level=-1) + transfo.process(mtree, *args, **kwargs) # 1- first split our path into dirs + basename + ext - match_tree = split_path_components(filename) + apply_transfo('split_path_components') - # try to detect the file type - filetype, other = guess_filetype(filename, filetype) - guessed({ 'type': filetype }, confidence = 1.0) - extguess = guessed(other, confidence = 1.0) + # 2- guess the file type now (will be useful later) + apply_transfo('guess_filetype', filetype) + if mtree.guess['type'] == 'unknown': + return - # guess the mimetype of the filename - # TODO: handle other mimetypes not found on the default type_maps - # mimetypes.types_map['.srt']='text/subtitle' - mime, _ = mimetypes.guess_type(filename, strict=False) - if mime is not None: - guessed({ 'mimetype': mime }, confidence = 1.0) + # 3- split each of those into explicit groups (separated by parentheses + # or square brackets) + apply_transfo('split_explicit_groups') - # remove the extension from the match tree, as all indices relative - # the the filename groups assume the basename is the last one - fileext = match_tree.pop(-1)[1:].lower() + # 4- try to match information for specific patterns + if mtree.guess['type'] in ('episode', 'episodesubtitle'): + strategy = ['guess_date', 'guess_video_rexps', + 'guess_episodes_rexps', 'guess_website', + 'guess_release_group', 'guess_properties', + 'guess_weak_episodes_rexps', 'guess_language'] + else: + strategy = ['guess_date', 'guess_year', 'guess_video_rexps', + 'guess_website', 'guess_release_group', + 'guess_properties', 'guess_language'] + for name in strategy: + apply_transfo(name) - # 2- split each of those into explicit groups, if any - # note: be careful, as this might split some regexps with more confidence such as - # Alfleni-Team, or [XCT] or split a date such as (14-01-2008) - match_tree = [ split_explicit_groups(part) for part in match_tree ] + # more guessers for both movies and episodes + for name in ['guess_bonus_features']: + apply_transfo(name) + # split into '-' separated subgroups (with required separator chars + # around the dash) + apply_transfo('split_on_dash') - # 3- try to match information in decreasing order of confidence and - # blank the matching group in the string if we found something - for pathpart in match_tree: - for gidx, explicit_group in enumerate(pathpart): - pathpart[gidx] = guess_groups(explicit_group, result, filetype = filetype) + # 5- try to identify the remaining unknown groups by looking at their + # position relative to other known elements + if mtree.guess['type'] in ('episode', 'episodesubtitle'): + apply_transfo('guess_episode_info_from_position') + else: + apply_transfo('guess_movie_title_from_position') - # 4- try to identify the remaining unknown groups by looking at their position - # relative to other known elements - - if filetype in ('episode', 'episodesubtitle'): - eps = find_group(match_tree, 'episodeNumber') - if eps: - match_tree = match_from_epnum_position(match_tree, eps[0], guessed, update_found) - - leftover = leftover_valid_groups(match_tree) - - if not eps: - # if we don't have the episode number, but at least 2 groups in the - # last path group, then it's probably series - eptitle - title_candidates = filter(lambda g:g[0].lower() not in non_episode_title, - filter(lambda g: g[1][0] == len(match_tree)-1, - leftover_valid_groups(match_tree))) - if len(title_candidates) >= 2: - guess = guessed({ 'series': title_candidates[0][0] }, confidence = 0.4) - leftover = update_found(leftover, title_candidates[0][1], guess) - guess = guessed({ 'title': title_candidates[1][0] }, confidence = 0.4) - leftover = update_found(leftover, title_candidates[1][1], guess) - - - # if there's a path group that only contains the season info, then the previous one - # is most likely the series title (ie: .../series/season X/...) - eps = [ gpos for gpos in find_group(match_tree, 'season') - if 'episodeNumber' not in get_group(match_tree, gpos)[2] ] - - if eps: - pidx, eidx, gidx = eps[0] - previous = [ group for group in leftover if group[1][0] == pidx - 1 ] - if len(previous) == 1: - guess = guessed({ 'series': previous[0][0] }, confidence = 0.5) - leftover = update_found(leftover, previous[0][1], guess) - - # reduce the confidence of unlikely series - for guess in result: - if 'series' in guess: - if guess['series'].lower() in unlikely_series: - guess.set_confidence('series', guess.confidence('series') * 0.5) - - - elif filetype in ('movie', 'moviesubtitle'): - leftover_all = leftover_valid_groups(match_tree) - - # specific cases: - # - movies/tttttt (yyyy)/tttttt.ccc - try: - if match_tree[-3][0][0][0].lower() == 'movies': - # Note:too generic, might solve all the unittests as they all contain 'movies' - # in their path - # - #if len(match_tree[-2][0]) == 1: - # title = match_tree[-2][0][0] - # guess = guessed({ 'title': clean_string(title[0]) }, confidence = 0.7) - # update_found(leftover_all, title, guess) - - year_group = filter(lambda gpos: gpos[0] == len(match_tree)-2, - find_group(match_tree, 'year'))[0] - leftover = leftover_valid_groups(match_tree, - valid = lambda g: ((g[0] and g[0][0] not in sep) and - g[1][0] == len(match_tree) - 2)) - if len(match_tree[-2]) == 2 and year_group[1] == 1: - title = leftover[0] - guess = guessed({ 'title': clean_string(title[0]) }, - confidence = 0.8) - update_found(leftover_all, title[1], guess) - raise Exception # to exit the try catch now - - leftover = [ g for g in leftover_all if (g[1][0] == year_group[0] and - g[1][1] < year_group[1] and - g[1][2] < year_group[2]) ] - leftover = sorted(leftover, key = lambda x:x[1]) - title = leftover[0] - guess = guessed({ 'title': title[0] }, confidence = 0.8) - leftover = update_found(leftover, title[1], guess) - except: - pass - - # if we have either format or videoCodec in the folder containing the file - # or one of its parents, then we should probably look for the title in - # there rather than in the basename - props = filter(lambda g: g[0] <= len(match_tree) - 2, - find_group(match_tree, 'videoCodec') + - find_group(match_tree, 'format') + - find_group(match_tree, 'language')) - leftover = None - if props and all(g[0] == props[0][0] for g in props): - leftover = [ g for g in leftover_all if g[1][0] == props[0][0] ] - - if props and leftover: - guess = guessed({ 'title': leftover[0][0] }, confidence = 0.7) - leftover = update_found(leftover, leftover[0][1], guess) - - else: - # first leftover group in the last path part sounds like a good candidate for title, - # except if it's only one word and that the first group before has at least 3 words in it - # (case where the filename contains an 8 chars short name and the movie title is - # actually in the parent directory name) - leftover = [ g for g in leftover_all if g[1][0] == len(match_tree)-1 ] - if leftover: - title, (pidx, eidx, gidx) = leftover[0] - previous_pgroup_leftover = filter(lambda g: g[1][0] == pidx-1, leftover_all) - - if (title.count(' ') == 0 and - previous_pgroup_leftover and - previous_pgroup_leftover[0][0].count(' ') >= 2): - - guess = guessed({ 'title': previous_pgroup_leftover[0][0] }, confidence = 0.6) - leftover = update_found(leftover, previous_pgroup_leftover[0][1], guess) - - else: - guess = guessed({ 'title': title }, confidence = 0.6) - leftover = update_found(leftover, leftover[0][1], guess) - else: - # if there were no leftover groups in the last path part, look in the one before that - previous_pgroup_leftover = filter(lambda g: g[1][0] == len(match_tree)-2, leftover_all) - if previous_pgroup_leftover: - guess = guessed({ 'title': previous_pgroup_leftover[0][0] }, confidence = 0.6) - leftover = update_found(leftover, previous_pgroup_leftover[0][1], guess) - - - - - - - # 5- perform some post-processing steps - - # 5.1- try to promote language to subtitle language where it makes sense - for pidx, eidx, gidx in find_group(match_tree, 'language'): - string, remaining, guess = get_group(match_tree, (pidx, eidx, gidx)) - - def promote_subtitle(): - guess.set('subtitleLanguage', guess['language'], confidence = guess.confidence('language')) - del guess['language'] - - # - if we matched a language in a file with a sub extension and that the group - # is the last group of the filename, it is probably the language of the subtitle - # (eg: 'xxx.english.srt') - if (fileext in subtitle_exts and - pidx == len(match_tree) - 1 and - eidx == len(match_tree[pidx]) - 1): - promote_subtitle() - - # - if a language is in an explicit group just preceded by "st", it is a subtitle - # language (eg: '...st[fr-eng]...') - if eidx > 0: - previous = get_group(match_tree, (pidx, eidx-1, -1)) - if previous[0][-2:].lower() == 'st': - promote_subtitle() - - - - # re-append the extension now - match_tree.append([[(fileext, deleted*len(fileext), extguess)]]) - - self.parts = result - self.match_tree = match_tree - - if filename.startswith('/'): - filename = ' ' + filename - - log.debug('Found match tree:\n%s\n%s' % (to_utf8(tree_to_string(match_tree)), - to_utf8(filename))) + # 6- perform some post-processing steps + apply_transfo('post_process') + log.debug('Found match tree:\n%s' % (to_utf8(unicode(mtree)))) def matched(self): # we need to make a copy here, as the merge functions work in place and # calling them on the match tree would modify it - parts = copy.deepcopy(self.parts) - # 1- start by doing some common preprocessing tasks + parts = [node.guess for node in self.match_tree.nodes() if node.guess] + parts = copy.deepcopy(parts) - # 1.1- ", the" at the end of a series title should be prepended to it - for part in parts: - if 'series' not in part: - continue - - series = part['series'] - lseries = series.lower() - - if lseries[-4:] == ',the': - part['series'] = 'The ' + series[:-4] - - if lseries[-5:] == ', the': - part['series'] = 'The ' + series[:-5] - - - # 2- try to merge similar information together and give it a higher confidence + # 1- try to merge similar information together and give it a higher + # confidence for int_part in ('year', 'season', 'episodeNumber'): merge_similar_guesses(parts, int_part, choose_int) - for string_part in ('title', 'series', 'container', 'format', 'releaseGroup', 'website', - 'audioCodec', 'videoCodec', 'screenSize', 'episodeFormat'): + for string_part in ('title', 'series', 'container', 'format', + 'releaseGroup', 'website', 'audioCodec', + 'videoCodec', 'screenSize', 'episodeFormat'): merge_similar_guesses(parts, string_part, choose_string) - result = merge_all(parts, append = ['language', 'subtitleLanguage', 'other']) - - # 3- some last minute post-processing - if (result['type'] == 'episode' and - 'season' not in result and - result.get('episodeFormat', '') == 'Minisode'): - result['season'] = 0 + result = merge_all(parts, + append=['language', 'subtitleLanguage', 'other']) log.debug('Final result: ' + result.nice_string()) return result diff --git a/libs/guessit/matchtree.py b/libs/guessit/matchtree.py index 944cab03..634cbf7a 100644 --- a/libs/guessit/matchtree.py +++ b/libs/guessit/matchtree.py @@ -18,136 +18,243 @@ # along with this program. If not, see . # -from guessit.patterns import deleted -from guessit.textutils import clean_string +from guessit import Guess +from guessit.textutils import clean_string, str_fill, to_utf8 +from guessit.patterns import group_delimiters import logging log = logging.getLogger("guessit.matchtree") +class BaseMatchTree(object): + """A MatchTree represents the hierarchical split of a string into its + constituent semantic groups.""" -def tree_to_string(tree): - """Return a string representation for the given tree. + def __init__(self, string='', span=None, parent=None): + self.string = string + self.span = span or (0, len(string)) + self.parent = parent + self.children = [] + self.guess = Guess() - The lines convey the following information: - - line 1: path idx - - line 2: explicit group idx - - line 3: group index - - line 4: remaining info - - line 5: meaning conveyed + @property + def value(self): + return self.string[self.span[0]:self.span[1]] - Meaning is a letter indicating what type of info was matched by this group, - for instance 't' = title, 'f' = format, 'l' = language, etc... + @property + def clean_value(self): + return clean_string(self.value) - An example is the following: + @property + def offset(self): + return self.span[0] - 0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111 - 0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000 - 0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000 - __________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___ - xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc - [XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv + @property + def info(self): + result = dict(self.guess) - (note: the last line representing the filename is not pat of the tree representation) - """ - m_tree = [ '', # path level index - '', # explicit group index - '', # matched regexp and dash-separated - '', # groups leftover that couldn't be matched - '', # meaning conveyed: E = episodenumber, S = season, ... - ] + for c in self.children: + result.update(c.info) + + return result + + @property + def root(self): + if not self.parent: + return self + + return self.parent.root + + @property + def depth(self): + if self.is_leaf(): + return 0 + + return 1 + max(c.depth for c in self.children) + + def is_leaf(self): + return self.children == [] + + def add_child(self, span): + child = MatchTree(self.string, span=span, parent=self) + self.children.append(child) + + def partition(self, indices): + indices = sorted(indices) + if indices[0] != 0: + indices.insert(0, 0) + if indices[-1] != len(self.value): + indices.append(len(self.value)) + + for start, end in zip(indices[:-1], indices[1:]): + self.add_child(span=(self.offset + start, + self.offset + end)) + + def split_on_components(self, components): + offset = 0 + for c in components: + start = self.value.find(c, offset) + end = start + len(c) + self.add_child(span=(self.offset + start, + self.offset + end)) + offset = end + + def nodes_at_depth(self, depth): + if depth == 0: + yield self + + for child in self.children: + for node in child.nodes_at_depth(depth - 1): + yield node + + @property + def node_idx(self): + if self.parent is None: + return () + return self.parent.node_idx + (self.parent.children.index(self),) + + def node_at(self, idx): + if not idx: + return self + + try: + return self.children[idx[0]].node_at(idx[1:]) + except: + raise ValueError('Non-existent node index: %s' % (idx,)) + + def nodes(self): + yield self + for child in self.children: + for node in child.nodes(): + yield node + + def _leaves(self): + if self.is_leaf(): + yield self + else: + for child in self.children: + # pylint: disable=W0212 + for leaf in child._leaves(): + yield leaf + + def leaves(self): + return list(self._leaves()) + + def to_string(self): + empty_line = ' ' * len(self.string) - def add_char(pidx, eidx, gidx, remaining, meaning = None): - nr = len(remaining) def to_hex(x): if isinstance(x, int): - return str(x) if x < 10 else chr(55+x) + return str(x) if x < 10 else chr(55 + x) return x - m_tree[0] = m_tree[0] + to_hex(pidx) * nr - m_tree[1] = m_tree[1] + to_hex(eidx) * nr - m_tree[2] = m_tree[2] + to_hex(gidx) * nr - m_tree[3] = m_tree[3] + remaining - m_tree[4] = m_tree[4] + str(meaning or ' ') * nr - def meaning(result): - mmap = { 'episodeNumber': 'E', - 'season': 'S', - 'extension': 'e', - 'format': 'f', - 'language': 'l', - 'videoCodec': 'v', - 'audioCodec': 'a', - 'website': 'w', - 'container': 'c', - 'series': 'T', - 'title': 't', - 'date': 'd', - 'year': 'y', - 'releaseGroup': 'r', - 'screenSize': 's' - } + def meaning(result): + mmap = { 'episodeNumber': 'E', + 'season': 'S', + 'extension': 'e', + 'format': 'f', + 'language': 'l', + 'videoCodec': 'v', + 'audioCodec': 'a', + 'website': 'w', + 'container': 'c', + 'series': 'T', + 'title': 't', + 'date': 'd', + 'year': 'y', + 'releaseGroup': 'r', + 'screenSize': 's' + } - if result is None: - return ' ' + if result is None: + return ' ' - for prop, l in mmap.items(): - if prop in result: - return l + for prop, l in mmap.items(): + if prop in result: + return l - return 'x' + return 'x' - for pidx, pathpart in enumerate(tree): - for eidx, explicit_group in enumerate(pathpart): - for gidx, (group, remaining, result) in enumerate(explicit_group): - add_char(pidx, eidx, gidx, remaining, meaning(result)) + lines = [ empty_line ] * (self.depth + 2) # +2: remaining, meaning + lines[-2] = self.string - # special conditions for the path separator - if pidx < len(tree) - 2: - add_char(' ', ' ', ' ', '/') - elif pidx == len(tree) - 2: - add_char(' ', ' ', ' ', '.') + for node in self.nodes(): + if node == self: + continue - return '\n'.join(m_tree) + idx = node.node_idx + depth = len(idx) - 1 + if idx: + lines[depth] = str_fill(lines[depth], node.span, + to_hex(idx[-1])) + if node.guess: + lines[-2] = str_fill(lines[-2], node.span, '_') + lines[-1] = str_fill(lines[-1], node.span, meaning(node.guess)) + + lines.append(self.string) + + return '\n'.join(lines) + + def __unicode__(self): + return self.to_string() + + def __str__(self): + return to_utf8(unicode(self)) +class MatchTree(BaseMatchTree): + """The MatchTree contains a few "utility" methods which are not necessary + for the BaseMatchTree, but add a lot of convenience for writing + higher-level rules.""" -def iterate_groups(match_tree): - """Iterate over all the groups in a match_tree and return them as pairs - of (group_pos, group) where: - - group_pos = (pidx, eidx, gidx) - - group = (string, remaining, guess) - """ - for pidx, pathpart in enumerate(match_tree): - for eidx, explicit_group in enumerate(pathpart): - for gidx, group in enumerate(explicit_group): - yield (pidx, eidx, gidx), group + def _unidentified_leaves(self, + valid=lambda leaf: len(leaf.clean_value) >= 2): + for leaf in self._leaves(): + if not leaf.guess and valid(leaf): + yield leaf + def unidentified_leaves(self, + valid=lambda leaf: len(leaf.clean_value) >= 2): + return list(self._unidentified_leaves(valid)) -def find_group(match_tree, prop): - """Find the list of groups that resulted in a guess that contains the - asked property.""" - result = [] - for gpos, (string, remaining, guess) in iterate_groups(match_tree): - if guess and prop in guess: - result.append(gpos) - return result + def _leaves_containing(self, property_name): + if isinstance(property_name, basestring): + property_name = [ property_name ] -def get_group(match_tree, gpos): - pidx, eidx, gidx = gpos - return match_tree[pidx][eidx][gidx] + for leaf in self._leaves(): + for prop in property_name: + if prop in leaf.guess: + yield leaf + break + def leaves_containing(self, property_name): + return list(self._leaves_containing(property_name)) -def leftover_valid_groups(match_tree, valid = lambda s: len(s[0]) > 3): - """Return the list of valid string groups (eg: len(s) > 3) that could not be - matched to anything as a list of pairs (cleaned_str, group_pos).""" - leftover = [] - for gpos, (group, remaining, guess) in iterate_groups(match_tree): - if not guess: - clean_str = clean_string(remaining) - if valid((clean_str, gpos)): - leftover.append((clean_str, gpos)) + def first_leaf_containing(self, property_name): + try: + return next(self._leaves_containing(property_name)) + except StopIteration: + return None - return leftover + def _previous_unidentified_leaves(self, node): + node_idx = node.node_idx + for leaf in self._unidentified_leaves(): + if leaf.node_idx < node_idx: + yield leaf + def previous_unidentified_leaves(self, node): + return list(self._previous_unidentified_leaves(node)) + def _previous_leaves_containing(self, node, property_name): + node_idx = node.node_idx + for leaf in self._leaves_containing(property_name): + if leaf.node_idx < node_idx: + yield leaf + def previous_leaves_containing(self, node, property_name): + return list(self._previous_leaves_containing(node, property_name)) + + def is_explicit(self): + """Return whether the group was explicitly enclosed by + parentheses/square brackets/etc.""" + return (self.value[0] + self.value[-1]) in group_delimiters diff --git a/libs/guessit/patterns.py b/libs/guessit/patterns.py old mode 100644 new mode 100755 index 8331723e..4125fb7b --- a/libs/guessit/patterns.py +++ b/libs/guessit/patterns.py @@ -22,10 +22,13 @@ subtitle_exts = [ 'srt', 'idx', 'sub', 'ssa', 'txt' ] -video_exts = [ 'avi', 'mkv', 'mpg', 'mp4', 'm4v', 'mov', 'ogg', 'ogm', 'ogv', 'wmv', 'divx' ] +video_exts = [ 'avi', 'mkv', 'mpg', 'mp4', 'm4v', 'mov', 'ogg', 'ogm', 'ogv', + 'wmv', 'divx' ] + +group_delimiters = [ '()', '[]', '{}' ] # separator character regexp -sep = r'[][)(}{+ \._-]' # regexp art, hehe :D +sep = r'[][)(}{+ /\._-]' # regexp art, hehe :D # character used to represent a deleted char (when matching groups) deleted = '_' @@ -35,28 +38,33 @@ episode_rexps = [ # ... Season 2 ... (r'season (?P[0-9]+)', 1.0, (0, 0)), (r'saison (?P[0-9]+)', 1.0, (0, 0)), - # ... s02-x01 ... - (r's(?P[0-9]{1,2})-x(?P[0-9]{1,2})[^0-9]', 1.0, (0, -1)), - # ... s02e13 ... (r'[Ss](?P[0-9]{1,2}).{,3}[EeXx](?P[0-9]{1,2})[^0-9]', 1.0, (0, -1)), # ... 2x13 ... - (r'[^0-9](?P[0-9]{1,2})[x\.](?P[0-9]{2})[^0-9]', 0.8, (1, -1)), + (r'[^0-9](?P[0-9]{1,2})x(?P[0-9]{2})[^0-9]', 0.8, (1, -1)), # ... s02 ... - (sep + r's(?P[0-9]{1,2})' + sep + '?', 0.6, (1, -1)), + #(sep + r's(?P[0-9]{1,2})' + sep, 0.6, (1, -1)), + (r's(?P[0-9]{1,2})[^0-9]', 0.6, (0, -1)), # v2 or v3 for some mangas which have multiples rips - (sep + r'(?P[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)), + (r'(?P[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)), + + # ... ep 23 ... + ('ep' + sep + r'(?P[0-9]{1,2})[^0-9]', 0.7, (0, -1)) ] weak_episode_rexps = [ # ... 213 or 0106 ... - (sep + r'(?P[0-9]{1,4})' + sep, 0.3, (1, -1)), + (sep + r'(?P[0-9]{1,4})' + sep, (1, -1)), + + # ... 2x13 ... + (sep + r'[^0-9](?P[0-9]{1,2})\.(?P[0-9]{2})[^0-9]' + sep, (1, -1)), + ] -non_episode_title = [ 'extras' ] +non_episode_title = [ 'extras', 'rip' ] video_rexps = [ # cd number @@ -76,7 +84,13 @@ video_rexps = [ # cd number (r'(?P[0-9]{3,4})x(?P[0-9]{3,4})', 0.9, (0, 0)), # website - (r'(?Pwww(\.[a-zA-Z0-9]+){2,3})', 0.8, (0, 0)) + (r'(?Pwww(\.[a-zA-Z0-9]+){2,3})', 0.8, (0, 0)), + + # bonusNumber: ... x01 ... + (r'x(?P[0-9]{1,2})', 1.0, (0, 0)), + + # filmNumber: ... f01 ... + (r'f(?P[0-9]{1,2})', 1.0, (0, 0)) ] websites = [ 'tvu.org.ru', 'emule-island.com', 'UsaBit.com', 'www.divx-overnet.com', 'sharethefiles.com' ] @@ -87,7 +101,7 @@ properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'B 'HDRip', 'DVD', 'DVDivX', 'HDTV', 'DVB', 'DVBRip', 'PDTV', 'WEBRip', 'DVDSCR', 'Screener', 'VHS', 'VIDEO_TS' ], - 'screenSize': [ '720p', '720' ], + 'screenSize': [ '720p', '720', '1080p', '1080' ], 'videoCodec': [ 'XviD', 'DivX', 'x264', 'h264', 'Rv10' ], @@ -98,18 +112,18 @@ properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'B 'releaseGroup': [ 'ESiR', 'WAF', 'SEPTiC', '[XCT]', 'iNT', 'PUKKA', 'CHD', 'ViTE', 'TLF', 'DEiTY', 'FLAiTE', 'MDX', 'GM4F', 'DVL', 'SVD', 'iLUMiNADOS', ' FiNaLe', - 'UnSeeN', 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL' ], + 'UnSeeN', 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL', + 'HDBRiSe' ], 'episodeFormat': [ 'Minisode', 'Minisodes' ], 'other': [ '5ch', 'PROPER', 'REPACK', 'LIMITED', 'DualAudio', 'iNTERNAL', 'Audiofixed', 'R5', 'complete', 'classic', # not so sure about these ones, could appear in a title 'ws', # widescreen - #'SE', # special edition - # TODO: director's cut ], } + def find_properties(filename): result = [] clow = filename.lower() @@ -119,7 +133,7 @@ def find_properties(filename): if pos != -1: end = pos + len(value) # make sure our word is always surrounded by separators - if ((pos > 0 and clow[pos-1] not in sep) or + if ((pos > 0 and clow[pos - 1] not in sep) or (end < len(clow) and clow[end] not in sep)): # note: sep is a regexp, but in this case using it as # a sequence achieves the same goal @@ -137,6 +151,7 @@ property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ], 'DivX': [ 'DVDivX' ], 'h264': [ 'x264' ], '720p': [ '720' ], + '1080p': [ '1080' ], 'AAC': [ 'He-AAC', 'AAC-He' ], 'Special Edition': [ 'Special' ], 'Collector Edition': [ 'Collector' ], @@ -145,14 +160,21 @@ property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ], } -reverse_synonyms = {} -for prop, values in properties.items(): - for value in values: - reverse_synonyms[value.lower()] = value +def revert_synonyms(): + reverse = {} + + for _, values in properties.items(): + for value in values: + reverse[value.lower()] = value + + for canonical, synonyms in property_synonyms.items(): + for synonym in synonyms: + reverse[synonym.lower()] = canonical + + return reverse + +reverse_synonyms = revert_synonyms() -for canonical, synonyms in property_synonyms.items(): - for synonym in synonyms: - reverse_synonyms[synonym.lower()] = canonical def canonical_form(string): return reverse_synonyms.get(string.lower(), string) diff --git a/libs/guessit/slogging.py b/libs/guessit/slogging.py index 308ef887..34d9af79 100644 --- a/libs/guessit/slogging.py +++ b/libs/guessit/slogging.py @@ -28,8 +28,8 @@ RED_FONT = "\x1B[0;31m" RESET_FONT = "\x1B[0m" -def setupLogging(colored = True): - """Sets up a nice colored logger as the main application logger (not only smewt itself).""" +def setupLogging(colored=True): + """Set up a nice colored logger as the main application logger.""" class SimpleFormatter(logging.Formatter): def __init__(self): @@ -38,7 +38,9 @@ def setupLogging(colored = True): class ColoredFormatter(logging.Formatter): def __init__(self): - self.fmt = '%(levelname)-8s ' + BLUE_FONT + '%(module)s:%(funcName)s' + RESET_FONT + ' -- %(message)s' + self.fmt = ('%(levelname)-8s ' + + BLUE_FONT + '%(name)s:%(funcName)s' + + RESET_FONT + ' -- %(message)s') logging.Formatter.__init__(self, self.fmt) def format(self, record): @@ -50,11 +52,9 @@ def setupLogging(colored = True): else: return RED_FONT + result - ch = logging.StreamHandler() if colored and sys.platform != 'win32': ch.setFormatter(ColoredFormatter()) else: ch.setFormatter(SimpleFormatter()) logging.getLogger().addHandler(ch) - diff --git a/libs/guessit/textutils.py b/libs/guessit/textutils.py index 2ab88815..cfe31c4d 100644 --- a/libs/guessit/textutils.py +++ b/libs/guessit/textutils.py @@ -18,47 +18,59 @@ # along with this program. If not, see . # -from guessit.patterns import sep, deleted +from guessit.patterns import sep import copy # string-related functions + def strip_brackets(s): if not s: return s - if s[0] == '[' and s[-1] == ']': return s[1:-1] - if s[0] == '(' and s[-1] == ')': return s[1:-1] - if s[0] == '{' and s[-1] == '}': return s[1:-1] + + if ((s[0] == '[' and s[-1] == ']') or + (s[0] == '(' and s[-1] == ')') or + (s[0] == '{' and s[-1] == '}')): + return s[1:-1] + return s def clean_string(s): - for c in sep: + for c in sep[:-2]: # do not remove dashes ('-') s = s.replace(c, ' ') parts = s.split() - return ' '.join(p for p in parts if p != '') + result = ' '.join(p for p in parts if p != '') + + # now also remove dashes on the outer part of the string + while result and result[0] in sep: + result = result[1:] + while result and result[-1] in sep: + result = result[:-1] + + return result def str_replace(string, pos, c): return string[:pos] + c + string[pos+1:] -def blank_region(string, region, blank_sep = deleted): + +def str_fill(string, region, c): start, end = region - return string[:start] + blank_sep * (end - start) + string[end:] + return string[:start] + c * (end - start) + string[end:] -def between(s, left, right): - return s.split(left)[1].split(right)[0] - def to_utf8(o): - '''converts all unicode strings found in the given object to utf-8 strings''' + """Convert all unicode strings found in the given object to utf-8 + strings.""" if isinstance(o, unicode): return o.encode('utf-8') elif isinstance(o, list): return [ to_utf8(i) for i in o ] elif isinstance(o, dict): - result = copy.deepcopy(o) # need to do it like that to handle Guess instances correctly + # need to do it like that to handle Guess instances correctly + result = copy.deepcopy(o) for key, value in o.items(): result[to_utf8(key)] = to_utf8(value) return result @@ -68,8 +80,10 @@ def to_utf8(o): def levenshtein(a, b): - if not a: return len(b) - if not b: return len(a) + if not a: + return len(b) + if not b: + return len(a) m = len(a) n = len(b) @@ -160,14 +174,13 @@ def split_on_groups(string, groups): if boundaries[-1] != len(string): boundaries.append(len(string)) - groups = [ string[start:end] for start, end in zip(boundaries[:-1], boundaries[1:]) ] + groups = [ string[start:end] for start, end in zip(boundaries[:-1], + boundaries[1:]) ] - return filter(bool, groups) # return only non-empty groups + return [ g for g in groups if g ] # return only non-empty groups - - -def find_first_level_groups(string, enclosing, blank_sep = None): +def find_first_level_groups(string, enclosing, blank_sep=None): """Return a list of groups that could be split because of explicit grouping. The groups are delimited by the given enclosing characters. @@ -203,8 +216,3 @@ def find_first_level_groups(string, enclosing, blank_sep = None): string = str_replace(string, end-1, blank_sep) return split_on_groups(string, groups) - - - - - diff --git a/libs/guessit/transfo/__init__.py b/libs/guessit/transfo/__init__.py new file mode 100644 index 00000000..eb72bebd --- /dev/null +++ b/libs/guessit/transfo/__init__.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import Guess +from guessit.patterns import canonical_form +from guessit.textutils import clean_string +import logging + +log = logging.getLogger('guessit.transfo') + + +def found_property(node, name, confidence): + node.guess = Guess({name: node.clean_value}, confidence=confidence) + log.debug('Found with confidence %.2f: %s' % (confidence, node.guess)) + + +def format_guess(guess): + """Format all the found values to their natural type. + For instance, a year would be stored as an int value, etc... + + Note that this modifies the dictionary given as input. + """ + for prop, value in guess.items(): + if prop in ('season', 'episodeNumber', 'year', 'cdNumber', + 'cdNumberTotal', 'bonusNumber', 'filmNumber'): + guess[prop] = int(guess[prop]) + elif isinstance(value, basestring): + if prop in ('edition',): + value = clean_string(value) + guess[prop] = canonical_form(value) + + return guess + + +def find_and_split_node(node, strategy, logger): + string = ' %s ' % node.value # add sentinels + for matcher, confidence in strategy: + if getattr(matcher, 'use_node', False): + result, span = matcher(string, node) + else: + result, span = matcher(string) + + if result: + # readjust span to compensate for sentinels + span = (span[0] - 1, span[1] - 1) + + if isinstance(result, Guess): + if confidence is None: + confidence = result.confidence(result.keys()[0]) + else: + if confidence is None: + confidence = 1.0 + + guess = format_guess(Guess(result, confidence=confidence)) + msg = 'Found with confidence %.2f: %s' % (confidence, guess) + (logger or log).debug(msg) + + node.partition(span) + absolute_span = (span[0] + node.offset, span[1] + node.offset) + for child in node.children: + if child.span == absolute_span: + child.guess = guess + else: + find_and_split_node(child, strategy, logger) + return + + +class SingleNodeGuesser(object): + def __init__(self, guess_func, confidence, logger=None): + self.guess_func = guess_func + self.confidence = confidence + self.logger = logger + + def process(self, mtree): + # strategy is a list of pairs (guesser, confidence) + # - if the guesser returns a guessit.Guess and confidence is specified, + # it will override it, otherwise it will leave the guess confidence + # - if the guesser returns a simple dict as a guess and confidence is + # specified, it will use it, or 1.0 otherwise + strategy = [ (self.guess_func, self.confidence) ] + + for node in mtree.unidentified_leaves(): + find_and_split_node(node, strategy, self.logger) diff --git a/libs/guessit/transfo/guess_bonus_features.py b/libs/guessit/transfo/guess_bonus_features.py new file mode 100644 index 00000000..dcb90b3f --- /dev/null +++ b/libs/guessit/transfo/guess_bonus_features.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.transfo import found_property +import logging + +log = logging.getLogger("guessit.transfo.guess_bonus_features") + + +def process(mtree): + def previous_group(g): + for leaf in mtree.unidentified_leaves()[::-1]: + if leaf.node_idx < g.node_idx: + return leaf + + def next_group(g): + for leaf in mtree.unidentified_leaves(): + if leaf.node_idx > g.node_idx: + return leaf + + def same_group(g1, g2): + return g1.node_idx[:2] == g2.node_idx[:2] + + bonus = [ node for node in mtree.leaves() if 'bonusNumber' in node.guess ] + if bonus: + bonusTitle = next_group(bonus[0]) + if same_group(bonusTitle, bonus[0]): + found_property(bonusTitle, 'bonusTitle', 0.8) + + filmNumber = [ node for node in mtree.leaves() + if 'filmNumber' in node.guess ] + if filmNumber: + filmSeries = previous_group(filmNumber[0]) + found_property(filmSeries, 'filmSeries', 0.9) + + title = next_group(filmNumber[0]) + found_property(title, 'title', 0.9) + + season = [ node for node in mtree.leaves() if 'season' in node.guess ] + if season and 'bonusNumber' in mtree.info: + series = previous_group(season[0]) + if same_group(series, season[0]): + found_property(series, 'series', 0.9) diff --git a/libs/guessit/transfo/guess_date.py b/libs/guessit/transfo/guess_date.py new file mode 100644 index 00000000..c72d66ad --- /dev/null +++ b/libs/guessit/transfo/guess_date.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.transfo import SingleNodeGuesser +from guessit.date import search_date +import logging + +log = logging.getLogger("guessit.transfo.guess_date") + + +def guess_date(string): + date, span = search_date(string) + if date: + return { 'date': date }, span + else: + return None, None + + +def process(mtree): + SingleNodeGuesser(guess_date, 1.0, log).process(mtree) diff --git a/libs/guessit/transfo/guess_episode_info_from_position.py b/libs/guessit/transfo/guess_episode_info_from_position.py new file mode 100644 index 00000000..fe1a7525 --- /dev/null +++ b/libs/guessit/transfo/guess_episode_info_from_position.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.transfo import found_property +from guessit.patterns import non_episode_title, unlikely_series +import logging + +log = logging.getLogger("guessit.transfo.guess_episode_info_from_position") + + +def match_from_epnum_position(mtree, node): + epnum_idx = node.node_idx + + # a few helper functions to be able to filter using high-level semantics + def before_epnum_in_same_pathgroup(): + return [ leaf for leaf in mtree.unidentified_leaves() + if (leaf.node_idx[0] == epnum_idx[0] and + leaf.node_idx[1:] < epnum_idx[1:]) ] + + def after_epnum_in_same_pathgroup(): + return [ leaf for leaf in mtree.unidentified_leaves() + if (leaf.node_idx[0] == epnum_idx[0] and + leaf.node_idx[1:] > epnum_idx[1:]) ] + + def after_epnum_in_same_explicitgroup(): + return [ leaf for leaf in mtree.unidentified_leaves() + if (leaf.node_idx[:2] == epnum_idx[:2] and + leaf.node_idx[2:] > epnum_idx[2:]) ] + + # epnumber is the first group and there are only 2 after it in same + # path group + # -> series title - episode title + title_candidates = [ n for n in after_epnum_in_same_pathgroup() + if n.clean_value.lower() not in non_episode_title ] + if ('title' not in mtree.info and # no title + before_epnum_in_same_pathgroup() == [] and # no groups before + len(title_candidates) == 2): # only 2 groups after + + found_property(title_candidates[0], 'series', confidence=0.4) + found_property(title_candidates[1], 'title', confidence=0.4) + return + + # if we have at least 1 valid group before the episodeNumber, then it's + # probably the series name + series_candidates = before_epnum_in_same_pathgroup() + if len(series_candidates) >= 1: + found_property(series_candidates[0], 'series', confidence=0.7) + + # only 1 group after (in the same path group) and it's probably the + # episode title + title_candidates = [ n for n in after_epnum_in_same_pathgroup() + if n.clean_value.lower() not in non_episode_title ] + + if len(title_candidates) == 1: + found_property(title_candidates[0], 'title', confidence=0.5) + return + else: + # try in the same explicit group, with lower confidence + title_candidates = [ n for n in after_epnum_in_same_explicitgroup() + if n.clean_value.lower() not in non_episode_title + ] + if len(title_candidates) == 1: + found_property(title_candidates[0], 'title', confidence=0.4) + return + elif len(title_candidates) > 1: + found_property(title_candidates[0], 'title', confidence=0.3) + return + + # get the one with the longest value + title_candidates = [ n for n in after_epnum_in_same_pathgroup() + if n.clean_value.lower() not in non_episode_title ] + if title_candidates: + maxidx = -1 + maxv = -1 + for i, c in enumerate(title_candidates): + if len(c.clean_value) > maxv: + maxidx = i + maxv = len(c.clean_value) + found_property(title_candidates[maxidx], 'title', confidence=0.3) + + +def process(mtree): + eps = [node for node in mtree.leaves() if 'episodeNumber' in node.guess] + if eps: + match_from_epnum_position(mtree, eps[0]) + + else: + # if we don't have the episode number, but at least 2 groups in the + # basename, then it's probably series - eptitle + basename = mtree.node_at((-2,)) + title_candidates = [ n for n in basename.unidentified_leaves() + if n.clean_value.lower() not in non_episode_title + ] + + if len(title_candidates) >= 2: + found_property(title_candidates[0], 'series', 0.4) + found_property(title_candidates[1], 'title', 0.4) + + # if we only have 1 remaining valid group in the folder containing the + # file, then it's likely that it is the series name + try: + series_candidates = mtree.node_at((-3,)).unidentified_leaves() + except ValueError: + series_candidates = [] + + if len(series_candidates) == 1: + found_property(series_candidates[0], 'series', 0.3) + + # if there's a path group that only contains the season info, then the + # previous one is most likely the series title (ie: ../series/season X/..) + eps = [ node for node in mtree.nodes() + if 'season' in node.guess and 'episodeNumber' not in node.guess ] + + if eps: + previous = [ node for node in mtree.unidentified_leaves() + if node.node_idx[0] == eps[0].node_idx[0] - 1 ] + if len(previous) == 1: + found_property(previous[0], 'series', 0.5) + + # reduce the confidence of unlikely series + for node in mtree.nodes(): + if 'series' in node.guess: + if node.guess['series'].lower() in unlikely_series: + new_confidence = node.guess.confidence('series') * 0.5 + node.guess.set_confidence('series', new_confidence) diff --git a/libs/guessit/transfo/guess_episodes_rexps.py b/libs/guessit/transfo/guess_episodes_rexps.py new file mode 100644 index 00000000..46dbc590 --- /dev/null +++ b/libs/guessit/transfo/guess_episodes_rexps.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import Guess +from guessit.transfo import SingleNodeGuesser +from guessit.patterns import episode_rexps +import re +import logging + +log = logging.getLogger("guessit.transfo.guess_episodes_rexps") + + +def guess_episodes_rexps(string): + for rexp, confidence, span_adjust in episode_rexps: + match = re.search(rexp, string, re.IGNORECASE) + if match: + return (Guess(match.groupdict(), confidence=confidence), + (match.start() + span_adjust[0], + match.end() + span_adjust[1])) + + return None, None + + +def process(mtree): + SingleNodeGuesser(guess_episodes_rexps, None, log).process(mtree) diff --git a/libs/guessit/transfo/guess_filetype.py b/libs/guessit/transfo/guess_filetype.py new file mode 100644 index 00000000..32bdc136 --- /dev/null +++ b/libs/guessit/transfo/guess_filetype.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import Guess +from guessit.patterns import (subtitle_exts, video_exts, episode_rexps, + find_properties, canonical_form) +import os.path +import re +import mimetypes +import logging + +log = logging.getLogger("guessit.transfo.guess_filetype") + + +def guess_filetype(filename, filetype): + other = {} + + # look at the extension first + fileext = os.path.splitext(filename)[1][1:].lower() + if fileext in subtitle_exts: + if 'movie' in filetype: + filetype = 'moviesubtitle' + elif 'episode' in filetype: + filetype = 'episodesubtitle' + else: + filetype = 'subtitle' + other = { 'container': fileext } + elif fileext in video_exts: + if filetype == 'autodetect': + filetype = 'video' + other = { 'container': fileext } + else: + if filetype == 'autodetect': + filetype = 'unknown' + other = { 'extension': fileext } + + # put the filetype inside a dummy container to be able to have the + # following functions work correctly as closures + # this is a workaround for python 2 which doesn't have the + # 'nonlocal' keyword (python 3 does have it) + filetype_container = [filetype] + + def upgrade_episode(): + if filetype_container[0] == 'video': + filetype_container[0] = 'episode' + elif filetype_container[0] == 'subtitle': + filetype_container[0] = 'episodesubtitle' + + def upgrade_movie(): + if filetype_container[0] == 'video': + filetype_container[0] = 'movie' + elif filetype_container[0] == 'subtitle': + filetype_container[0] = 'moviesubtitle' + + # now look whether there are some specific hints for episode vs movie + if filetype in ('video', 'subtitle'): + for rexp, _, _ in episode_rexps: + match = re.search(rexp, filename, re.IGNORECASE) + if match: + upgrade_episode() + break + + for prop, value, _, _ in find_properties(filename): + log.debug('prop: %s = %s' % (prop, value)) + if prop == 'episodeFormat': + upgrade_episode() + break + + elif canonical_form(value) == 'DVB': + upgrade_episode() + break + + # if no episode info found, assume it's a movie + upgrade_movie() + + filetype = filetype_container[0] + return filetype, other + + +def process(mtree, filetype='autodetect'): + filetype, other = guess_filetype(mtree.string, filetype) + + mtree.guess.set('type', filetype, confidence=1.0) + log.debug('Found with confidence %.2f: %s' % (1.0, mtree.guess)) + + filetype_info = Guess(other, confidence=1.0) + # guess the mimetype of the filename + # TODO: handle other mimetypes not found on the default type_maps + # mimetypes.types_map['.srt']='text/subtitle' + mime, _ = mimetypes.guess_type(mtree.string, strict=False) + if mime is not None: + filetype_info.update({'mimetype': mime}, confidence=1.0) + + node_ext = mtree.node_at((-1,)) + node_ext.guess = filetype_info + log.debug('Found with confidence %.2f: %s' % (1.0, node_ext.guess)) diff --git a/libs/guessit/transfo/guess_language.py b/libs/guessit/transfo/guess_language.py new file mode 100644 index 00000000..62f47d84 --- /dev/null +++ b/libs/guessit/transfo/guess_language.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import Guess +from guessit.transfo import SingleNodeGuesser +from guessit.language import search_language +from guessit.textutils import clean_string +import logging + +log = logging.getLogger("guessit.transfo.guess_language") + + +def guess_language(string): + language, span, confidence = search_language(string) + if language: + # is it a subtitle language? + if 'sub' in clean_string(string[:span[0]]).lower().split(' '): + return (Guess({'subtitleLanguage': language}, + confidence=confidence), + span) + else: + return (Guess({'language': language}, + confidence=confidence), + span) + + return None, None + + +def process(mtree): + SingleNodeGuesser(guess_language, None, log).process(mtree) diff --git a/libs/guessit/transfo/guess_movie_title_from_position.py b/libs/guessit/transfo/guess_movie_title_from_position.py new file mode 100644 index 00000000..dea56d63 --- /dev/null +++ b/libs/guessit/transfo/guess_movie_title_from_position.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import Guess +import logging + +log = logging.getLogger("guessit.transfo.guess_movie_title_from_position") + + +def process(mtree): + def found_property(node, name, value, confidence): + node.guess = Guess({ name: value }, + confidence=confidence) + log.debug('Found with confidence %.2f: %s' % (confidence, node.guess)) + + def found_title(node, confidence): + found_property(node, 'title', node.clean_value, confidence) + + basename = mtree.node_at((-2,)) + all_valid = lambda leaf: len(leaf.clean_value) > 0 + basename_leftover = basename.unidentified_leaves(valid=all_valid) + + try: + folder = mtree.node_at((-3,)) + folder_leftover = folder.unidentified_leaves() + except ValueError: + folder = None + folder_leftover = [] + + log.debug('folder: %s' % folder_leftover) + log.debug('basename: %s' % basename_leftover) + + # specific cases: + # if we find the same group both in the folder name and the filename, + # it's a good candidate for title + if (folder_leftover and basename_leftover and + folder_leftover[0].clean_value == basename_leftover[0].clean_value): + + found_title(folder_leftover[0], confidence=0.8) + return + + # specific cases: + # if the basename contains a number first followed by an unidentified + # group, and the folder only contains 1 unidentified one, then we have + # a series + # ex: Millenium Trilogy (2009)/(1)The Girl With The Dragon Tattoo(2009).mkv + try: + series = folder_leftover[0] + filmNumber = basename_leftover[0] + title = basename_leftover[1] + + basename_leaves = basename.leaves() + + num = int(filmNumber.clean_value) + + log.debug('series: %s' % series.clean_value) + log.debug('title: %s' % title.clean_value) + if (series.clean_value != title.clean_value and + series.clean_value != filmNumber.clean_value and + basename_leaves.index(filmNumber) == 0 and + basename_leaves.index(title) == 1): + + found_title(title, confidence=0.6) + found_property(series, 'filmSeries', + series.clean_value, confidence=0.6) + found_property(filmNumber, 'filmNumber', + num, confidence=0.6) + return + except Exception: + pass + + # specific cases: + # - movies/tttttt (yyyy)/tttttt.ccc + try: + if mtree.node_at((-4, 0)).value.lower() == 'movies': + folder = mtree.node_at((-3,)) + + # Note:too generic, might solve all the unittests as they all + # contain 'movies' in their path + # + #if containing_folder.is_leaf() and not containing_folder.guess: + # containing_folder.guess = + # Guess({ 'title': clean_string(containing_folder.value) }, + # confidence=0.7) + + year_group = folder.first_leaf_containing('year') + groups_before = folder.previous_unidentified_leaves(year_group) + + found_title(groups_before[0], confidence=0.8) + return + + except Exception: + pass + + # if we have either format or videoCodec in the folder containing the file + # or one of its parents, then we should probably look for the title in + # there rather than in the basename + try: + props = mtree.previous_leaves_containing(mtree.children[-2], + [ 'videoCodec', 'format', + 'language' ]) + except IndexError: + props = [] + + if props: + group_idx = props[0].node_idx[0] + if all(g.node_idx[0] == group_idx for g in props): + # if they're all in the same group, take leftover info from there + leftover = mtree.node_at((group_idx,)).unidentified_leaves() + + if leftover: + found_title(leftover[0], confidence=0.7) + return + + # look for title in basename if there are some remaining undidentified + # groups there + if basename_leftover: + title_candidate = basename_leftover[0] + + # if basename is only one word and the containing folder has at least + # 3 words in it, we should take the title from the folder name + # ex: Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi + # ex: Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi <-- TODO: gets caught here? + if (title_candidate.clean_value.count(' ') == 0 and + folder_leftover and + folder_leftover[0].clean_value.count(' ') >= 2): + + found_title(folder_leftover[0], confidence=0.7) + return + + # if there are only 2 unidentified groups, the first of which is inside + # brackets or parentheses, we take the second one for the title: + # ex: Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi + if len(basename_leftover) == 2 and basename_leftover[0].is_explicit(): + found_title(basename_leftover[1], confidence=0.8) + return + + # if all else fails, take the first remaining unidentified group in the + # basename as title + found_title(title_candidate, confidence=0.6) + return + + # if there are no leftover groups in the basename, look in the folder name + if folder_leftover: + found_title(folder_leftover[0], confidence=0.5) + return + + # if nothing worked, look if we have a very small group at the beginning + # of the basename + basename = mtree.node_at((-2,)) + basename_leftover = basename.unidentified_leaves(valid=lambda leaf: True) + if basename_leftover: + found_title(basename_leftover[0], confidence=0.4) + return diff --git a/libs/guessit/transfo/guess_properties.py b/libs/guessit/transfo/guess_properties.py new file mode 100644 index 00000000..3822d22e --- /dev/null +++ b/libs/guessit/transfo/guess_properties.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.transfo import SingleNodeGuesser +from guessit.patterns import find_properties +import logging + +log = logging.getLogger("guessit.transfo.guess_properties") + + +def guess_properties(string): + try: + prop, value, pos, end = find_properties(string)[0] + return { prop: value }, (pos, end) + except IndexError: + return None, None + + +def process(mtree): + SingleNodeGuesser(guess_properties, 1.0, log).process(mtree) diff --git a/libs/guessit/transfo/guess_release_group.py b/libs/guessit/transfo/guess_release_group.py new file mode 100644 index 00000000..9ec609db --- /dev/null +++ b/libs/guessit/transfo/guess_release_group.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.transfo import SingleNodeGuesser +import re +import logging + +log = logging.getLogger("guessit.transfo.guess_release_group") + + +def guess_release_group(string): + group_names = [ r'\.(Xvid)-(?P.*?)[ \.]', + r'\.(DivX)-(?P.*?)[\. ]', + r'\.(DVDivX)-(?P.*?)[\. ]', + ] + for rexp in group_names: + match = re.search(rexp, string, re.IGNORECASE) + if match: + metadata = match.groupdict() + metadata.update({ 'videoCodec': match.group(1) }) + return metadata, (match.start() + 1, match.end() - 1) + + return None, None + + +def process(mtree): + SingleNodeGuesser(guess_release_group, 0.8, log).process(mtree) diff --git a/libs/guessit/transfo/guess_video_rexps.py b/libs/guessit/transfo/guess_video_rexps.py new file mode 100644 index 00000000..36723c82 --- /dev/null +++ b/libs/guessit/transfo/guess_video_rexps.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import Guess +from guessit.transfo import SingleNodeGuesser +from guessit.patterns import video_rexps, sep +import re +import logging + +log = logging.getLogger("guessit.transfo.guess_video_rexps") + + +def guess_video_rexps(string): + string = '-' + string + '-' + for rexp, confidence, span_adjust in video_rexps: + match = re.search(sep + rexp + sep, string, re.IGNORECASE) + if match: + metadata = match.groupdict() + # is this the better place to put it? (maybe, as it is at least + # the soonest that we can catch it) + if metadata.get('cdNumberTotal', -1) is None: + del metadata['cdNumberTotal'] + return (Guess(metadata, confidence=confidence), + (match.start() + span_adjust[0], + match.end() + span_adjust[1] - 2)) + + return None, None + + +def process(mtree): + SingleNodeGuesser(guess_video_rexps, None, log).process(mtree) diff --git a/libs/guessit/transfo/guess_weak_episodes_rexps.py b/libs/guessit/transfo/guess_weak_episodes_rexps.py new file mode 100644 index 00000000..8fffe17f --- /dev/null +++ b/libs/guessit/transfo/guess_weak_episodes_rexps.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import Guess +from guessit.transfo import SingleNodeGuesser +from guessit.patterns import weak_episode_rexps +import re +import logging + +log = logging.getLogger("guessit.transfo.guess_weak_episodes_rexps") + + +def guess_weak_episodes_rexps(string, node): + if 'episodeNumber' in node.root.info: + return None, None + + for rexp, span_adjust in weak_episode_rexps: + match = re.search(rexp, string, re.IGNORECASE) + if match: + metadata = match.groupdict() + span = (match.start() + span_adjust[0], + match.end() + span_adjust[1]) + + epnum = int(metadata['episodeNumber']) + if epnum > 100: + return Guess({ 'season': epnum // 100, + 'episodeNumber': epnum % 100 }, + confidence=0.6), span + else: + return Guess(metadata, confidence=0.3), span + + return None, None + + +guess_weak_episodes_rexps.use_node = True + + +def process(mtree): + SingleNodeGuesser(guess_weak_episodes_rexps, 0.6, log).process(mtree) diff --git a/libs/guessit/transfo/guess_website.py b/libs/guessit/transfo/guess_website.py new file mode 100644 index 00000000..a169f97b --- /dev/null +++ b/libs/guessit/transfo/guess_website.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.transfo import SingleNodeGuesser +from guessit.patterns import websites +import logging + +log = logging.getLogger("guessit.transfo.guess_website") + + +def guess_website(string): + low = string.lower() + for site in websites: + pos = low.find(site.lower()) + if pos != -1: + return {'website': site}, (pos, pos + len(site)) + return None, None + + +def process(mtree): + SingleNodeGuesser(guess_website, 1.0, log).process(mtree) diff --git a/libs/guessit/transfo/guess_year.py b/libs/guessit/transfo/guess_year.py new file mode 100644 index 00000000..7a47ecf7 --- /dev/null +++ b/libs/guessit/transfo/guess_year.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.transfo import SingleNodeGuesser +from guessit.date import search_year +import logging + +log = logging.getLogger("guessit.transfo.guess_year") + + +def guess_year(string): + year, span = search_year(string) + if year: + return { 'year': year }, span + else: + return None, None + + +def process(mtree): + SingleNodeGuesser(guess_year, 1.0, log).process(mtree) diff --git a/libs/guessit/transfo/post_process.py b/libs/guessit/transfo/post_process.py new file mode 100644 index 00000000..0b5a4dfb --- /dev/null +++ b/libs/guessit/transfo/post_process.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.patterns import subtitle_exts +import logging + +log = logging.getLogger("guessit.transfo.post_process") + + +def process(mtree): + # 1- try to promote language to subtitle language where it makes sense + for node in mtree.nodes(): + if 'language' not in node.guess: + continue + + def promote_subtitle(): + # pylint: disable=W0631 + node.guess.set('subtitleLanguage', node.guess['language'], + confidence=node.guess.confidence('language')) + del node.guess['language'] + + # - if we matched a language in a file with a sub extension and that + # the group is the last group of the filename, it is probably the + # language of the subtitle + # (eg: 'xxx.english.srt') + if (mtree.node_at((-1,)).value.lower() in subtitle_exts and + node == mtree.leaves()[-2]): + promote_subtitle() + + # - if a language is in an explicit group just preceded by "st", + # it is a subtitle language (eg: '...st[fr-eng]...') + try: + idx = node.node_idx + previous = mtree.node_at((idx[0], idx[1] - 1)).leaves()[-1] + if previous.value.lower()[-2:] == 'st': + promote_subtitle() + except IndexError: + pass + + # 2- ", the" at the end of a series title should be prepended to it + for node in mtree.nodes(): + if 'series' not in node.guess: + continue + + series = node.guess['series'] + lseries = series.lower() + + if lseries[-4:] == ',the': + node.guess['series'] = 'The ' + series[:-4] + + if lseries[-5:] == ', the': + node.guess['series'] = 'The ' + series[:-5] diff --git a/libs/guessit/transfo/split_explicit_groups.py b/libs/guessit/transfo/split_explicit_groups.py new file mode 100644 index 00000000..797a886c --- /dev/null +++ b/libs/guessit/transfo/split_explicit_groups.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.textutils import find_first_level_groups +from guessit.patterns import group_delimiters +import logging + +log = logging.getLogger("guessit.transfo.split_explicit_groups") + + +def process(mtree): + """return the string split into explicit groups, that is, those either + between parenthese, square brackets or curly braces, and those separated + by a dash.""" + for c in mtree.children: + groups = find_first_level_groups(c.value, group_delimiters[0]) + for delimiters in group_delimiters: + flatten = lambda l, x: l + find_first_level_groups(x, delimiters) + groups = reduce(flatten, groups, []) + + # do not do this at this moment, it is not strong enough and can break other + # patterns, such as dates, etc... + #groups = reduce(lambda l, x: l + x.split('-'), groups, []) + + c.split_on_components(groups) diff --git a/libs/guessit/transfo/split_on_dash.py b/libs/guessit/transfo/split_on_dash.py new file mode 100644 index 00000000..fc10c49d --- /dev/null +++ b/libs/guessit/transfo/split_on_dash.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit.patterns import sep +import re +import logging + +log = logging.getLogger("guessit.transfo.split_on_dash") + + +def process(mtree): + for node in mtree.unidentified_leaves(): + indices = [] + + didx = 0 + pattern = re.compile(sep + '-' + sep) + match = pattern.search(node.value) + while match: + span = match.span() + indices.extend([ span[0], span[1] ]) + match = pattern.search(node.value, span[1]) + + didx = node.value.find('-') + while didx > 0: + if (didx > 10 and + (didx - 1 not in indices and + didx + 2 not in indices)): + + indices.extend([ didx, didx + 1 ]) + + didx = node.value.find('-', didx + 1) + + if indices: + node.partition(indices) diff --git a/libs/guessit/transfo/split_path_components.py b/libs/guessit/transfo/split_path_components.py new file mode 100644 index 00000000..0f8d1a5a --- /dev/null +++ b/libs/guessit/transfo/split_path_components.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# GuessIt - A library for guessing information from filenames +# Copyright (c) 2012 Nicolas Wack +# +# GuessIt is free software; you can redistribute it and/or modify it under +# the terms of the Lesser GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# GuessIt is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Lesser GNU General Public License for more details. +# +# You should have received a copy of the Lesser GNU General Public License +# along with this program. If not, see . +# + +from guessit import fileutils +import os.path +import logging + +log = logging.getLogger("guessit.transfo.split_path_components") + + +def process(mtree): + """Returns the filename split into [ dir*, basename, ext ].""" + components = fileutils.split_path(mtree.value) + basename = components.pop(-1) + components += list(os.path.splitext(basename)) + components[-1] = components[-1][1:] # remove the '.' from the extension + + mtree.split_on_components(components) diff --git a/libs/requests/__init__.py b/libs/requests/__init__.py index 48fb3899..0ace12b6 100644 --- a/libs/requests/__init__.py +++ b/libs/requests/__init__.py @@ -15,14 +15,13 @@ requests """ __title__ = 'requests' -__version__ = '0.10.1' -__build__ = 0x001001 +__version__ = '0.11.1' +__build__ = 0x001101 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2012 Kenneth Reitz' - from . import utils from .models import Request, Response from .api import request, get, head, post, patch, put, delete, options diff --git a/libs/requests/api.py b/libs/requests/api.py index b7d4158d..e0bf346c 100644 --- a/libs/requests/api.py +++ b/libs/requests/api.py @@ -33,6 +33,7 @@ def request(method, url, **kwargs): :param config: (optional) A configuration dictionary. :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. :param prefetch: (optional) if ``True``, the response content will be immediately downloaded. + :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. """ s = kwargs.pop('session') if 'session' in kwargs else sessions.session() @@ -44,7 +45,7 @@ def get(url, **kwargs): """Sends a GET request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ kwargs.setdefault('allow_redirects', True) @@ -55,7 +56,7 @@ def options(url, **kwargs): """Sends a OPTIONS request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ kwargs.setdefault('allow_redirects', True) @@ -66,10 +67,10 @@ def head(url, **kwargs): """Sends a HEAD request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ - kwargs.setdefault('allow_redirects', True) + kwargs.setdefault('allow_redirects', False) return request('head', url, **kwargs) @@ -78,7 +79,7 @@ def post(url, data=None, **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return request('post', url, data=data, **kwargs) @@ -89,7 +90,7 @@ def put(url, data=None, **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return request('put', url, data=data, **kwargs) @@ -100,7 +101,7 @@ def patch(url, data=None, **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return request('patch', url, data=data, **kwargs) @@ -110,7 +111,7 @@ def delete(url, **kwargs): """Sends a DELETE request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return request('delete', url, **kwargs) diff --git a/libs/requests/async.py b/libs/requests/async.py index 94884478..f12cf268 100644 --- a/libs/requests/async.py +++ b/libs/requests/async.py @@ -17,13 +17,13 @@ except ImportError: raise RuntimeError('Gevent is required for requests.async.') # Monkey-patch. -curious_george.patch_all(thread=False) +curious_george.patch_all(thread=False, select=False) from . import api __all__ = ( - 'map', + 'map', 'imap', 'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request' ) @@ -46,15 +46,15 @@ def patched(f): return wrapped -def send(r, pool=None): - """Sends the request object using the specified pool. If a pool isn't +def send(r, pool=None, prefetch=False): + """Sends the request object using the specified pool. If a pool isn't specified this method blocks. Pools are useful because you can specify size and can hence limit concurrency.""" if pool != None: - return pool.spawn(r.send) + return pool.spawn(r.send, prefetch=prefetch) - return gevent.spawn(r.send) + return gevent.spawn(r.send, prefetch=prefetch) # Patched requests.api functions. @@ -79,10 +79,28 @@ def map(requests, prefetch=True, size=None): requests = list(requests) pool = Pool(size) if size else None - jobs = [send(r, pool) for r in requests] + jobs = [send(r, pool, prefetch=prefetch) for r in requests] gevent.joinall(jobs) - if prefetch: - [r.response.content for r in requests] + return [r.response for r in requests] - return [r.response for r in requests] \ No newline at end of file + +def imap(requests, prefetch=True, size=2): + """Concurrently converts a generator object of Requests to + a generator of Responses. + + :param requests: a generator of Request objects. + :param prefetch: If False, the content will not be downloaded immediately. + :param size: Specifies the number of requests to make at a time. default is 2 + """ + + pool = Pool(size) + + def send(r): + r.send(prefetch) + return r.response + + for r in pool.imap_unordered(send, requests): + yield r + + pool.join() \ No newline at end of file diff --git a/libs/requests/auth.py b/libs/requests/auth.py index 183731bd..385dd27d 100644 --- a/libs/requests/auth.py +++ b/libs/requests/auth.py @@ -7,13 +7,11 @@ requests.auth This module contains the authentication handlers for Requests. """ -from __future__ import unicode_literals - import time import hashlib from base64 import b64encode -from .compat import urlparse, str, bytes +from .compat import urlparse, str from .utils import randombytes, parse_dict_header @@ -21,7 +19,7 @@ from .utils import randombytes, parse_dict_header def _basic_auth_str(username, password): """Returns a Basic Auth string.""" - return 'Basic ' + b64encode(("%s:%s" % (username, password)).encode('utf-8')).strip().decode('utf-8') + return 'Basic ' + b64encode(('%s:%s' % (username, password)).encode('latin1')).strip().decode('latin1') class AuthBase(object): diff --git a/libs/requests/compat.py b/libs/requests/compat.py index 224bfd02..fec7a01d 100644 --- a/libs/requests/compat.py +++ b/libs/requests/compat.py @@ -86,8 +86,10 @@ if is_py2: from .packages.oreos.monkeys import SimpleCookie from StringIO import StringIO - str = unicode bytes = str + str = unicode + basestring = basestring + elif is_py3: @@ -99,4 +101,5 @@ elif is_py3: str = str bytes = bytes + basestring = (str,bytes) diff --git a/libs/requests/defaults.py b/libs/requests/defaults.py index 424d3731..6a7ea270 100644 --- a/libs/requests/defaults.py +++ b/libs/requests/defaults.py @@ -15,10 +15,16 @@ Configurations: :max_retries: The number of times a request should be retried in the event of a connection failure. :danger_mode: If true, Requests will raise errors immediately. :safe_mode: If true, Requests will catch all errors. +:strict_mode: If true, Requests will do its best to follow RFCs (e.g. POST redirects). :pool_maxsize: The maximium size of an HTTP connection pool. :pool_connections: The number of active HTTP connection pools to use. +:encode_uri: If true, URIs will automatically be percent-encoded. +:trust_env: If true, the surrouding environment will be trusted (environ, netrc). + """ +SCHEMAS = ['http', 'https'] + from . import __version__ defaults = dict() @@ -37,4 +43,9 @@ defaults['pool_maxsize'] = 10 defaults['max_retries'] = 0 defaults['danger_mode'] = False defaults['safe_mode'] = False +defaults['strict_mode'] = False defaults['keep_alive'] = True +defaults['encode_uri'] = True +defaults['trust_env'] = True + + diff --git a/libs/requests/exceptions.py b/libs/requests/exceptions.py index c7b98e6f..3c262e36 100644 --- a/libs/requests/exceptions.py +++ b/libs/requests/exceptions.py @@ -8,12 +8,13 @@ This module contains the set of Requests' exceptions. """ -class RequestException(Exception): +class RequestException(RuntimeError): """There was an ambiguous exception that occurred while handling your request.""" class HTTPError(RequestException): """An HTTP error occurred.""" + response = None class ConnectionError(RequestException): """A Connection error occurred.""" @@ -29,3 +30,9 @@ class URLRequired(RequestException): class TooManyRedirects(RequestException): """Too many redirects.""" + +class MissingSchema(RequestException, ValueError): + """The URL schema (e.g. http or https) is missing.""" + +class InvalidSchema(RequestException, ValueError): + """See defaults.py for valid schemas.""" \ No newline at end of file diff --git a/libs/requests/models.py b/libs/requests/models.py index c200896c..70e35036 100644 --- a/libs/requests/models.py +++ b/libs/requests/models.py @@ -21,14 +21,16 @@ from .packages.urllib3.exceptions import SSLError as _SSLError from .packages.urllib3.exceptions import HTTPError as _HTTPError from .packages.urllib3 import connectionpool, poolmanager from .packages.urllib3.filepost import encode_multipart_formdata +from .defaults import SCHEMAS from .exceptions import ( ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects, - URLRequired, SSLError) + URLRequired, SSLError, MissingSchema, InvalidSchema) from .utils import ( - get_encoding_from_headers, stream_decode_response_unicode, - stream_decompress, guess_filename, requote_path, dict_from_string) - -from .compat import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, str, bytes, SimpleCookie, is_py3, is_py2 + get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri, + dict_from_string, stream_decode_response_unicode, get_netrc_auth) +from .compat import ( + urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes, + SimpleCookie, is_py2) # Import chardet if it is available. try: @@ -39,7 +41,6 @@ except ImportError: REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) - class Request(object): """The :class:`Request ` object. It carries out all functionality of Requests. Recommended interface is with the Requests functions. @@ -62,18 +63,17 @@ class Request(object): config=None, _poolmanager=None, verify=None, - session=None): + session=None, + cert=None): + + #: Dictionary of configurations for this request. + self.config = dict(config or []) #: Float describes the timeout of the request. # (Use socket.setdefaulttimeout() as fallback) self.timeout = timeout #: Request URL. - - # if isinstance(url, str): - # url = url.encode('utf-8') - # print(dir(url)) - self.url = url #: Dictionary of HTTP Headers to attach to the :class:`Request `. @@ -103,6 +103,14 @@ class Request(object): # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) self.proxies = dict(proxies or []) + # If no proxies are given, allow configuration by environment variables + # HTTP_PROXY and HTTPS_PROXY. + if not self.proxies and self.config.get('trust_env'): + if 'HTTP_PROXY' in os.environ: + self.proxies['http'] = os.environ['HTTP_PROXY'] + if 'HTTPS_PROXY' in os.environ: + self.proxies['https'] = os.environ['HTTPS_PROXY'] + self.data, self._enc_data = self._encode_params(data) self.params, self._enc_params = self._encode_params(params) @@ -116,9 +124,6 @@ class Request(object): #: CookieJar to attach to :class:`Request `. self.cookies = dict(cookies or []) - #: Dictionary of configurations for this request. - self.config = dict(config or []) - #: True if Request has been sent. self.sent = False @@ -139,6 +144,9 @@ class Request(object): #: SSL Verification. self.verify = verify + #: SSL Certificate + self.cert = cert + if headers: headers = CaseInsensitiveDict(self.headers) else: @@ -152,15 +160,9 @@ class Request(object): self.headers = headers self._poolmanager = _poolmanager - # Pre-request hook. - r = dispatch_hook('pre_request', hooks, self) - self.__dict__.update(r.__dict__) - - def __repr__(self): return '' % (self.method) - def _build_response(self, resp): """Build internal :class:`Response ` object from given response. @@ -200,29 +202,36 @@ class Request(object): # Save original response for later. response.raw = resp - response.url = self.full_url + if isinstance(self.full_url, bytes): + response.url = self.full_url.decode('utf-8') + else: + response.url = self.full_url return response history = [] r = build(resp) - cookies = self.cookies + self.cookies.update(r.cookies) if r.status_code in REDIRECT_STATI and not self.redirect: - while ( - ('location' in r.headers) and - ((r.status_code is codes.see_other) or (self.allow_redirects)) - ): + while (('location' in r.headers) and + ((r.status_code is codes.see_other) or (self.allow_redirects))): + + r.content # Consume socket so it can be released if not len(history) < self.config.get('max_redirects'): raise TooManyRedirects() + # Release the connection back into the pool. + r.raw.release_conn() + history.append(r) url = r.headers['location'] + data = self.data # Handle redirection without scheme (see: RFC 1808 Section 4) if url.startswith('//'): @@ -232,14 +241,29 @@ class Request(object): # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: - url = urljoin(r.url, url) + url = urljoin(r.url, + # Compliant with RFC3986, we percent + # encode the url. + requote_uri(url)) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is codes.see_other: method = 'GET' + data = None else: method = self.method + # Do what the browsers do if strict_mode is off... + if (not self.config.get('strict_mode')): + + if r.status_code in (codes.moved, codes.found) and self.method == 'POST': + method = 'GET' + data = None + + if (r.status_code == 303) and self.method != 'HEAD': + method = 'GET' + data = None + # Remove the cookie headers that were sent. headers = self.headers try: @@ -254,18 +278,19 @@ class Request(object): method=method, params=self.session.params, auth=self.auth, - cookies=cookies, + cookies=self.cookies, redirect=True, + data=data, config=self.config, timeout=self.timeout, _poolmanager=self._poolmanager, - proxies = self.proxies, - verify = self.verify, - session = self.session + proxies=self.proxies, + verify=self.verify, + session=self.session, + cert=self.cert ) request.send() - cookies.update(request.response.cookies) r = request.response self.cookies.update(r.cookies) @@ -275,7 +300,6 @@ class Request(object): self.response.request = self self.response.cookies.update(self.cookies) - @staticmethod def _encode_params(data): """Encode parameters in a piece of data. @@ -288,10 +312,12 @@ class Request(object): returns it twice. """ + if isinstance(data, bytes): + return data, data + if hasattr(data, '__iter__') and not isinstance(data, str): data = dict(data) - if hasattr(data, 'items'): result = [] for k, vs in list(data.items()): @@ -314,30 +340,44 @@ class Request(object): # Support for unicode domain names and paths. scheme, netloc, path, params, query, fragment = urlparse(url) - if not scheme: - raise ValueError("Invalid URL %r: No schema supplied" % url) + raise MissingSchema("Invalid URL %r: No schema supplied" % url) + + if not scheme in SCHEMAS: + raise InvalidSchema("Invalid scheme %r" % scheme) netloc = netloc.encode('idna').decode('utf-8') + if not path: + path = '/' + + if is_py2: + if isinstance(scheme, str): + scheme = scheme.encode('utf-8') + if isinstance(netloc, str): + netloc = netloc.encode('utf-8') if isinstance(path, str): path = path.encode('utf-8') + if isinstance(params, str): + params = params.encode('utf-8') + if isinstance(query, str): + query = query.encode('utf-8') + if isinstance(fragment, str): + fragment = fragment.encode('utf-8') - path = requote_path(path) - - # print([ scheme, netloc, path, params, query, fragment ]) - # print('---------------------') - - url = (urlunparse([ scheme, netloc, path, params, query, fragment ])) + url = (urlunparse([scheme, netloc, path, params, query, fragment])) if self._enc_params: if urlparse(url).query: - return '%s&%s' % (url, self._enc_params) + url = '%s&%s' % (url, self._enc_params) else: - return '%s?%s' % (url, self._enc_params) - else: - return url + url = '%s?%s' % (url, self._enc_params) + + if self.config.get('encode_uri', True): + url = requote_uri(url) + + return url @property def path_url(self): @@ -355,9 +395,6 @@ class Request(object): if not path: path = '/' - # if is_py3: - path = quote(path.encode('utf-8')) - url.append(path) query = p.query @@ -365,19 +402,15 @@ class Request(object): url.append('?') url.append(query) - # print(url) - return ''.join(url) - def register_hook(self, event, hook): """Properly register a hook.""" return self.hooks[event].append(hook) - def send(self, anyway=False, prefetch=False): - """Sends the request. Returns True of successful, false if not. + """Sends the request. Returns True of successful, False if not. If there was an HTTPError during transmission, self.response.status_code will contain the HTTPError code. @@ -435,6 +468,10 @@ class Request(object): if (content_type) and (not 'content-type' in self.headers): self.headers['Content-Type'] = content_type + # Use .netrc auth if none was provided. + if not self.auth and self.config.get('trust_env'): + self.auth = get_netrc_auth(url) + if self.auth: if isinstance(self.auth, tuple) and len(self.auth) == 2: # special-case basic HTTP auth @@ -472,13 +509,12 @@ class Request(object): if self.verify is not True: cert_loc = self.verify - # Look for configuration. - if not cert_loc: + if not cert_loc and self.config.get('trust_env'): cert_loc = os.environ.get('REQUESTS_CA_BUNDLE') # Curl compatiblity. - if not cert_loc: + if not cert_loc and self.config.get('trust_env'): cert_loc = os.environ.get('CURL_CA_BUNDLE') # Use the awesome certifi list. @@ -491,6 +527,13 @@ class Request(object): conn.cert_reqs = 'CERT_NONE' conn.ca_certs = None + if self.cert and self.verify: + if len(self.cert) == 2: + conn.cert_file = self.cert[0] + conn.key_file = self.cert[1] + else: + conn.cert_file = self.cert + if not self.sent or anyway: if self.cookies: @@ -509,6 +552,10 @@ class Request(object): # Attach Cookie header to request. self.headers['Cookie'] = cookie_header + # Pre-request hook. + r = dispatch_hook('pre_request', self.hooks, self) + self.__dict__.update(r.__dict__) + try: # The inner try .. except re-raises certain exceptions as # internal exception types; the outer suppresses exceptions @@ -523,7 +570,7 @@ class Request(object): redirect=False, assert_same_host=False, preload_content=False, - decode_content=True, + decode_content=False, retries=self.config.get('max_retries', 0), timeout=self.timeout, ) @@ -613,7 +660,6 @@ class Response(object): #: Dictionary of configurations for this request. self.config = {} - def __repr__(self): return '' % (self.status_code) @@ -629,11 +675,10 @@ class Response(object): def ok(self): try: self.raise_for_status() - except HTTPError: + except RequestException: return False return True - def iter_content(self, chunk_size=10 * 1024, decode_unicode=False): """Iterates over the response data. This avoids reading the content at once into memory for large responses. The chunk size is the number @@ -653,81 +698,39 @@ class Response(object): yield chunk self._content_consumed = True - def generate_chunked(): - resp = self.raw._original_response - fp = resp.fp - if resp.chunk_left is not None: - pending_bytes = resp.chunk_left - while pending_bytes: - chunk = fp.read(min(chunk_size, pending_bytes)) - pending_bytes-=len(chunk) - yield chunk - fp.read(2) # throw away crlf - while 1: - #XXX correct line size? (httplib has 64kb, seems insane) - pending_bytes = fp.readline(40).strip() - pending_bytes = int(pending_bytes, 16) - if pending_bytes == 0: - break - while pending_bytes: - chunk = fp.read(min(chunk_size, pending_bytes)) - pending_bytes-=len(chunk) - yield chunk - fp.read(2) # throw away crlf - self._content_consumed = True - fp.close() - - - if getattr(getattr(self.raw, '_original_response', None), 'chunked', False): - gen = generate_chunked() - else: - gen = generate() - - if 'gzip' in self.headers.get('content-encoding', ''): - gen = stream_decompress(gen, mode='gzip') - elif 'deflate' in self.headers.get('content-encoding', ''): - gen = stream_decompress(gen, mode='deflate') + gen = stream_untransfer(generate(), self) if decode_unicode: gen = stream_decode_response_unicode(gen, self) return gen - def iter_lines(self, chunk_size=10 * 1024, decode_unicode=None): """Iterates over the response data, one line at a time. This avoids reading the content at once into memory for large responses. """ - #TODO: why rstrip by default pending = None - for chunk in self.iter_content(chunk_size, decode_unicode=decode_unicode): + for chunk in self.iter_content( + chunk_size=chunk_size, + decode_unicode=decode_unicode): if pending is not None: chunk = pending + chunk - lines = chunk.splitlines(True) + lines = chunk.splitlines() - for line in lines[:-1]: - yield line.rstrip() - - # Save the last part of the chunk for next iteration, to keep full line together - # lines may be empty for the last chunk of a chunked response - - if lines: - pending = lines[-1] - #if pending is a complete line, give it baack - if pending[-1] == '\n': - yield pending.rstrip() - pending = None + if lines[-1][-1] == chunk[-1]: + pending = lines.pop() else: pending = None - # Yield the last line - if pending is not None: - yield pending.rstrip() + for line in lines: + yield line + if pending is not None: + yield pending @property def content(self): @@ -740,13 +743,26 @@ class Response(object): raise RuntimeError( 'The content for this response was already consumed') - self._content = self.raw.read() + if self.status_code is 0: + self._content = None + else: + self._content = bytes().join(self.iter_content()) or bytes() + except AttributeError: self._content = None self._content_consumed = True return self._content + def _detected_encoding(self): + try: + detected = chardet.detect(self.content) or {} + return detected.get('encoding') + + # Trust that chardet isn't available or something went terribly wrong. + except Exception: + pass + @property def text(self): @@ -762,43 +778,40 @@ class Response(object): # Fallback to auto-detected encoding if chardet is available. if self.encoding is None: - try: - detected = chardet.detect(self.content) or {} - encoding = detected.get('encoding') - - # Trust that chardet isn't available or something went terribly wrong. - except Exception: - pass + encoding = self._detected_encoding() # Decode unicode from given encoding. try: - content = str(self.content, encoding) + content = str(self.content, encoding, errors='replace') + except LookupError: + # A LookupError is raised if the encoding was not found which could + # indicate a misspelling or similar mistake. + # + # So we try blindly encoding. + content = str(self.content, errors='replace') except (UnicodeError, TypeError): pass - # Try to fall back: - if not content: - try: - content = str(content, encoding, errors='replace') - except (UnicodeError, TypeError): - pass - return content - - def raise_for_status(self): + def raise_for_status(self, allow_redirects=True): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred.""" if self.error: raise self.error - if (self.status_code >= 300) and (self.status_code < 400): - raise HTTPError('%s Redirection' % self.status_code) + if (self.status_code >= 300) and (self.status_code < 400) and not allow_redirects: + http_error = HTTPError('%s Redirection' % self.status_code) + http_error.response = self + raise http_error elif (self.status_code >= 400) and (self.status_code < 500): - raise HTTPError('%s Client Error' % self.status_code) + http_error = HTTPError('%s Client Error' % self.status_code) + http_error.response = self + raise http_error + elif (self.status_code >= 500) and (self.status_code < 600): - raise HTTPError('%s Server Error' % self.status_code) - - + http_error = HTTPError('%s Server Error' % self.status_code) + http_error.response = self + raise http_error diff --git a/libs/requests/packages/oreos/monkeys.py b/libs/requests/packages/oreos/monkeys.py index 2269e30b..2cf90163 100644 --- a/libs/requests/packages/oreos/monkeys.py +++ b/libs/requests/packages/oreos/monkeys.py @@ -249,10 +249,13 @@ class CookieError(Exception): # quoted with a preceeding '\' slash. # # These are taken from RFC2068 and RFC2109. +# _RFC2965Forbidden is the list of forbidden chars we accept anyway # _LegalChars is the list of chars which don't require "'s # _Translator hash-table for fast quoting # -_LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~[]_" +_RFC2965Forbidden = "[]:{}=" +_LegalChars = ( string.ascii_letters + string.digits + + "!#$%&'*+-.^_`|~_@" + _RFC2965Forbidden ) _Translator = { '\000' : '\\000', '\001' : '\\001', '\002' : '\\002', '\003' : '\\003', '\004' : '\\004', '\005' : '\\005', diff --git a/libs/requests/packages/oreos/structures.py b/libs/requests/packages/oreos/structures.py index 063d5f96..83292777 100644 --- a/libs/requests/packages/oreos/structures.py +++ b/libs/requests/packages/oreos/structures.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """ -oreos.sructures -~~~~~~~~~~~~~~~ +oreos.structures +~~~~~~~~~~~~~~~~ The plastic blue packaging. @@ -362,7 +362,7 @@ class MultiDict(TypeConversionDict): """ try: return dict.pop(self, key)[0] - except KeyError, e: + except KeyError as e: if default is not _missing: return default raise KeyError(str(e)) @@ -372,7 +372,7 @@ class MultiDict(TypeConversionDict): try: item = dict.popitem(self) return (item[0], item[1][0]) - except KeyError, e: + except KeyError as e: raise KeyError(str(e)) def poplist(self, key): @@ -389,7 +389,7 @@ class MultiDict(TypeConversionDict): """Pop a ``(key, list)`` tuple from the dict.""" try: return dict.popitem(self) - except KeyError, e: + except KeyError as e: raise KeyError(str(e)) def __copy__(self): diff --git a/libs/requests/packages/urllib3/__init__.py b/libs/requests/packages/urllib3/__init__.py index 5f70c563..2d6fecec 100644 --- a/libs/requests/packages/urllib3/__init__.py +++ b/libs/requests/packages/urllib3/__init__.py @@ -10,26 +10,20 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.1' +__version__ = '1.3' from .connectionpool import ( HTTPConnectionPool, HTTPSConnectionPool, - connection_from_url, - get_host, - make_headers) - - -from .exceptions import ( - HTTPError, - MaxRetryError, - SSLError, - TimeoutError) + connection_from_url +) +from . import exceptions +from .filepost import encode_multipart_formdata from .poolmanager import PoolManager, ProxyManager, proxy_from_url from .response import HTTPResponse -from .filepost import encode_multipart_formdata +from .util import make_headers, get_host # Set default logging handler to avoid "No handler found" warnings. diff --git a/libs/requests/packages/urllib3/connectionpool.py b/libs/requests/packages/urllib3/connectionpool.py index 52b1802b..c3cb3b12 100644 --- a/libs/requests/packages/urllib3/connectionpool.py +++ b/libs/requests/packages/urllib3/connectionpool.py @@ -9,45 +9,51 @@ import socket from socket import error as SocketError, timeout as SocketTimeout -try: - from select import poll, POLLIN -except ImportError: # Doesn't exist on OSX and other platforms - from select import select - poll = False - try: # Python 3 - from http.client import HTTPConnection, HTTPSConnection, HTTPException + from http.client import HTTPConnection, HTTPException from http.client import HTTP_PORT, HTTPS_PORT except ImportError: - from httplib import HTTPConnection, HTTPSConnection, HTTPException + from httplib import HTTPConnection, HTTPException from httplib import HTTP_PORT, HTTPS_PORT try: # Python 3 - from queue import Queue, Empty, Full + from queue import LifoQueue, Empty, Full except ImportError: - from Queue import Queue, Empty, Full + from Queue import LifoQueue, Empty, Full + try: # Compiled with SSL? + HTTPSConnection = object + BaseSSLError = None + ssl = None + + try: # Python 3 + from http.client import HTTPSConnection + except ImportError: + from httplib import HTTPSConnection + import ssl BaseSSLError = ssl.SSLError -except ImportError: - ssl = None - BaseSSLError = None + +except (ImportError, AttributeError): + pass -from .packages.ssl_match_hostname import match_hostname, CertificateError from .request import RequestMethods from .response import HTTPResponse -from .exceptions import (SSLError, - MaxRetryError, - TimeoutError, - HostChangedError, +from .util import get_host, is_connection_dropped +from .exceptions import ( EmptyPoolError, + HostChangedError, + MaxRetryError, + SSLError, + TimeoutError, ) from .packages.ssl_match_hostname import match_hostname, CertificateError from .packages import six + xrange = six.moves.xrange log = logging.getLogger(__name__) @@ -59,6 +65,7 @@ port_by_scheme = { 'https': HTTPS_PORT, } + ## Connection objects (extension of httplib) class VerifiedHTTPSConnection(HTTPSConnection): @@ -94,6 +101,7 @@ class VerifiedHTTPSConnection(HTTPSConnection): if self.ca_certs: match_hostname(self.sock.getpeercert(), self.host) + ## Pool objects class ConnectionPool(object): @@ -103,6 +111,7 @@ class ConnectionPool(object): """ scheme = None + QueueCls = LifoQueue def __init__(self, host, port=None): self.host = host @@ -156,11 +165,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1, block=False, headers=None): - self.host = host - self.port = port + super(HTTPConnectionPool, self).__init__(host, port) + self.strict = strict self.timeout = timeout - self.pool = Queue(maxsize) + self.pool = self.QueueCls(maxsize) self.block = block self.headers = headers or {} @@ -198,7 +207,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn = self.pool.get(block=self.block, timeout=timeout) # If this is a persistent connection, check if it got disconnected - if conn and conn.sock and is_connection_dropped(conn): + if conn and is_connection_dropped(conn): log.info("Resetting dropped connection: %s" % self.host) conn.close() @@ -242,9 +251,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): timeout = self.timeout conn.timeout = timeout # This only does anything in Py26+ - conn.request(method, url, **httplib_request_kw) - conn.sock.settimeout(timeout) + + # Set timeout + sock = getattr(conn, 'sock', False) # AppEngine doesn't have sock attr. + if sock: + sock.settimeout(timeout) + httplib_response = conn.getresponse() log.debug("\"%s %s %s\" %s %s" % @@ -281,7 +294,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): .. note:: More commonly, it's appropriate to use a convenience method provided - by :class:`.RequestMethods`, such as :meth:`.request`. + by :class:`.RequestMethods`, such as :meth:`request`. .. note:: @@ -468,7 +481,11 @@ class HTTPSConnectionPool(HTTPConnectionPool): log.info("Starting new HTTPS connection (%d): %s" % (self.num_connections, self.host)) - if not ssl: + if not ssl: # Platform-specific: Python compiled without +ssl + if not HTTPSConnection or HTTPSConnection is object: + raise SSLError("Can't connect to HTTPS URL because the SSL " + "module is not available.") + return HTTPSConnection(host=self.host, port=self.port) connection = VerifiedHTTPSConnection(host=self.host, port=self.port) @@ -477,87 +494,6 @@ class HTTPSConnectionPool(HTTPConnectionPool): return connection -## Helpers - -def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, - basic_auth=None): - """ - Shortcuts for generating request headers. - - :param keep_alive: - If ``True``, adds 'connection: keep-alive' header. - - :param accept_encoding: - Can be a boolean, list, or string. - ``True`` translates to 'gzip,deflate'. - List will get joined by comma. - String will be used as provided. - - :param user_agent: - String representing the user-agent you want, such as - "python-urllib3/0.6" - - :param basic_auth: - Colon-separated username:password string for 'authorization: basic ...' - auth header. - - Example: :: - - >>> make_headers(keep_alive=True, user_agent="Batman/1.0") - {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} - >>> make_headers(accept_encoding=True) - {'accept-encoding': 'gzip,deflate'} - """ - headers = {} - if accept_encoding: - if isinstance(accept_encoding, str): - pass - elif isinstance(accept_encoding, list): - accept_encoding = ','.join(accept_encoding) - else: - accept_encoding = 'gzip,deflate' - headers['accept-encoding'] = accept_encoding - - if user_agent: - headers['user-agent'] = user_agent - - if keep_alive: - headers['connection'] = 'keep-alive' - - if basic_auth: - headers['authorization'] = 'Basic ' + \ - basic_auth.encode('base64').strip() - - return headers - - -def get_host(url): - """ - Given a url, return its scheme, host and port (None if it's not there). - - For example: :: - - >>> get_host('http://google.com/mail/') - ('http', 'google.com', None) - >>> get_host('google.com:80') - ('http', 'google.com', 80) - """ - # This code is actually similar to urlparse.urlsplit, but much - # simplified for our needs. - port = None - scheme = 'http' - if '://' in url: - scheme, url = url.split('://', 1) - if '/' in url: - url, _path = url.split('/', 1) - if '@' in url: - _auth, url = url.split('@', 1) - if ':' in url: - url, port = url.split(':', 1) - port = int(port) - return scheme, url, port - - def connection_from_url(url, **kw): """ Given a url, return an :class:`.ConnectionPool` instance of its host. @@ -583,22 +519,3 @@ def connection_from_url(url, **kw): return HTTPSConnectionPool(host, port=port, **kw) else: return HTTPConnectionPool(host, port=port, **kw) - - -def is_connection_dropped(conn): - """ - Returns True if the connection is dropped and should be closed. - - :param conn: - ``HTTPConnection`` object. - """ - if not poll: - return select([conn.sock], [], [], 0.0)[0] - - # This version is better on platforms that support it. - p = poll() - p.register(conn.sock, POLLIN) - for (fno, ev) in p.poll(0.0): - if fno == conn.sock.fileno(): - # Either data is buffered (bad), or the connection is dropped. - return True diff --git a/libs/requests/packages/urllib3/exceptions.py b/libs/requests/packages/urllib3/exceptions.py index 0bffeb47..15c9699e 100644 --- a/libs/requests/packages/urllib3/exceptions.py +++ b/libs/requests/packages/urllib3/exceptions.py @@ -4,6 +4,7 @@ # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php + ## Base Exceptions class HTTPError(Exception): @@ -27,18 +28,20 @@ class SSLError(HTTPError): class MaxRetryError(PoolError): "Raised when the maximum number of retries is exceeded." + def __init__(self, pool, url): - PoolError.__init__(self, pool, - "Max retries exceeded with url: %s" % url) + message = "Max retries exceeded with url: %s" % url + PoolError.__init__(self, pool, message) self.url = url class HostChangedError(PoolError): "Raised when an existing pool gets a request for a foreign host." + def __init__(self, pool, url, retries=3): - PoolError.__init__(self, pool, - "Tried to open a foreign host with url: %s" % url) + message = "Tried to open a foreign host with url: %s" % url + PoolError.__init__(self, pool, message) self.url = url self.retries = retries @@ -52,3 +55,13 @@ class TimeoutError(PoolError): class EmptyPoolError(PoolError): "Raised when a pool runs out of connections and no more are allowed." pass + + +class LocationParseError(ValueError, HTTPError): + "Raised when get_host or similar fails to parse the URL input." + + def __init__(self, location): + message = "Failed to parse: %s" % location + super(LocationParseError, self).__init__(self, message) + + self.location = location diff --git a/libs/requests/packages/urllib3/filepost.py b/libs/requests/packages/urllib3/filepost.py index e1ec8af7..344a1030 100644 --- a/libs/requests/packages/urllib3/filepost.py +++ b/libs/requests/packages/urllib3/filepost.py @@ -24,15 +24,29 @@ def get_content_type(filename): return mimetypes.guess_type(filename)[0] or 'application/octet-stream' +def iter_fields(fields): + """ + Iterate over fields. + + Supports list of (k, v) tuples and dicts. + """ + if isinstance(fields, dict): + return ((k, v) for k, v in six.iteritems(fields)) + + return ((k, v) for k, v in fields) + + def encode_multipart_formdata(fields, boundary=None): """ Encode a dictionary of ``fields`` using the multipart/form-data mime format. :param fields: - Dictionary of fields. The key is treated as the field name, and the - value as the body of the form-data. If the value is a tuple of two - elements, then the first element is treated as the filename of the - form-data section. + Dictionary of fields or list of (key, value) field tuples. The key is + treated as the field name, and the value as the body of the form-data + bytes. If the value is a tuple of two elements, then the first element + is treated as the filename of the form-data section. + + Field names and filenames must be unicode. :param boundary: If not specified, then a random boundary will be generated using @@ -42,7 +56,7 @@ def encode_multipart_formdata(fields, boundary=None): if boundary is None: boundary = choose_boundary() - for fieldname, value in six.iteritems(fields): + for fieldname, value in iter_fields(fields): body.write(b('--%s\r\n' % (boundary))) if isinstance(value, tuple): diff --git a/libs/requests/packages/urllib3/poolmanager.py b/libs/requests/packages/urllib3/poolmanager.py index f194b2e8..310ea21d 100644 --- a/libs/requests/packages/urllib3/poolmanager.py +++ b/libs/requests/packages/urllib3/poolmanager.py @@ -39,11 +39,11 @@ class PoolManager(RequestMethods): Example: :: - >>> manager = PoolManager() + >>> manager = PoolManager(num_pools=2) >>> r = manager.urlopen("http://google.com/") >>> r = manager.urlopen("http://google.com/mail") >>> r = manager.urlopen("http://yahoo.com/") - >>> len(r.pools) + >>> len(manager.pools) 2 """ @@ -117,9 +117,19 @@ class ProxyManager(RequestMethods): def __init__(self, proxy_pool): self.proxy_pool = proxy_pool + def _set_proxy_headers(self, headers=None): + headers = headers or {} + + # Same headers are curl passes for --proxy1.0 + headers['Accept'] = '*/*' + headers['Proxy-Connection'] = 'Keep-Alive' + + return headers + def urlopen(self, method, url, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." kw['assert_same_host'] = False + kw['headers'] = self._set_proxy_headers(kw.get('headers')) return self.proxy_pool.urlopen(method, url, **kw) diff --git a/libs/requests/packages/urllib3/request.py b/libs/requests/packages/urllib3/request.py index 5ea26a0f..569ac966 100644 --- a/libs/requests/packages/urllib3/request.py +++ b/libs/requests/packages/urllib3/request.py @@ -44,7 +44,7 @@ class RequestMethods(object): def urlopen(self, method, url, body=None, headers=None, encode_multipart=True, multipart_boundary=None, - **kw): + **kw): # Abstract raise NotImplemented("Classes extending RequestMethods must implement " "their own ``urlopen`` method.") @@ -126,22 +126,3 @@ class RequestMethods(object): return self.urlopen(method, url, body=body, headers=headers, **urlopen_kw) - - # Deprecated: - - def get_url(self, url, fields=None, **urlopen_kw): - """ - .. deprecated:: 1.0 - Use :meth:`request` instead. - """ - return self.request_encode_url('GET', url, fields=fields, - **urlopen_kw) - - def post_url(self, url, fields=None, headers=None, **urlopen_kw): - """ - .. deprecated:: 1.0 - Use :meth:`request` instead. - """ - return self.request_encode_body('POST', url, fields=fields, - headers=headers, - **urlopen_kw) diff --git a/libs/requests/packages/urllib3/response.py b/libs/requests/packages/urllib3/response.py index e0239703..5fab8243 100644 --- a/libs/requests/packages/urllib3/response.py +++ b/libs/requests/packages/urllib3/response.py @@ -11,12 +11,7 @@ import zlib from io import BytesIO from .exceptions import HTTPError - - -try: - basestring = basestring -except NameError: # Python 3 - basestring = (str, bytes) +from .packages.six import string_types as basestring log = logging.getLogger(__name__) @@ -176,11 +171,22 @@ class HTTPResponse(object): with ``original_response=r``. """ + # Normalize headers between different versions of Python + headers = {} + for k, v in r.getheaders(): + # Python 3: Header keys are returned capitalised + k = k.lower() + + has_value = headers.get(k) + if has_value: # Python 3: Repeating header keys are unmerged. + v = ', '.join([has_value, v]) + + headers[k] = v + # HTTPResponse objects in Python 3 don't have a .strict attribute strict = getattr(r, 'strict', 0) return ResponseCls(body=r, - # In Python 3, the header keys are returned capitalised - headers=dict((k.lower(), v) for k,v in r.getheaders()), + headers=headers, status=r.status, version=r.version, reason=r.reason, diff --git a/libs/requests/packages/urllib3/util.py b/libs/requests/packages/urllib3/util.py new file mode 100644 index 00000000..2684a2fd --- /dev/null +++ b/libs/requests/packages/urllib3/util.py @@ -0,0 +1,136 @@ +# urllib3/util.py +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + + +from base64 import b64encode + +try: + from select import poll, POLLIN +except ImportError: # `poll` doesn't exist on OSX and other platforms + poll = False + try: + from select import select + except ImportError: # `select` doesn't exist on AppEngine. + select = False + +from .packages import six +from .exceptions import LocationParseError + + +def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, + basic_auth=None): + """ + Shortcuts for generating request headers. + + :param keep_alive: + If ``True``, adds 'connection: keep-alive' header. + + :param accept_encoding: + Can be a boolean, list, or string. + ``True`` translates to 'gzip,deflate'. + List will get joined by comma. + String will be used as provided. + + :param user_agent: + String representing the user-agent you want, such as + "python-urllib3/0.6" + + :param basic_auth: + Colon-separated username:password string for 'authorization: basic ...' + auth header. + + Example: :: + + >>> make_headers(keep_alive=True, user_agent="Batman/1.0") + {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} + >>> make_headers(accept_encoding=True) + {'accept-encoding': 'gzip,deflate'} + """ + headers = {} + if accept_encoding: + if isinstance(accept_encoding, str): + pass + elif isinstance(accept_encoding, list): + accept_encoding = ','.join(accept_encoding) + else: + accept_encoding = 'gzip,deflate' + headers['accept-encoding'] = accept_encoding + + if user_agent: + headers['user-agent'] = user_agent + + if keep_alive: + headers['connection'] = 'keep-alive' + + if basic_auth: + headers['authorization'] = 'Basic ' + \ + b64encode(six.b(basic_auth)).decode('utf-8') + + return headers + + +def get_host(url): + """ + Given a url, return its scheme, host and port (None if it's not there). + + For example: :: + + >>> get_host('http://google.com/mail/') + ('http', 'google.com', None) + >>> get_host('google.com:80') + ('http', 'google.com', 80) + """ + + # This code is actually similar to urlparse.urlsplit, but much + # simplified for our needs. + port = None + scheme = 'http' + + if '://' in url: + scheme, url = url.split('://', 1) + if '/' in url: + url, _path = url.split('/', 1) + if '@' in url: + _auth, url = url.split('@', 1) + if ':' in url: + url, port = url.split(':', 1) + + if not port.isdigit(): + raise LocationParseError("Failed to parse: %s" % url) + + port = int(port) + + return scheme, url, port + + + +def is_connection_dropped(conn): + """ + Returns True if the connection is dropped and should be closed. + + :param conn: + ``HTTPConnection`` object. + + Note: For platforms like AppEngine, this will always return ``False`` to + let the platform handle connection recycling transparently for us. + """ + sock = getattr(conn, 'sock', False) + if not sock: #Platform-specific: AppEngine + return False + + if not poll: # Platform-specific + if not select: #Platform-specific: AppEngine + return False + + return select([sock], [], [], 0.0)[0] + + # This version is better on platforms that support it. + p = poll() + p.register(sock, POLLIN) + for (fno, ev) in p.poll(0.0): + if fno == sock.fileno(): + # Either data is buffered (bad), or the connection is dropped. + return True diff --git a/libs/requests/sessions.py b/libs/requests/sessions.py index d9683b03..94c94bf9 100644 --- a/libs/requests/sessions.py +++ b/libs/requests/sessions.py @@ -40,7 +40,7 @@ def merge_kwargs(local_kwarg, default_kwarg): kwargs.update(local_kwarg) # Remove keys that are set to None. - for (k,v) in list(local_kwarg.items()): + for (k, v) in list(local_kwarg.items()): if v is None: del kwargs[k] @@ -52,7 +52,7 @@ class Session(object): __attrs__ = [ 'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks', - 'params', 'config'] + 'params', 'config', 'verify', 'cert'] def __init__(self, @@ -64,7 +64,9 @@ class Session(object): hooks=None, params=None, config=None, - verify=True): + prefetch=False, + verify=True, + cert=None): self.headers = headers or {} self.cookies = cookies or {} @@ -74,15 +76,14 @@ class Session(object): self.hooks = hooks or {} self.params = params or {} self.config = config or {} + self.prefetch = prefetch self.verify = verify + self.cert = cert for (k, v) in list(defaults.items()): self.config.setdefault(k, v) - self.poolmanager = PoolManager( - num_pools=self.config.get('pool_connections'), - maxsize=self.config.get('pool_maxsize') - ) + self.init_poolmanager() # Set up a CookieJar to be used by default self.cookies = {} @@ -91,6 +92,12 @@ class Session(object): if cookies is not None: self.cookies.update(cookies) + def init_poolmanager(self): + self.poolmanager = PoolManager( + num_pools=self.config.get('pool_connections'), + maxsize=self.config.get('pool_maxsize') + ) + def __repr__(self): return '' % (id(self)) @@ -108,13 +115,14 @@ class Session(object): files=None, auth=None, timeout=None, - allow_redirects=False, + allow_redirects=True, proxies=None, hooks=None, return_response=True, config=None, prefetch=False, - verify=None): + verify=None, + cert=None): """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. @@ -128,12 +136,13 @@ class Session(object): :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. - :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. + :param allow_redirects: (optional) Boolean. Set to True by default. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param return_response: (optional) If False, an un-sent Request object will returned. :param config: (optional) A configuration dictionary. :param prefetch: (optional) if ``True``, the response content will be immediately downloaded. :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. + :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. """ method = str(method).upper() @@ -145,9 +154,7 @@ class Session(object): headers = {} if headers is None else headers params = {} if params is None else params hooks = {} if hooks is None else hooks - - if verify is None: - verify = self.verify + prefetch = self.prefetch or prefetch # use session's hooks as defaults for key, cb in list(self.hooks.items()): @@ -173,6 +180,7 @@ class Session(object): proxies=proxies, config=config, verify=verify, + cert=cert, _poolmanager=self.poolmanager ) @@ -210,7 +218,7 @@ class Session(object): """Sends a GET request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ kwargs.setdefault('allow_redirects', True) @@ -221,7 +229,7 @@ class Session(object): """Sends a OPTIONS request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ kwargs.setdefault('allow_redirects', True) @@ -232,10 +240,10 @@ class Session(object): """Sends a HEAD request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ - kwargs.setdefault('allow_redirects', True) + kwargs.setdefault('allow_redirects', False) return self.request('head', url, **kwargs) @@ -244,7 +252,7 @@ class Session(object): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return self.request('post', url, data=data, **kwargs) @@ -255,7 +263,7 @@ class Session(object): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return self.request('put', url, data=data, **kwargs) @@ -266,7 +274,7 @@ class Session(object): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return self.request('patch', url, data=data, **kwargs) @@ -276,11 +284,20 @@ class Session(object): """Sends a DELETE request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param **kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. """ return self.request('delete', url, **kwargs) + def __getstate__(self): + return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + + def __setstate__(self, state): + for attr, value in state.items(): + setattr(self, attr, value) + + self.init_poolmanager() + def session(**kwargs): """Returns a :class:`Session` for context-management.""" diff --git a/libs/requests/utils.py b/libs/requests/utils.py index 0e0f69ee..b722d990 100644 --- a/libs/requests/utils.py +++ b/libs/requests/utils.py @@ -15,9 +15,54 @@ import os import random import re import zlib +from netrc import netrc, NetrcParseError from .compat import parse_http_list as _parse_list_header -from .compat import quote, unquote, cookielib, SimpleCookie, is_py2 +from .compat import quote, cookielib, SimpleCookie, is_py2, urlparse +from .compat import basestring, bytes, str + + +NETRC_FILES = ('.netrc', '_netrc') + + +def dict_to_sequence(d): + """Returns an internal sequence dictionary update.""" + + if hasattr(d, 'items'): + d = d.items() + + return d + + +def get_netrc_auth(url): + """Returns the Requests tuple auth for a given url from netrc.""" + + locations = (os.path.expanduser('~/{0}'.format(f)) for f in NETRC_FILES) + netrc_path = None + + for loc in locations: + if os.path.exists(loc) and not netrc_path: + netrc_path = loc + + # Abort early if there isn't one. + if netrc_path is None: + return netrc_path + + ri = urlparse(url) + + # Strip port numbers from netloc + host = ri.netloc.split(':')[0] + + try: + _netrc = netrc(netrc_path).authenticators(host) + if _netrc: + # Return with login / password + login_i = (0 if _netrc[0] else 1) + return (_netrc[login_i], _netrc[2]) + except (NetrcParseError, IOError, AttributeError): + # If there was a parsing error or a permissions issue reading the file, + # we'll just skip netrc auth + pass def dict_from_string(s): @@ -25,20 +70,26 @@ def dict_from_string(s): cookies = dict() - c = SimpleCookie() - c.load(s) + try: + c = SimpleCookie() + c.load(s) - for k,v in list(c.items()): - cookies.update({k: v.value}) + for k, v in list(c.items()): + cookies.update({k: v.value}) + # This stuff is not to be trusted. + except Exception: + pass return cookies + def guess_filename(obj): """Tries to guess the filename of the given object.""" name = getattr(obj, 'name', None) if name and name[0] != '<' and name[-1] != '>': return name + # From mitsuhiko/werkzeug (used with permission). def parse_list_header(value): """Parse lists as described by RFC 2068 Section 2. @@ -145,8 +196,14 @@ def header_expand(headers): if isinstance(headers, dict): headers = list(headers.items()) - + elif isinstance(headers, basestring): + return headers elif isinstance(headers, str): + # As discussed in https://github.com/kennethreitz/requests/issues/400 + # latin-1 is the most conservative encoding used on the web. Anyone + # who needs more can encode to a byte-string before calling + return headers.encode("latin-1") + elif headers is None: return headers for i, (value, params) in enumerate(headers): @@ -164,10 +221,9 @@ def header_expand(headers): collector.append('; '.join(_params)) - if not len(headers) == i+1: + if not len(headers) == i + 1: collector.append(', ') - # Remove trailing separators. if collector[-1] in (', ', '; '): del collector[-1] @@ -175,7 +231,6 @@ def header_expand(headers): return ''.join(collector) - def randombytes(n): """Return n random bytes.""" if is_py2: @@ -286,23 +341,6 @@ def get_encoding_from_headers(headers): return 'ISO-8859-1' -def unicode_from_html(content): - """Attempts to decode an HTML string into unicode. - If unsuccessful, the original content is returned. - """ - - encodings = get_encodings_from_content(content) - - for encoding in encodings: - - try: - return str(content, encoding) - except (UnicodeError, TypeError): - pass - - return content - - def stream_decode_response_unicode(iterator, r): """Stream decodes a iterator.""" @@ -354,15 +392,6 @@ def get_unicode_from_response(r): return r.content -def decode_gzip(content): - """Return gzip-decoded string. - - :param content: bytestring to gzip-decode. - """ - - return zlib.decompress(content, 16 + zlib.MAX_WBITS) - - def stream_decompress(iterator, mode='gzip'): """ Stream decodes an iterator over compressed data @@ -390,18 +419,53 @@ def stream_decompress(iterator, mode='gzip'): yield chunk else: # Make sure everything has been returned from the decompression object - buf = dec.decompress('') + buf = dec.decompress(bytes()) rv = buf + dec.flush() if rv: yield rv -def requote_path(path): - """Re-quote the given URL path component. +def stream_untransfer(gen, resp): + if 'gzip' in resp.headers.get('content-encoding', ''): + gen = stream_decompress(gen, mode='gzip') + elif 'deflate' in resp.headers.get('content-encoding', ''): + gen = stream_decompress(gen, mode='deflate') - This function passes the given path through an unquote/quote cycle to + return gen + + +# The unreserved URI characters (RFC 3986) +UNRESERVED_SET = frozenset( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + "0123456789-._~") + + +def unquote_unreserved(uri): + """Un-escape any percent-escape sequences in a URI that are unreserved + characters. + This leaves all reserved, illegal and non-ASCII bytes encoded. + """ + parts = uri.split('%') + for i in range(1, len(parts)): + h = parts[i][0:2] + if len(h) == 2: + c = chr(int(h, 16)) + if c in UNRESERVED_SET: + parts[i] = c + parts[i][2:] + else: + parts[i] = '%' + parts[i] + else: + parts[i] = '%' + parts[i] + return ''.join(parts) + + +def requote_uri(uri): + """Re-quote the given URI. + + This function passes the given URI through an unquote/quote cycle to ensure that it is fully and consistently quoted. """ - parts = path.split(b"/") - parts = (quote(unquote(part), safe=b"") for part in parts) - return b"/".join(parts) + # Unquote only the unreserved characters + # Then quote only illegal characters (do not quote reserved, unreserved, + # or '%') + return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~") diff --git a/libs/subliminal/__init__.py b/libs/subliminal/__init__.py index 155cf244..666440b6 100755 --- a/libs/subliminal/__init__.py +++ b/libs/subliminal/__init__.py @@ -1,23 +1,34 @@ # -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin # -# Subliminal - Subtitles, faster than your thoughts -# Copyright (c) 2011 Antoine Bertin +# This file is part of subliminal. # -# This file is part of Subliminal. -# -# Subliminal is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# Subliminal is distributed in the hope that it will be useful, +# subliminal is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . -# -__all__ = ['Subliminal'] +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from .api import list_subtitles, download_subtitles +from .async import Pool +from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, + MATCHING_CONFIDENCE) +from .infos import __version__ +import logging +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass -from core import Subliminal + +__all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE', + 'MATCHING_CONFIDENCE', 'list_subtitles', 'download_subtitles', 'Pool'] +logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/libs/subliminal/api.py b/libs/subliminal/api.py new file mode 100755 index 00000000..baff5f1a --- /dev/null +++ b/libs/subliminal/api.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin +# +# This file is part of subliminal. +# +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, + MATCHING_CONFIDENCE, create_list_tasks, consume_task, create_download_tasks, + group_by_video, key_subtitles) +from .languages import list_languages +import logging + + +__all__ = ['list_subtitles', 'download_subtitles'] +logger = logging.getLogger(__name__) + + +def list_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3): + """List subtitles in given paths according to the criteria + + :param paths: path(s) to video file or folder + :type paths: string or list + :param list languages: languages to search for, in preferred order + :param list services: services to use for the search, in preferred order + :param bool force: force searching for subtitles even if some are detected + :param bool multi: search multiple languages for the same video + :param string cache_dir: path to the cache directory to use + :param int max_depth: maximum depth for scanning entries + :return: found subtitles + :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`] + + """ + services = services or SERVICES + languages = set(languages or list_languages(1)) + if isinstance(paths, basestring): + paths = [paths] + if any([not isinstance(p, unicode) for p in paths]): + logger.warning(u'Not all entries are unicode') + results = [] + service_instances = {} + tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth) + for task in tasks: + try: + result = consume_task(task, service_instances) + results.append((task.video, result)) + except: + logger.error(u'Error consuming task %r' % task, exc_info=True) + for service_instance in service_instances.itervalues(): + service_instance.terminate() + return group_by_video(results) + + +def download_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, order=None): + """Download subtitles in given paths according to the criteria + + :param paths: path(s) to video file or folder + :type paths: string or list + :param list languages: languages to search for, in preferred order + :param list services: services to use for the search, in preferred order + :param bool force: force searching for subtitles even if some are detected + :param bool multi: search multiple languages for the same video + :param string cache_dir: path to the cache directory to use + :param int max_depth: maximum depth for scanning entries + :param order: preferred order for subtitles sorting + :type list: list of :data:`~subliminal.core.LANGUAGE_INDEX`, :data:`~subliminal.core.SERVICE_INDEX`, :data:`~subliminal.core.SERVICE_CONFIDENCE`, :data:`~subliminal.core.MATCHING_CONFIDENCE` + :return: found subtitles + :rtype: list of (:class:`~subliminal.videos.Video`, [:class:`~subliminal.subtitles.ResultSubtitle`]) + + """ + services = services or SERVICES + languages = languages or list_languages(1) + if isinstance(paths, basestring): + paths = [paths] + order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE] + subtitles_by_video = list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth) + for video, subtitles in subtitles_by_video.iteritems(): + subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True) + results = [] + service_instances = {} + tasks = create_download_tasks(subtitles_by_video, multi) + for task in tasks: + try: + result = consume_task(task, service_instances) + results.append(result) + except: + logger.error(u'Error consuming task %r' % task, exc_info=True) + for service_instance in service_instances.itervalues(): + service_instance.terminate() + return results diff --git a/libs/subliminal/async.py b/libs/subliminal/async.py new file mode 100755 index 00000000..ce18a278 --- /dev/null +++ b/libs/subliminal/async.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin +# +# This file is part of subliminal. +# +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from .core import (consume_task, LANGUAGE_INDEX, SERVICE_INDEX, + SERVICE_CONFIDENCE, MATCHING_CONFIDENCE, SERVICES, create_list_tasks, + create_download_tasks, group_by_video, key_subtitles) +from .languages import list_languages +from .tasks import StopTask +import Queue +import logging +import threading + + +logger = logging.getLogger(__name__) + + +class Worker(threading.Thread): + """Consume tasks and put the result in the queue""" + def __init__(self, tasks, results): + super(Worker, self).__init__() + self.tasks = tasks + self.results = results + self.services = {} + + def run(self): + while 1: + result = [] + try: + task = self.tasks.get(block=True) + if isinstance(task, StopTask): + break + result = consume_task(task, self.services) + self.results.put((task.video, result)) + except: + logger.error(u'Exception raised in worker %s' % self.name, exc_info=True) + finally: + self.tasks.task_done() + self.terminate() + logger.debug(u'Thread %s terminated' % self.name) + + def terminate(self): + """Terminate instantiated services""" + for service_name, service in self.services.iteritems(): + try: + service.terminate() + except: + logger.error(u'Exception raised when terminating service %s' % service_name, exc_info=True) + + +class Pool(object): + """Pool of workers""" + def __init__(self, size): + self.tasks = Queue.Queue() + self.results = Queue.Queue() + self.workers = [] + for _ in range(size): + self.workers.append(Worker(self.tasks, self.results)) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + self.stop() + self.join() + + def start(self): + """Start workers""" + for worker in self.workers: + worker.start() + + def stop(self): + """Stop workers""" + for _ in self.workers: + self.tasks.put(StopTask()) + + def join(self): + """Join the task queue""" + self.tasks.join() + + def collect(self): + """Collect available results + + :return: results of tasks + :rtype: list of :class:`~subliminal.tasks.Task` + + """ + results = [] + while 1: + try: + result = self.results.get(block=False) + results.append(result) + except Queue.Empty: + break + return results + + def list_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3): + """See :meth:`subliminal.list_subtitles`""" + services = services or SERVICES + languages = set(languages or list_languages(1)) + if isinstance(paths, basestring): + paths = [paths] + if any([not isinstance(p, unicode) for p in paths]): + logger.warning(u'Not all entries are unicode') + tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth) + for task in tasks: + self.tasks.put(task) + self.join() + results = self.collect() + return group_by_video(results) + + def download_subtitles(self, paths, languages=None, services=None, cache_dir=None, max_depth=3, force=True, multi=False, order=None): + """See :meth:`subliminal.download_subtitles`""" + services = services or SERVICES + languages = languages or list_languages(1) + if isinstance(paths, basestring): + paths = [paths] + order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE] + subtitles_by_video = self.list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth) + for video, subtitles in subtitles_by_video.iteritems(): + subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True) + tasks = create_download_tasks(subtitles_by_video, multi) + for task in tasks: + self.tasks.put(task) + self.join() + results = self.collect() + return results diff --git a/libs/subliminal/core.py b/libs/subliminal/core.py index 8f762b48..56f4347e 100755 --- a/libs/subliminal/core.py +++ b/libs/subliminal/core.py @@ -1,346 +1,173 @@ # -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin # -# Subliminal - Subtitles, faster than your thoughts -# Copyright (c) 2011 Antoine Bertin +# This file is part of subliminal. # -# This file is part of Subliminal. -# -# Subliminal is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# Subliminal is distributed in the hope that it will be useful, +# subliminal is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . -# -__all__ = ['PLUGINS', 'API_PLUGINS', 'IDLE', 'RUNNING', 'PAUSED', 'Subliminal', 'PluginWorker', 'matching_confidence', - 'LANGUAGE_INDEX', 'PLUGIN_INDEX', 'PLUGIN_CONFIDENCE', 'MATCHING_CONFIDENCE'] - - +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from .exceptions import DownloadFailedError +from .services import ServiceConfig +from .tasks import DownloadTask, ListTask +from .utils import get_keywords +from .videos import Episode, Movie, scan from collections import defaultdict -from exceptions import InvalidLanguageError, PluginError, BadStateError, \ - WrongTaskError, DownloadFailedError from itertools import groupby -from languages import list_languages -from utils import NullHandler -from tasks import Task, DownloadTask, ListTask, StopTask -import Queue import guessit import logging -import os -import plugins -import subtitles -import threading -import utils -import videos - -# init logger -logger = logging.getLogger('subliminal') -logger.addHandler(NullHandler()) - -# const -PLUGINS = ['OpenSubtitles', 'BierDopje', 'TheSubDB', 'SubsWiki', 'Subtitulos'] -API_PLUGINS = filter(lambda p: getattr(plugins, p).api_based, PLUGINS) -IDLE, RUNNING, PAUSED = range(3) -LANGUAGE_INDEX, PLUGIN_INDEX, PLUGIN_CONFIDENCE, MATCHING_CONFIDENCE = range(4) -class Subliminal(object): - """Main Subliminal class""" +__all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE', 'MATCHING_CONFIDENCE', + 'create_list_tasks', 'create_download_tasks', 'consume_task', 'matching_confidence', + 'key_subtitles', 'group_by_video'] +logger = logging.getLogger(__name__) +SERVICES = ['opensubtitles', 'bierdopje', 'subswiki', 'subtitulos', 'thesubdb'] +LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE = range(4) - def __init__(self, cache_dir=None, workers=None, multi=False, force=False, - max_depth=None, filemode=None, sort_order=None, plugins=None, languages=None): - self.multi = multi - self.sort_order = sort_order or [LANGUAGE_INDEX, PLUGIN_INDEX, PLUGIN_CONFIDENCE, MATCHING_CONFIDENCE] - self.force = force - self.max_depth = max_depth or 3 - self.taskQueue = Queue.PriorityQueue() - self.listResultQueue = Queue.Queue() - self.downloadResultQueue = Queue.Queue() - self.languages = languages or [] - self.plugins = plugins or API_PLUGINS - self._workers = workers or 4 - self.filemode = filemode - self.state = IDLE - self.cache_dir = cache_dir - try: - if cache_dir and not os.path.isdir(cache_dir): - os.makedirs(cache_dir) - logger.debug(u'Creating cache directory: %r' % cache_dir) - except: - self.cache_dir = None - logger.error(u'Failed to use the cache directory, continue without it') - def __enter__(self): - self.startWorkers() - return self +def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth): + """Create a list of :class:`~subliminal.tasks.ListTask` from one or more paths using the given criteria - def __exit__(self, *args): - self.stopWorkers(0) + :param paths: path(s) to video file or folder + :type paths: string or list + :param set languages: languages to search for + :param list services: services to use for the search + :param bool force: force searching for subtitles even if some are detected + :param bool multi: search multiple languages for the same video + :param string cache_dir: path to the cache directory to use + :param int max_depth: maximum depth for scanning entries + :return: the created tasks + :rtype: list of :class:`~subliminal.tasks.ListTask` - @property - def workers(self): - return self._workers - - @workers.setter - def workers(self, value): - if self.state == RUNNING: - raise BadStateError(self.state, IDLE) - self._workers = value - - @property - def languages(self): - """Getter for languages""" - return self._languages - - @languages.setter - def languages(self, languages): - """Setter for languages""" - logger.debug(u'Setting languages to %r' % languages) - self._languages = [] - for l in languages: - if l not in list_languages(1): - raise InvalidLanguageError(l) - if not l in self._languages: - self._languages.append(l) - - @property - def plugins(self): - """Getter for plugins""" - return self._plugins - - @plugins.setter - def plugins(self, plugins): - """Setter for plugins""" - logger.debug(u'Setting plugins to %r' % plugins) - self._plugins = [] - for p in plugins: - if p not in PLUGINS: - raise PluginError(p) - if not p in self._plugins: - self._plugins.append(p) - - def listSubtitles(self, entries, auto=False): - """ - Search subtitles within the plugins and return all found subtitles in a list of Subtitle object. - - Attributes: - entries -- filepath or folderpath of video file or a list of that - auto -- automaticaly manage workers (default to False)""" - if auto: - if self.state != IDLE: - raise BadStateError(self.state, IDLE) - self.startWorkers() - if isinstance(entries, basestring): - entries = [entries] - config = utils.PluginConfig(self.multi, self.cache_dir, self.filemode) - scan_result = [] - for e in entries: - if not isinstance(e, unicode): - logger.warning(u'Entry %r is not unicode' % e) - scan_result.extend(videos.scan(e)) - task_count = 0 - for video, subtitles in scan_result: - languages = set([s.language for s in subtitles if s.language]) - wanted_languages = set(self._languages) + """ + scan_result = [] + for p in paths: + scan_result.extend(scan(p, max_depth)) + logger.debug(u'Found %d videos in %r with maximum depth %d' % (len(scan_result), paths, max_depth)) + tasks = [] + config = ServiceConfig(multi, cache_dir) + for video, detected_subtitles in scan_result: + detected_languages = set([s.language for s in detected_subtitles]) + wanted_languages = languages.copy() + if not force and multi: + wanted_languages -= detected_languages if not wanted_languages: - wanted_languages = list_languages(1) - if not self.force and self.multi: - wanted_languages = set(wanted_languages) - languages - if not wanted_languages: - logger.debug(u'No need to list multi subtitles %r for %r because %r subtitles detected' % (self._languages, video.path, languages)) - continue - if not self.force and not self.multi and None in [s.language for s in subtitles]: - logger.debug(u'No need to list single subtitles %r for %r because one detected' % (self._languages, video.path)) + logger.debug(u'No need to list multi subtitles %r for %r because %r detected' % (languages, video, detected_languages)) continue - logger.debug(u'Listing subtitles %r for %r with %r' % (wanted_languages, video.path, self._plugins)) - for plugin_name in self._plugins: - plugin = getattr(plugins, plugin_name) - to_list_languages = wanted_languages & plugin.availableLanguages() - if not to_list_languages: - logger.debug(u'Skipping %r: none of wanted languages %r available in %r for plugin %s' % (video.path, wanted_languages, plugin.availableLanguages(), plugin_name)) - continue - if not plugin.isValidVideo(video): - logger.debug(u'Skipping %r: video %r is not part of supported videos %r for plugin %s' % (video.path, video, plugin.videos, plugin_name)) - continue - self.taskQueue.put((5, ListTask(video, to_list_languages, plugin_name, config))) - task_count += 1 - subtitles = [] - for _ in range(task_count): - subtitles.extend(self.listResultQueue.get()) - if auto: - self.stopWorkers() - return subtitles - - def downloadSubtitles(self, entries, auto=False): - """ - Download subtitles using the plugins preferences and languages. Also use internal algorithm to find - the best match inside a plugin. - - Attributes: - entries -- filepath or folderpath of video file or a list of that - auto -- automaticaly manage workers (default to False)""" - if auto: - if self.state != IDLE: - raise BadStateError(self.state, IDLE) - self.startWorkers() - by_video = self.groupByVideo(self.listSubtitles(entries, False)) - # Define an order with LANGUAGE_INDEX first for multi sorting - order = self.sort_order - if self.multi: - order.insert(0, LANGUAGE_INDEX) - task_count = 0 - for video, subtitles in by_video.iteritems(): - ordered_subtitles = sorted(subtitles, key=lambda s: self.keySubtitles(s, video, order), reverse=True) - if not self.multi: - self.taskQueue.put((5, DownloadTask(video, list(ordered_subtitles)))) - task_count += 1 + if not force and not multi and None in detected_languages: + logger.debug(u'No need to list single subtitles %r for %r because one detected' % (languages, video)) + continue + logger.debug(u'Listing subtitles %r for %r with services %r' % (wanted_languages, video, services)) + for service_name in services: + mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1) + service = mod.Service + service_languages = wanted_languages & service.available_languages() + if not service_languages: + logger.debug(u'Skipping %r: none of wanted languages %r available for service %s' % (video, wanted_languages, service_name)) continue - for _, by_language in groupby(ordered_subtitles, lambda s: s.language): - self.taskQueue.put((5, DownloadTask(video, list(by_language)))) - task_count += 1 - downloaded = [] - for _ in range(task_count): - downloaded.extend(self.downloadResultQueue.get()) - if auto: - self.stopWorkers() - return downloaded - - def keySubtitles(self, subtitle, video, order): - """Create a key to sort subtitle using preferences""" - key = '' - for sort_item in order: - if sort_item == LANGUAGE_INDEX: - key += '{0:03d}'.format(len(self._languages) - self._languages.index(subtitle.language) - 1) - elif sort_item == PLUGIN_INDEX: - key += '{0:02d}'.format(len(self._plugins) - self._plugins.index(subtitle.plugin) - 1) - elif sort_item == PLUGIN_CONFIDENCE: - key += '{0:04d}'.format(int(subtitle.confidence * 1000)) - elif sort_item == MATCHING_CONFIDENCE: - confidence = 0 - if subtitle.release: - confidence = matching_confidence(video, subtitle) - key += '{0:04d}'.format(int(confidence * 1000)) - return int(key) - - def groupByVideo(self, list_result): - '''Because list outputs a list of tuples from different plugins, we need to put them back - together under a single video key''' - result = defaultdict(list) - for video, subtitles in list_result: - result[video] += subtitles - return result - - def startWorkers(self): - """Create a pool of workers and start them""" - if self.state == RUNNING: - raise BadStateError(self.state, IDLE) - self.pool = [] - for _ in range(self._workers): - worker = PluginWorker(self.taskQueue, self.listResultQueue, self.downloadResultQueue) - worker.start() - self.pool.append(worker) - logger.debug(u'Worker %s added to the pool' % worker.name) - self.state = RUNNING - - def stopWorkers(self, priority=10): - """Stop workers using a lowest priority stop signal and wait for them to terminate properly""" - for _ in range(self._workers): - self.taskQueue.put((priority, StopTask())) - for worker in self.pool: - worker.join() - self.state = IDLE - if not self.taskQueue.empty(): - self.state = PAUSED - - def pauseWorkers(self): - """Pause workers using a highest priority stop signal and wait for them to terminate properly""" - self.stopWorkers(0) - - def addTask(self, task): - """Add a task with default priority""" - if not isinstance(task, Task) or isinstance(task, StopTask): - raise WrongTaskError() - self.taskQueue.put((5, task)) + if not service.is_valid_video(video): + logger.debug(u'Skipping %r: not part of supported videos %r for service %s' % (video, service.videos, service_name)) + continue + task = ListTask(video, service_languages, service_name, config) + logger.debug(u'Created task %r' % task) + tasks.append(task) + return tasks -class PluginWorker(threading.Thread): - """Threaded plugin worker""" - def __init__(self, taskQueue, listResultQueue, downloadResultQueue): - threading.Thread.__init__(self) - self.taskQueue = taskQueue - self.listResultQueue = listResultQueue - self.downloadResultQueue = downloadResultQueue - self.logger = logging.getLogger('subliminal.worker') - self.plugins = {} +def create_download_tasks(subtitles_by_video, multi): + """Create a list of :class:`~subliminal.tasks.DownloadTask` from a list results grouped by video - def run(self): - while True: - task = self.taskQueue.get()[1] - if isinstance(task, StopTask): - self.logger.debug(u'Poison pill received in thread %s' % self.name) - self.taskQueue.task_done() + :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results grouped by video and sorted + :type subtitles_by_video: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`] + :param order: preferred order for subtitles sorting + :type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE` + :param bool multi: download multiple languages for the same video + :return: the created tasks + :rtype: list of :class:`~subliminal.tasks.DownloadTask` + + """ + tasks = [] + for video, subtitles in subtitles_by_video.iteritems(): + if not subtitles: + continue + if not multi: + task = DownloadTask(video, list(subtitles)) + logger.debug(u'Created task %r' % task) + tasks.append(task) + continue + for _, by_language in groupby(subtitles, lambda s: s.language): + task = DownloadTask(video, list(by_language)) + logger.debug(u'Created task %r' % task) + tasks.append(task) + return tasks + + +def consume_task(task, services=None): + """Consume a task. If the ``services`` parameter is given, the function will attempt + to get the service from it. In case the service is not in ``services``, it will be initialized + and put in ``services`` + + :param task: task to consume + :type task: :class:`~subliminal.tasks.ListTask` or :class:`~subliminal.tasks.DownloadTask` + :param dict services: mapping between the service name and an instance of this service + :return: the result of the task + :rtype: list of :class:`~subliminal.subtitles.ResultSubtitle` or :class:`~subliminal.subtitles.Subtitle` + + """ + if services is None: + services = {} + logger.info(u'Consuming %r' % task) + result = None + if isinstance(task, ListTask): + if task.service not in services: + mod = __import__('services.' + task.service, globals=globals(), locals=locals(), fromlist=['Service'], level=-1) + services[task.service] = mod.Service(task.config) + services[task.service].init() + subtitles = services[task.service].list(task.video, task.languages) + result = subtitles + elif isinstance(task, DownloadTask): + for subtitle in task.subtitles: + if subtitle.service not in services: + mod = __import__('services.' + subtitle.service, globals=globals(), locals=locals(), fromlist=['Service'], level=-1) + services[subtitle.service] = mod.Service() + services[subtitle.service].init() + try: + services[subtitle.service].download(subtitle) + result = subtitle break - result = [] - try: - if isinstance(task, ListTask): - if task.plugin not in self.plugins: # init the plugin - self.plugins[task.plugin] = getattr(plugins, task.plugin)() - self.plugins[task.plugin].init() - # Retrieve the plugin list subtitles and return [(video, [subtitle])] - plugin = self.plugins[task.plugin] - plugin.config = task.config - subtitles = plugin.list(task.video, task.languages) - result = [(task.video, subtitles)] - elif isinstance(task, DownloadTask): - # Attempt to download one subtitle from the given list - for subtitle in task.subtitles: - if subtitle.plugin not in self.plugins: # init the plugin - self.plugins[subtitle.plugin] = getattr(plugins, subtitle.plugin)() - self.plugins[subtitle.plugin].init() - plugin = self.plugins[subtitle.plugin] - try: - result = [plugin.download(subtitle)] - break - except DownloadFailedError: # try the next one - self.logger.warning(u'Could not download subtitle %r, trying next' % subtitle) - continue - if not result: - self.logger.error(u'No subtitles could be downloaded for video %r' % task.video.path or task.video.release) - except: - self.logger.error(u'Exception raised in worker %s' % self.name, exc_info=True) - finally: - # Put the result in the correct queue - if isinstance(task, ListTask): - self.listResultQueue.put(result) - elif isinstance(task, DownloadTask): - self.downloadResultQueue.put(result) - self.taskQueue.task_done() - self.terminate() - self.logger.debug(u'Thread %s terminated' % self.name) - - def terminate(self): - """Terminate instanciated plugins""" - for plugin_name, plugin in self.plugins.iteritems(): - try: - plugin.terminate() - except: - self.logger.error(u'Exception raised when terminating plugin %s' % plugin_name, exc_info=True) + except DownloadFailedError: + logger.warning(u'Could not download subtitle %r, trying next' % subtitle) + continue + if result is None: + logger.error(u'No subtitles could be downloaded for video %r' % task.video) + return result def matching_confidence(video, subtitle): - '''Compute the confidence that the subtitle matches the video. - Returns a float between 0 and 1. 1 being the perfect match.''' + """Compute the probability (confidence) that the subtitle matches the video + + :param video: video to match + :type video: :class:`~subliminal.videos.Video` + :param subtitle: subtitle to match + :type subtitle: :class:`~subliminal.subtitles.Subtitle` + :return: the matching probability + :rtype: float + + """ guess = guessit.guess_file_info(subtitle.release, 'autodetect') - video_keywords = utils.get_keywords(video.guess) - subtitle_keywords = utils.get_keywords(guess) | subtitle.keywords + video_keywords = get_keywords(video.guess) + subtitle_keywords = get_keywords(guess) | subtitle.keywords replacement = {'keywords': len(video_keywords & subtitle_keywords)} - if isinstance(video, videos.Episode): + if isinstance(video, Episode): replacement.update({'series': 0, 'season': 0, 'episode': 0}) matching_format = '{series:b}{season:b}{episode:b}{keywords:03b}' best = matching_format.format(series=1, season=1, episode=1, keywords=len(video_keywords)) @@ -351,7 +178,7 @@ def matching_confidence(video, subtitle): replacement['season'] = 1 if 'episodeNumber' in guess and guess['episodeNumber'] == video.episode: replacement['episode'] = 1 - elif isinstance(video, videos.Movie): + elif isinstance(video, Movie): replacement.update({'title': 0, 'year': 0}) matching_format = '{title:b}{year:b}{keywords:03b}' best = matching_format.format(title=1, year=1, keywords=len(video_keywords)) @@ -364,3 +191,50 @@ def matching_confidence(video, subtitle): return 0 confidence = float(int(matching_format.format(**replacement), 2)) / float(int(best, 2)) return confidence + + +def key_subtitles(subtitle, video, languages, services, order): + """Create a key to sort subtitle using the given order + + :param subtitle: subtitle to sort + :type subtitle: :class:`~subliminal.subtitles.ResultSubtitle` + :param video: video to match + :type video: :class:`~subliminal.videos.Video` + :param list languages: languages in preferred order + :param list services: services in preferred order + :param order: preferred order for subtitles sorting + :type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE` + :return: a key ready to use for subtitles sorting + :rtype: int + + """ + key = '' + for sort_item in order: + if sort_item == LANGUAGE_INDEX: + key += '{0:03d}'.format(len(languages) - languages.index(subtitle.language) - 1) + elif sort_item == SERVICE_INDEX: + key += '{0:02d}'.format(len(services) - services.index(subtitle.service) - 1) + elif sort_item == SERVICE_CONFIDENCE: + key += '{0:04d}'.format(int(subtitle.confidence * 1000)) + elif sort_item == MATCHING_CONFIDENCE: + confidence = 0 + if subtitle.release: + confidence = matching_confidence(video, subtitle) + key += '{0:04d}'.format(int(confidence * 1000)) + return int(key) + + +def group_by_video(list_results): + """Group the results of :class:`ListTasks ` into a + dictionary of :class:`~subliminal.videos.Video` => :class:`~subliminal.subtitles.Subtitle` + + :param list_results: + :type list_results: list of result of :class:`~subliminal.tasks.ListTask` + :return: subtitles grouped by videos + :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`] + + """ + result = defaultdict(list) + for video, subtitles in list_results: + result[video] += subtitles + return result diff --git a/libs/subliminal/exceptions.py b/libs/subliminal/exceptions.py index 29b417f7..93001110 100755 --- a/libs/subliminal/exceptions.py +++ b/libs/subliminal/exceptions.py @@ -1,23 +1,20 @@ # -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin # -# Subliminal - Subtitles, faster than your thoughts -# Copyright (c) 2011 Antoine Bertin +# This file is part of subliminal. # -# This file is part of Subliminal. -# -# Subliminal is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# Subliminal is distributed in the hope that it will be useful, +# subliminal is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. -# -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . +# GNU Lesser General Public License for more details. # +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . class Error(Exception): @@ -25,21 +22,6 @@ class Error(Exception): pass -class BadStateError(Error): - """Exception raised when an invalid action is asked - - Attributes: - current -- current state of Subliminal instance - expected -- expected state of Subliminal instance - """ - def __init__(self, current, expected): - self.current = current - self.expected = expected - - def __str__(self): - return 'Expected state %d but current state is %d' % (self.expected, self.current) - - class InvalidLanguageError(Error): """Exception raised when invalid language is submitted @@ -66,21 +48,21 @@ class MissingLanguageError(Error): return self.language -class InvalidPluginError(Error): - """"Exception raised when invalid plugin is submitted +class InvalidServiceError(Error): + """Exception raised when invalid service is submitted + + :param string service: service that causes the error - Attributes: - plugin -- plugin that cause the error """ - def __init__(self, plugin): - self.plugin = plugin + def __init__(self, service): + self.service = service def __str__(self): - return self.plugin + return self.service -class PluginError(Error): - """"Exception raised by plugins""" +class ServiceError(Error): + """"Exception raised by services""" pass @@ -90,7 +72,7 @@ class WrongTaskError(Error): class DownloadFailedError(Error): - """"Exception raised when a download task has failed in plugin""" + """"Exception raised when a download task has failed in service""" pass diff --git a/libs/subliminal/infos.py b/libs/subliminal/infos.py index 32ea82f1..b28fda00 100755 --- a/libs/subliminal/infos.py +++ b/libs/subliminal/infos.py @@ -1,26 +1,18 @@ # -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin # -# Subliminal - Subtitles, faster than your thoughts -# Copyright (c) 2011 Antoine Bertin +# This file is part of subliminal. # -# This file is part of Subliminal. -# -# Subliminal is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# Subliminal is distributed in the hope that it will be useful, +# subliminal is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . -# -__title__ = 'subliminal' -__description__ = 'Subtitles, faster than your thoughts' -__version__ = '0.5' -__author__ = 'Antoine Bertin' -__license__ = 'LGPLv3' -__copyright__ = 'Copyright 2010-2011 Antoine Bertin' +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +__version__ = '0.5.1' diff --git a/libs/subliminal/languages.py b/libs/subliminal/languages.py index d38d48ba..f743953f 100755 --- a/libs/subliminal/languages.py +++ b/libs/subliminal/languages.py @@ -1,29 +1,34 @@ # -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin # -# Subliminal - Subtitles, faster than your thoughts -# Copyright (c) 2011 Antoine Bertin +# This file is part of subliminal. # -# This file is part of Subliminal. -# -# Subliminal is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# Subliminal is distributed in the hope that it will be useful, +# subliminal is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. -# -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . +# GNU Lesser General Public License for more details. # +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . __all__ = ['convert_language', 'list_languages', 'LANGUAGES'] def convert_language(language, to_iso, from_iso=None): - # if no from_iso is given, try to guess it - if from_iso == None: + """Convert a language into another format + + :param string language: language + :param int to_iso: convert language to ISO-639-x + :param int from_iso: convert language from ISO-639-x + :return: converted language + :rtype: string + + """ + if from_iso == None: # if no from_iso is given, try to guess it if language.startswith(language[:1].upper()): from_iso = 0 elif len(language) == 2: @@ -43,10 +48,17 @@ def convert_language(language, to_iso, from_iso=None): def list_languages(iso): + """List languages in the given ISO-639-x format + + :param int iso: ISO-639-x format to list + :return: languages in the requested format + :rtype: list + + """ return [l[iso] for l in LANGUAGES if l[iso]] -# ISO-639-2 languages list from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt -# + ('Brazilian', 'po', 'pob') +#: ISO-639-2 languages list from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt +#: + ('Brazilian', 'po', 'pob') LANGUAGES = [('Afar', 'aa', 'aar'), ('Abkhazian', 'ab', 'abk'), ('Achinese', '', 'ace'), diff --git a/libs/subliminal/plugins.py b/libs/subliminal/plugins.py deleted file mode 100755 index 1b2dc04d..00000000 --- a/libs/subliminal/plugins.py +++ /dev/null @@ -1,890 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Subliminal - Subtitles, faster than your thoughts -# Copyright (c) 2011 Antoine Bertin -# -# This file is part of Subliminal. -# -# Subliminal is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# Subliminal is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. -# -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . -# -__all__ = ['PluginBase', 'OpenSubtitles', 'BierDopje', 'TheSubDB', 'SubsWiki', 'Subtitulos'] - - -from exceptions import DownloadFailedError, MissingLanguageError, PluginError -from utils import get_keywords, PluginConfig, split_keyword -from videos import Episode, Movie, UnknownVideo -from subtitles import ResultSubtitle, get_subtitle_path -import BeautifulSoup -import abc -import gzip -import logging -import os -import re -import requests -import suds.client -import threading -import unicodedata -import urllib -import xmlrpclib -try: - import cPickle as pickle -except ImportError: - import pickle - - -#TODO: use ISO-639-2 in plugins instead of ISO-639-1 -class PluginBase(object): - __metaclass__ = abc.ABCMeta - site_url = '' - site_name = '' - server_url = '' - user_agent = 'Subliminal v0.5' - api_based = False - timeout = 5 - lock = threading.Lock() - languages = {} - reverted_languages = False - videos = [] - require_video = False - shared_support = False - - @abc.abstractmethod - def __init__(self, config=None): - self.config = config or PluginConfig() - self.logger = logging.getLogger('subliminal.%s' % self.__class__.__name__) - - @abc.abstractmethod - def init(self): - """Initiate connection""" - self.session = requests.session(timeout=10, headers={'User-Agent': self.user_agent}) - - @abc.abstractmethod - def terminate(self): - """Terminate connection""" - - @abc.abstractmethod - def query(self, *args): - """Make the actual query""" - - @abc.abstractmethod - def list(self, video, languages): - """List subtitles""" - - @abc.abstractmethod - def download(self, subtitle): - """Download a subtitle""" - - @classmethod - def availableLanguages(cls): - if not cls.reverted_languages: - return set(cls.languages.keys()) - if cls.reverted_languages: - return set(cls.languages.values()) - - @classmethod - def isValidVideo(cls, video): - if cls.require_video and not video.exists: - return False - if not isinstance(video, tuple(cls.videos)): - return False - return True - - @classmethod - def isValidLanguage(cls, language): - if language in cls.availableLanguages(): - return True - return False - - @classmethod - def getRevertLanguage(cls, language): - """ISO-639-1 language code from plugin language code""" - if not cls.reverted_languages and language in cls.languages.values(): - return [k for k, v in cls.languages.iteritems() if v == language][0] - if cls.reverted_languages and language in cls.languages.keys(): - return cls.languages[language] - raise MissingLanguageError(language) - - @classmethod - def getLanguage(cls, language): - """Plugin language code from ISO-639-1 language code""" - if not cls.reverted_languages and language in cls.languages.keys(): - return cls.languages[language] - if cls.reverted_languages and language in cls.languages.values(): - return [k for k, v in cls.languages.iteritems() if v == language][0] - raise MissingLanguageError(language) - - def adjustPermissions(self, filepath): - if self.config.filemode != None: - os.chmod(filepath, self.config.filemode) - - def downloadFile(self, url, filepath): - """Download a subtitle file""" - self.logger.info(u'Downloading %s' % url) - try: - r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent}) - with open(filepath, 'wb') as f: - f.write(r.content) - except Exception as e: - self.logger.error(u'Download %s failed: %s' % (url, e)) - if os.path.exists(filepath): - os.remove(filepath) - raise DownloadFailedError(str(e)) - self.logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath))) - - -class OpenSubtitles(PluginBase): - site_url = 'http://www.opensubtitles.org' - site_name = 'OpenSubtitles' - server_url = 'http://api.opensubtitles.org/xml-rpc' - user_agent = 'Subliminal v0.5' - api_based = True - languages = {'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'alb', 'am': 'amh', 'ar': 'ara', - 'an': 'arg', 'hy': 'arm', 'as': 'asm', 'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze', - 'ba': 'bak', 'bm': 'bam', 'eu': 'baq', 'be': 'bel', 'bn': 'ben', 'bh': 'bih', 'bi': 'bis', - 'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'bur', 'ca': 'cat', 'ch': 'cha', 'ce': 'che', - 'zh': 'chi', 'cu': 'chu', 'cv': 'chv', 'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'cze', - 'da': 'dan', 'dv': 'div', 'nl': 'dut', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo', 'et': 'est', - 'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin', 'fr': 'fre', 'fy': 'fry', 'ff': 'ful', - 'ka': 'geo', 'de': 'ger', 'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell', - 'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb', 'hz': 'her', 'hi': 'hin', - 'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun', 'ig': 'ibo', 'is': 'ice', 'io': 'ido', 'ii': 'iii', - 'iu': 'iku', 'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita', 'jv': 'jav', - 'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas', 'kr': 'kau', 'kk': 'kaz', 'km': 'khm', - 'ki': 'kik', 'rw': 'kin', 'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua', - 'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim', 'ln': 'lin', 'lt': 'lit', - 'lb': 'ltz', 'lu': 'lub', 'lg': 'lug', 'mk': 'mac', 'mh': 'mah', 'ml': 'mal', 'mi': 'mao', - 'mr': 'mar', 'ms': 'may', 'mg': 'mlg', 'mt': 'mlt', 'mo': 'mol', 'mn': 'mon', 'na': 'nau', - 'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep', 'nn': 'nno', 'nb': 'nob', - 'no': 'nor', 'ny': 'nya', 'oc': 'oci', 'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss', - 'pa': 'pan', 'fa': 'per', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus', 'qu': 'que', - 'rm': 'roh', 'rn': 'run', 'ru': 'rus', 'sg': 'sag', 'sa': 'san', 'sr': 'scc', 'si': 'sin', - 'sk': 'slo', 'sl': 'slv', 'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som', - 'st': 'sot', 'es': 'spa', 'sc': 'srd', 'ss': 'ssw', 'su': 'sun', 'sw': 'swa', 'sv': 'swe', - 'ty': 'tah', 'ta': 'tam', 'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha', - 'bo': 'tib', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso', 'tk': 'tuk', 'tr': 'tur', - 'tw': 'twi', 'ug': 'uig', 'uk': 'ukr', 'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie', - 'vo': 'vol', 'cy': 'wel', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid', 'yo': 'yor', - 'za': 'zha', 'zu': 'zul', 'ro': 'rum', 'po': 'pob', 'un': 'unk', 'ay': 'ass'} - reverted_languages = False - videos = [Episode, Movie] - require_video = False - confidence_order = ['moviehash', 'imdbid', 'fulltext'] - - def __init__(self, config=None): - super(OpenSubtitles, self).__init__(config) - self.server = xmlrpclib.ServerProxy(self.server_url) - self.token = None - - def __enter__(self): - self.init() - return self - - def __exit__(self, *args): - self.terminate() - - def init(self): - self.logger.debug(u'Initializing') - super(OpenSubtitles, self).init() - result = self.server.LogIn('', '', 'eng', self.user_agent) - if result['status'] != '200 OK': - raise PluginError('Login failed') - self.token = result['token'] - - def terminate(self): - self.logger.debug(u'Terminating') - if self.token: - self.server.LogOut(self.token) - - def query(self, filepath, languages, moviehash=None, size=None, imdbid=None, query=None): - searches = [] - if moviehash and size: - searches.append({'moviehash': moviehash, 'moviebytesize': size}) - if imdbid: - searches.append({'imdbid': imdbid}) - if query: - searches.append({'query': query}) - if not searches: - raise PluginError('One or more parameter missing') - for search in searches: - search['sublanguageid'] = ','.join([self.getLanguage(l) for l in languages]) - self.logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token)) - results = self.server.SearchSubtitles(self.token, searches) - if not results['data']: - self.logger.debug(u'Could not find subtitles for %r with token %s' % (searches, self.token)) - return [] - subtitles = [] - for result in results['data']: - language = self.getRevertLanguage(result['SubLanguageID']) - path = get_subtitle_path(filepath, language, self.config.multi) - confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order)) - subtitle = ResultSubtitle(path, language, self.__class__.__name__, result['SubDownloadLink'], result['SubFileName'], confidence) - subtitles.append(subtitle) - return subtitles - - def list(self, video, languages): - languages = languages & self.availableLanguages() - if not languages: - self.logger.debug(u'No language available') - return [] - if not self.isValidVideo(video): - self.logger.debug(u'Not a valid video') - return [] - results = [] - if video.exists: - results = self.query(video.path or video.release, languages, moviehash=video.hashes['OpenSubtitles'], size=str(video.size)) - elif video.imdbid: - results = self.query(video.path or video.release, languages, imdbid=video.imdbid) - elif isinstance(video, Episode): - results = self.query(video.path or video.release, languages, query=video.series) - elif isinstance(video, Movie): - results = self.query(video.path or video.release, languages, query=video.title) - return results - - def download(self, subtitle): - #TODO: Use OpenSubtitles DownloadSubtitles method - try: - self.downloadFile(subtitle.link, subtitle.path + '.gz') - with open(subtitle.path, 'wb') as dump: - gz = gzip.open(subtitle.path + '.gz') - dump.write(gz.read()) - gz.close() - self.adjustPermissions(subtitle.path) - except Exception as e: - if os.path.exists(subtitle.path): - os.remove(subtitle.path) - raise DownloadFailedError(str(e)) - finally: - if os.path.exists(subtitle.path + '.gz'): - os.remove(subtitle.path + '.gz') - return subtitle - - -class BierDopje(PluginBase): - site_url = 'http://bierdopje.com' - site_name = 'BierDopje' - server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/' - api_based = True - languages = {'en': 'en', 'nl': 'nl'} - reverted_languages = False - videos = [Episode] - require_video = False - - def __init__(self, config=None): - super(BierDopje, self).__init__(config) - self.showids = {} - if self.config and self.config.cache_dir: - self.initCache() - - def __enter__(self): - self.init() - return self - - def __exit__(self, *args): - self.terminate() - - def init(self): - self.logger.debug(u'Initializing') - super(BierDopje, self).init() - - def terminate(self): - self.logger.debug(u'Terminating') - - def initCache(self): - self.logger.debug(u'Initializing cache...') - if not self.config or not self.config.cache_dir: - raise PluginError('Cache directory is required') - self.showids_cache = os.path.join(self.config.cache_dir, 'bierdopje_showids.cache') - if not os.path.exists(self.showids_cache): - self.saveToCache() - - def saveToCache(self): - self.logger.debug(u'Saving showids to cache...') - with self.lock: - with open(self.showids_cache, 'w') as f: - pickle.dump(self.showids, f) - - def loadFromCache(self): - self.logger.debug(u'Loading showids from cache...') - with self.lock: - with open(self.showids_cache, 'r') as f: - self.showids = pickle.load(f) - - def query(self, season, episode, languages, filepath, tvdbid=None, series=None): - self.initCache() - self.loadFromCache() - if series: - if series.lower() in self.showids: # from cache - request_id = self.showids[series.lower()] - self.logger.debug(u'Retreived showid %d for %s from cache' % (request_id, series)) - else: # query to get showid - self.logger.debug(u'Getting showid from show name %s...' % series) - r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower()))) - if r.status_code != 200: - self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) - return [] - soup = BeautifulSoup.BeautifulStoneSoup(r.content) - if soup.status.contents[0] == 'false': - self.logger.debug(u'Could not find show %s' % series) - return [] - request_id = int(soup.showid.contents[0]) - self.showids[series.lower()] = request_id - self.saveToCache() - request_source = 'showid' - request_is_tvdbid = 'false' - elif tvdbid: - request_id = tvdbid - request_source = 'tvdbid' - request_is_tvdbid = 'true' - else: - raise PluginError('One or more parameter missing') - subtitles = [] - for language in languages: - self.logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language)) - r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language, request_is_tvdbid)) - if r.status_code != 200: - self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) - return [] - soup = BeautifulSoup.BeautifulStoneSoup(r.content) - if soup.status.contents[0] == 'false': - self.logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language)) - continue - path = get_subtitle_path(filepath, language, self.config.multi) - for result in soup.results('result'): - subtitle = ResultSubtitle(path, language, self.__class__.__name__, result.downloadlink.contents[0], result.filename.contents[0]) - subtitles.append(subtitle) - return subtitles - - def list(self, video, languages): - languages = languages & self.availableLanguages() - if not languages: - self.logger.debug(u'No language available') - return [] - if not self.isValidVideo(video): - self.logger.debug(u'Not a valid video') - return [] - results = self.query(video.season, video.episode, languages, video.path or video.release, video.tvdbid, video.series) - return results - - def download(self, subtitle): - self.downloadFile(subtitle.link, subtitle.path) - return subtitle - - -class TheSubDB(PluginBase): - site_url = 'http://thesubdb.com' - site_name = 'SubDB' - server_url = 'http://api.thesubdb.com/' # for testing purpose, use http://sandbox.thesubdb.com/ instead - api_based = True - user_agent = 'SubDB/1.0 (Subliminal/0.5; https://github.com/Diaoul/subliminal)' # defined by the API - languages = {'af': 'af', 'cs': 'cs', 'da': 'da', 'de': 'de', 'en': 'en', 'es': 'es', 'fi': 'fi', - 'fr': 'fr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'la': 'la', 'nl': 'nl', 'no': 'no', - 'oc': 'oc', 'pl': 'pl', 'pt': 'pt', 'ro': 'ro', 'ru': 'ru', 'sl': 'sl', 'sr': 'sr', - 'sv': 'sv', 'tr': 'tr'} # list available with the API at http://sandbox.thesubdb.com/?action=languages - reverted_languages = False - videos = [Movie, Episode, UnknownVideo] - require_video = True - - def __init__(self, config=None): - super(TheSubDB, self).__init__(config) - - def __enter__(self): - self.init() - return self - - def __exit__(self, *args): - self.terminate() - - def init(self): - self.logger.debug(u'Initializing') - super(TheSubDB, self).init() - - def terminate(self): - self.logger.debug(u'Terminating') - - def list(self, video, languages): - languages = languages & self.availableLanguages() - if not languages: - self.logger.debug(u'No language available') - return [] - if not self.isValidVideo(video): - self.logger.debug(u'Not a valid video') - return [] - results = self.query(video.path, video.hashes['TheSubDB'], languages) - return results - - def query(self, filepath, moviehash, languages): - r = self.session.get(self.server_url, params={'action': 'search', 'hash': moviehash}) - if r.status_code == 404: - self.logger.debug(u'Could not find subtitles for hash %s' % moviehash) - return [] - if r.status_code != 200: - self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) - return [] - available_languages = set([self.getRevertLanguage(l) for l in r.content.split(',')]) - filtered_languages = languages & available_languages - if not filtered_languages: - self.logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages)) - return [] - subtitles = [] - for language in filtered_languages: - path = get_subtitle_path(filepath, language, self.config.multi) - subtitle = ResultSubtitle(path, language, self.__class__.__name__, '%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, self.getLanguage(language))) - subtitles.append(subtitle) - return subtitles - - def download(self, subtitle): - self.downloadFile(subtitle.link, subtitle.path) - return subtitle - - -class SubsWiki(PluginBase): - site_url = 'http://www.subswiki.com' - site_name = 'SubsWiki' - server_url = 'http://www.subswiki.com' - api_based = False - languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po', - u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es', - u'Italian': 'it', u'Català': 'ca'} - reverted_languages = True - videos = [Episode, Movie] - require_video = False - release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs') - - def __init__(self, config=None): - super(SubsWiki, self).__init__(config) - - def __enter__(self): - self.init() - return self - - def __exit__(self, *args): - self.terminate() - - def init(self): - self.logger.debug(u'Initializing') - super(SubsWiki, self).init() - - def terminate(self): - self.logger.debug(u'Terminating') - - def list(self, video, languages): - languages = languages & self.availableLanguages() - if not languages: - self.logger.debug(u'No language available') - return [] - if not self.isValidVideo(video): - self.logger.debug(u'Not a valid video') - return [] - results = [] - if isinstance(video, Episode): - results = self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode) - elif isinstance(video, Movie) and video.year: - results = self.query(video.path or video.release, languages, get_keywords(video.guess), movie=video.title, year=video.year) - return results - - def query(self, filepath, languages, keywords=None, series=None, season=None, episode=None, movie=None, year=None): - if series and season and episode: - request_series = series.lower().replace(' ', '_') - if isinstance(request_series, unicode): - request_series = request_series.encode('utf-8') - self.logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) - r = self.session.get('%s/serie/%s/%s/%s/' % (self.server_url, urllib.quote(request_series), season, episode)) - if r.status_code == 404: - self.logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) - return [] - elif movie and year: - request_movie = movie.title().replace(' ', '_') - if isinstance(request_movie, unicode): - request_movie = request_movie.encode('utf-8') - self.logger.debug(u'Getting subtitles for %s (%d) with languages %r' % (movie, year, languages)) - r = self.session.get('%s/film/%s_(%d)' % (self.server_url, urllib.quote(request_movie), year)) - if r.status_code == 404: - self.logger.debug(u'Could not find subtitles for %s (%d) with languages %r' % (movie, year, languages)) - return [] - else: - raise PluginError('One or more parameter missing') - if r.status_code != 200: - self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) - return [] - soup = BeautifulSoup.BeautifulSoup(r.content) - subtitles = [] - for sub in soup('td', {'class': 'NewsTitle'}): - sub_keywords = split_keyword(self.release_pattern.search(sub.contents[1]).group(1).lower()) - if not keywords & sub_keywords: - self.logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords)) - continue - for html_language in sub.parent.parent.findAll('td', {'class': 'language'}): - language = self.getRevertLanguage(html_language.string.strip()) - if not language in languages: - self.logger.debug(u'Language %r not in wanted languages %r' % (language, languages)) - continue - html_status = html_language.findNextSibling('td') - status = html_status.find('strong').string.strip() - if status != 'Completed': - self.logger.debug(u'Wrong subtitle status %s' % status) - continue - path = get_subtitle_path(filepath, language, self.config.multi) - subtitle = ResultSubtitle(path, language, self.__class__.__name__, '%s%s' % (self.server_url, html_status.findNext('td').find('a')['href'])) - subtitles.append(subtitle) - return subtitles - - def download(self, subtitle): - self.downloadFile(subtitle.link, subtitle.path) - return subtitle - - -class Subtitulos(PluginBase): - site_url = 'http://www.subtitulos.es/' - site_name = 'Subtitulos' - server_url = 'http://www.subtitulos.es' - api_based = False - languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po', - u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es', - u'Italian': 'it', u'Català': 'ca'} - reverted_languages = True - videos = [Episode] - require_video = False - release_pattern = re.compile('Versión (.+) ([0-9]+).([0-9])+ megabytes') - - def __init__(self, config=None): - super(Subtitulos, self).__init__(config) - - def __enter__(self): - self.init() - return self - - def __exit__(self, *args): - self.terminate() - - def init(self): - self.logger.debug(u'Initializing') - super(Subtitulos, self).init() - - def terminate(self): - self.logger.debug(u'Terminating') - - def list(self, video, languages): - languages = languages & self.availableLanguages() - if not languages: - self.logger.debug(u'No language available') - return [] - if not self.isValidVideo(video): - self.logger.debug(u'Not a valid video') - return [] - results = self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode) - return results - - def query(self, filepath, languages, keywords, series, season, episode): - request_series = series.lower().replace(' ', '_') - if isinstance(request_series, unicode): - request_series = unicodedata.normalize('NFKD', request_series).encode('ascii', 'ignore') - self.logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) - r = self.session.get('%s/%s/%sx%.2d' % (self.server_url, urllib.quote(request_series), season, episode)) - if r.status_code == 404: - self.logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) - return [] - if r.status_code != 200: - self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) - return [] - soup = BeautifulSoup.BeautifulSoup(r.content) - subtitles = [] - for sub in soup('div', {'id': 'version'}): - sub_keywords = split_keyword(self.release_pattern.search(sub.find('p', {'class': 'title-sub'}).contents[1]).group(1).lower()) - if not keywords & sub_keywords: - self.logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords)) - continue - for html_language in sub.findAllNext('ul', {'class': 'sslist'}): - language = self.getRevertLanguage(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip()) - if not language in languages: - self.logger.debug(u'Language %r not in wanted languages %r' % (language, languages)) - continue - html_status = html_language.findNext('li', {'class': 'li-estado green'}) - status = html_status.contents[0].string.strip() - if status != 'Completado': - self.logger.debug(u'Wrong subtitle status %s' % status) - continue - path = get_subtitle_path(filepath, language, self.config.multi) - subtitle = ResultSubtitle(path, language, self.__class__.__name__, html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords) - subtitles.append(subtitle) - return subtitles - - def download(self, subtitle): - self.downloadFile(subtitle.link, subtitle.path) - return subtitle - - -class GetSubtitle(PluginBase): - site_url = 'http://www.subtitles.com.br/' - site_name = 'GetSubtitle' - server_url = 'http://api.getsubtitle.com/server.php?wsdl' - api_based = True - languages = {'sq': 'ALB', 'ar': 'ARA', 'hy': 'ARM', 'bs': 'BOS', 'bg': 'BUL', 'ca': 'CAT', 'zh': 'CHI', 'hr': 'HRV', - 'cs': 'CZE', 'da': 'DAN', 'nl': 'NLD', 'en': 'ENG', 'eo': 'ESP', 'et': 'EST', 'fi': 'FIN', 'fr': 'FRA', - 'gl': 'GLG', 'ka': 'GEO', 'de': 'DEU', 'el': 'GRC', 'he': 'ISR', 'hi': 'HIN', 'hu': 'HUN', 'is': 'ISL', - 'id': 'IND', 'it': 'ITA', 'ja': 'JPN', 'kk': 'KAZ', 'ko': 'KOR', 'lv': 'LVA', 'lt': 'LIT', 'lb': 'LTZ', - 'mk': 'MKD', 'ms': 'MAY', 'no': 'NOR', 'oc': 'OCC', 'pl': 'POL', 'pt': 'POR', 'ro': 'RUM', 'ru': 'RUS', - 'sr': 'ZAF', 'sk': 'SLK', 'sl': 'SLV', 'es': 'SPA', 'sv': 'SWE', 'th': 'THA', 'tr': 'TUR', 'uk': 'UKR', - 'ur': 'URD', 'vi': 'VTN'} - reverted_languages = False - videos = [Movie] - require_video = False - max_results = 100 - - def __init__(self, config=None): - super(GetSubtitle, self).__init__(config) - - def __enter__(self): - self.init() - return self - - def __exit__(self, *args): - pass - - def init(self): - self.logger.debug(u'Initializing') - self.server = suds.client.Client(self.server_url) - - def terminate(self): - self.logger.debug(u'Terminating') - - def query(self, *args): - #TODO - pass - - def list(self, video, languages): - languages = languages & self.availableLanguages() - if not languages: - self.logger.debug(u'No language available') - return [] - if not self.isValidVideo(video): - self.logger.debug(u'Not a valid video') - return [] - #TODO - - def download(self, subtitle): - #TODO - pass - - -''' -class Addic7ed(PluginBase.PluginBase): - site_url = 'http://www.addic7ed.com' - site_name = 'Addic7ed' - server_url = 'http://www.addic7ed.com' - api_based = False - _plugin_languages = {u'English': 'en', - u'English (US)': 'en', - u'English (UK)': 'en', - u'Italian': 'it', - u'Portuguese': 'pt', - u'Portuguese (Brazilian)': 'po', - u'Romanian': 'ro', - u'Español (Latinoamérica)': 'es', - u'Español (España)': 'es', - u'Spanish (Latin America)': 'es', - u'Español': 'es', - u'Spanish': 'es', - u'Spanish (Spain)': 'es', - u'French': 'fr', - u'Greek': 'el', - u'Arabic': 'ar', - u'German': 'de', - u'Croatian': 'hr', - u'Indonesian': 'id', - u'Hebrew': 'he', - u'Russian': 'ru', - u'Turkish': 'tr', - u'Swedish': 'se', - u'Czech': 'cs', - u'Dutch': 'nl', - u'Hungarian': 'hu', - u'Norwegian': 'no', - u'Polish': 'pl', - u'Persian': 'fa'} - - def __init__(self, config_dict=None): - super(Addic7ed, self).__init__(self._plugin_languages, config_dict, isRevert=True) - #http://www.addic7ed.com/serie/Smallville/9/11/Absolute_Justice - self.release_pattern = re.compile(' \nVersion (.+), ([0-9]+).([0-9])+ MBs') - - def list(self, filepath, languages): - if not self.checkLanguages(languages): - return [] - guess = guessit.guess_file_info(filepath, 'autodetect') - if guess['type'] != 'episode': - self.logger.debug(u'Not an episode') - return [] - # add multiple things to the release group set - release_group = set() - if 'releaseGroup' in guess: - release_group.add(guess['releaseGroup'].lower()) - else: - if 'title' in guess: - release_group.add(guess['title'].lower()) - if 'screenSize' in guess: - release_group.add(guess['screenSize'].lower()) - if 'series' not in guess or len(release_group) == 0: - self.logger.debug(u'Not enough information to proceed') - return [] - self.release_group = release_group # used to sort results - return self.query(guess['series'], guess['season'], guess['episodeNumber'], release_group, filepath, languages) - - def query(self, name, season, episode, release_group, filepath, languages=None): - searchname = name.lower().replace(' ', '_') - if isinstance(searchname, unicode): - searchname = searchname.encode('utf-8') - searchurl = '%s/serie/%s/%s/%s/%s' % (self.server_url, urllib2.quote(searchname), season, episode, urllib2.quote(searchname)) - self.logger.debug(u'Searching in %s' % searchurl) - try: - req = urllib2.Request(searchurl, headers={'User-Agent': self.user_agent}) - page = urllib2.urlopen(req, timeout=self.timeout) - except urllib2.HTTPError as inst: - self.logger.info(u'Error: %s - %s' % (searchurl, inst)) - return [] - except urllib2.URLError as inst: - self.logger.info(u'TimeOut: %s' % inst) - return [] - soup = BeautifulSoup(page.read()) - sublinks = [] - for html_sub in soup('td', {'class': 'NewsTitle', 'colspan': '3'}): - if not self.release_pattern.match(str(html_sub.contents[1])): # On not needed soup td result - continue - sub_teams = self.listTeams([self.release_pattern.match(str(html_sub.contents[1])).groups()[0].lower()], ['.', '_', ' ', '/', '-']) - if not release_group.intersection(sub_teams): # On wrong team - continue - html_language = html_sub.findNext('td', {'class': 'language'}) - sub_language = self.getRevertLanguage(html_language.contents[0].strip().replace(' ', '')) - if languages and not sub_language in languages: # On wrong language - continue - html_status = html_language.findNextSibling('td') - sub_status = html_status.find('b').string.strip() - if not sub_status == 'Completed': # On not completed subtitles - continue - sub_link = self.server_url + html_status.findNextSibling('td', {'colspan': '3'}).find('a')['href'] - self.logger.debug(u'Found a match with teams: %s' % sub_teams) - result = Subtitle(filepath, self.getSubtitlePath(filepath, sub_language), self.__class__.__name__, sub_language, sub_link, keywords=sub_teams) - sublinks.append(result) - sublinks.sort(self._cmpReleaseGroup) - return sublinks - - def download(self, subtitle): - self.downloadFile(subtitle.link, subtitle.path) - return subtitle - - -class Podnapisi(PluginBase.PluginBase): - site_url = "http://www.podnapisi.net" - site_name = "Podnapisi" - server_url = 'http://ssp.podnapisi.net:8000' - api_based = True - _plugin_languages = {"sl": "1", - "en": "2", - "no": "3", - "ko": "4", - "de": "5", - "is": "6", - "cs": "7", - "fr": "8", - "it": "9", - "bs": "10", - "ja": "11", - "ar": "12", - "ro": "13", - "es-ar": "14", - "hu": "15", - "el": "16", - "zh": "17", - "lt": "19", - "et": "20", - "lv": "21", - "he": "22", - "nl": "23", - "da": "24", - "se": "25", - "pl": "26", - "ru": "27", - "es": "28", - "sq": "29", - "tr": "30", - "fi": "31", - "pt": "32", - "bg": "33", - "mk": "35", - "sk": "37", - "hr": "38", - "zh": "40", - "hi": "42", - "th": "44", - "uk": "46", - "sr": "47", - "po": "48", - "ga": "49", - "be": "50", - "vi": "51", - "fa": "52", - "ca": "53", - "id": "54"} - - def __init__(self, config_dict=None): - super(Podnapisi, self).__init__(self._plugin_languages, config_dict) - # Podnapisi uses two reference for latin serbian and cyrillic serbian (36 and 47) - # add the 36 manually as cyrillic seems to be more used - self.revertPluginLanguages["36"] = "sr" - - def list(self, filenames, languages): - """Main method to call when you want to list subtitles""" - filepath = filenames[0] - if not os.path.isfile(filepath): - return [] - return self.query(self.hashFile(filepath), languages) - - def download(self, subtitle): - return [] - - def query(self, moviehash, languages=None): - """Makes a query on podnapisi and returns info (link, lang) about found subtitles""" - # login - self.server = xmlrpclib.ServerProxy(self.server_url) - try: - log_result = self.server.initiate(self.user_agent) - self.logger.debug(u"Result: %s" % log_result) - token = log_result["session"] - nonce = log_result["nonce"] - except Exception: - self.logger.error(u"Cannot login" % log_result) - return [] - username = 'getmesubs' - password = '99D31$$' - hash = md5() - hash.update(password) - password = hash.hexdigest() - hash = sha256() - hash.update(password) - hash.update(nonce) - password = hash.hexdigest() - self.server.authenticate(token, username, password) - self.logger.debug(u'Authenticated') - #if languages: - # self.logger.debug([self.getLanguage(l) for l in languages]) - # self.server.setFilters(token, [self.getLanguage(l) for l in languages]) - # self.logger.debug('Filers set for languages %s' % languages) - self.logger.debug(u"Starting search with token %s and hashs %s" % (token, [moviehash])) - results = self.server.search(token, [moviehash]) - return results - subs = [] - for sub in results['results']: - subs.append(sub) - self.server.terminate(token) - return subs -''' diff --git a/libs/subliminal/services/__init__.py b/libs/subliminal/services/__init__.py new file mode 100755 index 00000000..67e457fe --- /dev/null +++ b/libs/subliminal/services/__init__.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin +# +# This file is part of subliminal. +# +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from ..exceptions import MissingLanguageError, DownloadFailedError +import logging +import os +import requests +import threading + + +__all__ = ['ServiceBase', 'ServiceConfig'] +logger = logging.getLogger(__name__) + + +class ServiceBase(object): + """Service base class + + :param config: service configuration + :type config: :class:`ServiceConfig` + + """ + #: URL to the service server + server_url = '' + + #: User Agent for any HTTP-based requests + user_agent = 'subliminal v0.5' + + #: Whether based on an API or not + api_based = False + + #: Timeout for web requests + timeout = 5 + + #: Lock for cache interactions + lock = threading.Lock() + + #: Mapping to Service's language codes and subliminal's + languages = {} + + #: Whether the mapping is reverted or not + reverted_languages = False + + #: Accepted video classes (:class:`~subliminal.videos.Episode`, :class:`~subliminal.videos.Movie`, :class:`~subliminal.videos.UnknownVideo`) + videos = [] + + #: Whether the video has to exist or not + require_video = False + + def __init__(self, config=None): + self.config = config or ServiceConfig() + + def __enter__(self): + self.init() + return self + + def __exit__(self, *args): + self.terminate() + + def init(self): + """Initialize connection""" + logger.debug(u'Initializing %s' % self.__class__.__name__) + self.session = requests.session(timeout=10, headers={'User-Agent': self.user_agent}) + + def terminate(self): + """Terminate connection""" + logger.debug(u'Terminating %s' % self.__class__.__name__) + + def query(self, *args): + """Make the actual query""" + pass + + def list(self, video, languages): + """List subtitles""" + pass + + def download(self, subtitle): + """Download a subtitle""" + self.download_file(subtitle.link, subtitle.path) + + @classmethod + def available_languages(cls): + """Available languages in the Service + + :return: available languages + :rtype: set + + """ + if not cls.reverted_languages: + return set(cls.languages.keys()) + return set(cls.languages.values()) + + @classmethod + def check_validity(cls, video, languages): + """Check for video and languages validity in the Service + + :param video: the video to check + :type video: :class:`~subliminal.videos.video` + :param set languages: languages to check + :rtype: bool + + """ + languages &= cls.available_languages() + if not languages: + logger.debug(u'No language available for service %s' % cls.__class__.__name__.lower()) + return False + if not cls.is_valid_video(video): + logger.debug(u'%r is not valid for service %s' % (video, cls.__class__.__name__.lower())) + return False + return True + + @classmethod + def is_valid_video(cls, video): + """Check if video is valid in the Service + + :param video: the video to check + :type video: :class:`~subliminal.videos.Video` + :rtype: bool + + """ + if cls.require_video and not video.exists: + return False + if not isinstance(video, tuple(cls.videos)): + return False + return True + + @classmethod + def is_valid_language(cls, language): + """Check if language is valid in the Service + + :param string language: the language to check + :rtype: bool + + """ + if language in cls.available_languages(): + return True + return False + + @classmethod + def get_revert_language(cls, language): + """ISO-639-1 language code from service language code + + :param string language: service language code + :return: ISO-639-1 language code + :rtype: string + + """ + if not cls.reverted_languages and language in cls.languages.values(): + return [k for k, v in cls.languages.iteritems() if v == language][0] + if cls.reverted_languages and language in cls.languages.keys(): + return cls.languages[language] + raise MissingLanguageError(language) + + @classmethod + def get_language(cls, language): + """Service language code from ISO-639-1 language code + + :param string language: ISO-639-1 language code + :return: service language code + :rtype: string + + """ + if not cls.reverted_languages and language in cls.languages.keys(): + return cls.languages[language] + if cls.reverted_languages and language in cls.languages.values(): + return [k for k, v in cls.languages.iteritems() if v == language][0] + raise MissingLanguageError(language) + + def download_file(self, url, filepath): + """Attempt to download a file and remove it in case of failure + + :param string url: URL to download + :param string filepath: destination path + + """ + logger.info(u'Downloading %s' % url) + try: + r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent}) + with open(filepath, 'wb') as f: + f.write(r.content) + except Exception as e: + logger.error(u'Download %s failed: %s' % (url, e)) + if os.path.exists(filepath): + os.remove(filepath) + raise DownloadFailedError(str(e)) + logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath))) + + +class ServiceConfig(object): + """Configuration for any :class:`Service` + + :param bool multi: whether to download one subtitle per language or not + :param string cache_dir: cache directory + + """ + def __init__(self, multi=False, cache_dir=None): + self.multi = multi + self.cache_dir = cache_dir + + def __repr__(self): + return 'ServiceConfig(%r, %s)' % (self.multi, self.cache_dir) diff --git a/libs/subliminal/services/bierdopje.py b/libs/subliminal/services/bierdopje.py new file mode 100755 index 00000000..15401ada --- /dev/null +++ b/libs/subliminal/services/bierdopje.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin +# +# This file is part of subliminal. +# +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from . import ServiceBase +from ..exceptions import ServiceError +from ..subtitles import get_subtitle_path, ResultSubtitle +from ..videos import Episode +from ..utils import to_unicode +import BeautifulSoup +import logging +import os.path +import urllib +try: + import cPickle as pickle +except ImportError: + import pickle + + +logger = logging.getLogger(__name__) + + +class BierDopje(ServiceBase): + server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/' + api_based = True + languages = {'en': 'en', 'nl': 'nl'} + reverted_languages = False + videos = [Episode] + require_video = False + + def __init__(self, config=None): + super(BierDopje, self).__init__(config) + self.showids = {} + if self.config and self.config.cache_dir: + self.init_cache() + + def init_cache(self): + logger.debug(u'Initializing cache...') + if not self.config or not self.config.cache_dir: + raise ServiceError('Cache directory is required') + self.showids_cache = os.path.join(self.config.cache_dir, 'bierdopje_showids.cache') + if not os.path.exists(self.showids_cache): + self.save_cache() + + def save_cache(self): + logger.debug(u'Saving showids to cache...') + with self.lock: + with open(self.showids_cache, 'w') as f: + pickle.dump(self.showids, f) + + def load_cache(self): + logger.debug(u'Loading showids from cache...') + with self.lock: + with open(self.showids_cache, 'r') as f: + self.showids = pickle.load(f) + + def query(self, season, episode, languages, filepath, tvdbid=None, series=None): + self.load_cache() + if series: + if series.lower() in self.showids: # from cache + request_id = self.showids[series.lower()] + logger.debug(u'Retreived showid %d for %s from cache' % (request_id, series)) + else: # query to get showid + logger.debug(u'Getting showid from show name %s...' % series) + r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower()))) + if r.status_code != 200: + logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) + return [] + soup = BeautifulSoup.BeautifulStoneSoup(r.content) + if soup.status.contents[0] == 'false': + logger.debug(u'Could not find show %s' % series) + return [] + request_id = int(soup.showid.contents[0]) + self.showids[series.lower()] = request_id + self.save_cache() + request_source = 'showid' + request_is_tvdbid = 'false' + elif tvdbid: + request_id = tvdbid + request_source = 'tvdbid' + request_is_tvdbid = 'true' + else: + raise ServiceError('One or more parameter missing') + subtitles = [] + for language in languages: + logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language)) + r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language, request_is_tvdbid)) + if r.status_code != 200: + logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) + return [] + soup = BeautifulSoup.BeautifulStoneSoup(r.content) + if soup.status.contents[0] == 'false': + logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language)) + continue + path = get_subtitle_path(filepath, language, self.config.multi) + for result in soup.results('result'): + subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result.downloadlink.contents[0], + release=to_unicode(result.filename.contents[0])) + subtitles.append(subtitle) + return subtitles + + def list(self, video, languages): + if not self.check_validity(video, languages): + return [] + results = self.query(video.season, video.episode, languages, video.path or video.release, video.tvdbid, video.series) + return results + + +Service = BierDopje diff --git a/libs/subliminal/services/opensubtitles.py b/libs/subliminal/services/opensubtitles.py new file mode 100755 index 00000000..9dee27b9 --- /dev/null +++ b/libs/subliminal/services/opensubtitles.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin +# +# This file is part of subliminal. +# +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from . import ServiceBase +from ..exceptions import ServiceError, DownloadFailedError +from ..subtitles import get_subtitle_path, ResultSubtitle +from ..videos import Episode, Movie +from ..utils import to_unicode +import gzip +import logging +import os.path +import xmlrpclib + + +logger = logging.getLogger(__name__) + + +class OpenSubtitles(ServiceBase): + server_url = 'http://api.opensubtitles.org/xml-rpc' + api_based = True + languages = {'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'alb', 'am': 'amh', 'ar': 'ara', + 'an': 'arg', 'hy': 'arm', 'as': 'asm', 'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze', + 'ba': 'bak', 'bm': 'bam', 'eu': 'baq', 'be': 'bel', 'bn': 'ben', 'bh': 'bih', 'bi': 'bis', + 'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'bur', 'ca': 'cat', 'ch': 'cha', 'ce': 'che', + 'zh': 'chi', 'cu': 'chu', 'cv': 'chv', 'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'cze', + 'da': 'dan', 'dv': 'div', 'nl': 'dut', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo', 'et': 'est', + 'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin', 'fr': 'fre', 'fy': 'fry', 'ff': 'ful', + 'ka': 'geo', 'de': 'ger', 'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell', + 'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb', 'hz': 'her', 'hi': 'hin', + 'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun', 'ig': 'ibo', 'is': 'ice', 'io': 'ido', 'ii': 'iii', + 'iu': 'iku', 'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita', 'jv': 'jav', + 'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas', 'kr': 'kau', 'kk': 'kaz', 'km': 'khm', + 'ki': 'kik', 'rw': 'kin', 'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua', + 'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim', 'ln': 'lin', 'lt': 'lit', + 'lb': 'ltz', 'lu': 'lub', 'lg': 'lug', 'mk': 'mac', 'mh': 'mah', 'ml': 'mal', 'mi': 'mao', + 'mr': 'mar', 'ms': 'may', 'mg': 'mlg', 'mt': 'mlt', 'mo': 'mol', 'mn': 'mon', 'na': 'nau', + 'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep', 'nn': 'nno', 'nb': 'nob', + 'no': 'nor', 'ny': 'nya', 'oc': 'oci', 'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss', + 'pa': 'pan', 'fa': 'per', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus', 'qu': 'que', + 'rm': 'roh', 'rn': 'run', 'ru': 'rus', 'sg': 'sag', 'sa': 'san', 'sr': 'scc', 'si': 'sin', + 'sk': 'slo', 'sl': 'slv', 'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som', + 'st': 'sot', 'es': 'spa', 'sc': 'srd', 'ss': 'ssw', 'su': 'sun', 'sw': 'swa', 'sv': 'swe', + 'ty': 'tah', 'ta': 'tam', 'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha', + 'bo': 'tib', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso', 'tk': 'tuk', 'tr': 'tur', + 'tw': 'twi', 'ug': 'uig', 'uk': 'ukr', 'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie', + 'vo': 'vol', 'cy': 'wel', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid', 'yo': 'yor', + 'za': 'zha', 'zu': 'zul', 'ro': 'rum', 'po': 'pob', 'un': 'unk', 'ay': 'ass'} + reverted_languages = False + videos = [Episode, Movie] + require_video = False + confidence_order = ['moviehash', 'imdbid', 'fulltext'] + + def __init__(self, config=None): + super(OpenSubtitles, self).__init__(config) + self.server = xmlrpclib.ServerProxy(self.server_url) + self.token = None + + def init(self): + super(OpenSubtitles, self).init() + result = self.server.LogIn('', '', 'eng', self.user_agent) + if result['status'] != '200 OK': + raise ServiceError('Login failed') + self.token = result['token'] + + def terminate(self): + super(OpenSubtitles, self).terminate() + if self.token: + self.server.LogOut(self.token) + + def query(self, filepath, languages, moviehash=None, size=None, imdbid=None, query=None): + searches = [] + if moviehash and size: + searches.append({'moviehash': moviehash, 'moviebytesize': size}) + if imdbid: + searches.append({'imdbid': imdbid}) + if query: + searches.append({'query': query}) + if not searches: + raise ServiceError('One or more parameter missing') + for search in searches: + search['sublanguageid'] = ','.join([self.get_language(l) for l in languages]) + logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token)) + results = self.server.SearchSubtitles(self.token, searches) + if not results['data']: + logger.debug(u'Could not find subtitles for %r with token %s' % (searches, self.token)) + return [] + subtitles = [] + for result in results['data']: + language = self.get_revert_language(result['SubLanguageID']) + path = get_subtitle_path(filepath, language, self.config.multi) + confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order)) + subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['SubDownloadLink'], + release=to_unicode(result['SubFileName']), confidence=confidence) + subtitles.append(subtitle) + return subtitles + + def list(self, video, languages): + if not self.check_validity(video, languages): + return [] + results = [] + if video.exists: + results = self.query(video.path or video.release, languages, moviehash=video.hashes['OpenSubtitles'], size=str(video.size)) + elif video.imdbid: + results = self.query(video.path or video.release, languages, imdbid=video.imdbid) + elif isinstance(video, Episode): + results = self.query(video.path or video.release, languages, query=video.series) + elif isinstance(video, Movie): + results = self.query(video.path or video.release, languages, query=video.title) + return results + + def download(self, subtitle): + #TODO: Use OpenSubtitles DownloadSubtitles method + try: + self.download_file(subtitle.link, subtitle.path + '.gz') + with open(subtitle.path, 'wb') as dump: + gz = gzip.open(subtitle.path + '.gz') + dump.write(gz.read()) + gz.close() + except Exception as e: + if os.path.exists(subtitle.path): + os.remove(subtitle.path) + raise DownloadFailedError(str(e)) + finally: + if os.path.exists(subtitle.path + '.gz'): + os.remove(subtitle.path + '.gz') + return subtitle + + +Service = OpenSubtitles diff --git a/libs/subliminal/services/subswiki.py b/libs/subliminal/services/subswiki.py new file mode 100755 index 00000000..c5670c1c --- /dev/null +++ b/libs/subliminal/services/subswiki.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin +# +# This file is part of subliminal. +# +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from . import ServiceBase +from ..exceptions import ServiceError +from ..subtitles import get_subtitle_path, ResultSubtitle +from ..videos import Episode, Movie +from subliminal.utils import get_keywords, split_keyword +import BeautifulSoup +import logging +import re +import urllib + + +logger = logging.getLogger(__name__) + + +class SubsWiki(ServiceBase): + server_url = 'http://www.subswiki.com' + api_based = False + languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po', + u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es', + u'Italian': 'it', u'Català': 'ca'} + reverted_languages = True + videos = [Episode, Movie] + require_video = False + release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs') + + def list(self, video, languages): + if not self.check_validity(video, languages): + return [] + results = [] + if isinstance(video, Episode): + results = self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode) + elif isinstance(video, Movie) and video.year: + results = self.query(video.path or video.release, languages, get_keywords(video.guess), movie=video.title, year=video.year) + return results + + def query(self, filepath, languages, keywords=None, series=None, season=None, episode=None, movie=None, year=None): + if series and season and episode: + request_series = series.lower().replace(' ', '_') + if isinstance(request_series, unicode): + request_series = request_series.encode('utf-8') + logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) + r = self.session.get('%s/serie/%s/%s/%s/' % (self.server_url, urllib.quote(request_series), season, episode)) + if r.status_code == 404: + logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) + return [] + elif movie and year: + request_movie = movie.title().replace(' ', '_') + if isinstance(request_movie, unicode): + request_movie = request_movie.encode('utf-8') + logger.debug(u'Getting subtitles for %s (%d) with languages %r' % (movie, year, languages)) + r = self.session.get('%s/film/%s_(%d)' % (self.server_url, urllib.quote(request_movie), year)) + if r.status_code == 404: + logger.debug(u'Could not find subtitles for %s (%d) with languages %r' % (movie, year, languages)) + return [] + else: + raise ServiceError('One or more parameter missing') + if r.status_code != 200: + logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) + return [] + soup = BeautifulSoup.BeautifulSoup(r.content) + subtitles = [] + for sub in soup('td', {'class': 'NewsTitle'}): + sub_keywords = split_keyword(self.release_pattern.search(sub.contents[1]).group(1).lower()) + if not keywords & sub_keywords: + logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords)) + continue + for html_language in sub.parent.parent.findAll('td', {'class': 'language'}): + language = self.get_revert_language(html_language.string.strip()) + if not language in languages: + logger.debug(u'Language %r not in wanted languages %r' % (language, languages)) + continue + html_status = html_language.findNextSibling('td') + status = html_status.find('strong').string.strip() + if status != 'Completed': + logger.debug(u'Wrong subtitle status %s' % status) + continue + path = get_subtitle_path(filepath, language, self.config.multi) + subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s%s' % (self.server_url, html_status.findNext('td').find('a')['href'])) + subtitles.append(subtitle) + return subtitles + +Service = SubsWiki diff --git a/libs/subliminal/services/subtitulos.py b/libs/subliminal/services/subtitulos.py new file mode 100755 index 00000000..44888e70 --- /dev/null +++ b/libs/subliminal/services/subtitulos.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin +# +# This file is part of subliminal. +# +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from . import ServiceBase +from ..subtitles import get_subtitle_path, ResultSubtitle +from ..videos import Episode +from subliminal.utils import get_keywords, split_keyword +import BeautifulSoup +import logging +import re +import unicodedata +import urllib + + +logger = logging.getLogger(__name__) + + +class Subtitulos(ServiceBase): + server_url = 'http://www.subtitulos.es' + api_based = False + languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po', + u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es', + u'Italian': 'it', u'Català': 'ca'} + reverted_languages = True + videos = [Episode] + require_video = False + release_pattern = re.compile('Versión (.+) ([0-9]+).([0-9])+ megabytes') + + def list(self, video, languages): + if not self.check_validity(video, languages): + return [] + results = self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode) + return results + + def query(self, filepath, languages, keywords, series, season, episode): + request_series = series.lower().replace(' ', '_') + if isinstance(request_series, unicode): + request_series = unicodedata.normalize('NFKD', request_series).encode('ascii', 'ignore') + logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) + r = self.session.get('%s/%s/%sx%.2d' % (self.server_url, urllib.quote(request_series), season, episode)) + if r.status_code == 404: + logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages)) + return [] + if r.status_code != 200: + logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) + return [] + soup = BeautifulSoup.BeautifulSoup(r.content) + subtitles = [] + for sub in soup('div', {'id': 'version'}): + sub_keywords = split_keyword(self.release_pattern.search(sub.find('p', {'class': 'title-sub'}).contents[1]).group(1).lower()) + if not keywords & sub_keywords: + logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords)) + continue + for html_language in sub.findAllNext('ul', {'class': 'sslist'}): + language = self.get_revert_language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip()) + if not language in languages: + logger.debug(u'Language %r not in wanted languages %r' % (language, languages)) + continue + html_status = html_language.findNext('li', {'class': 'li-estado green'}) + status = html_status.contents[0].string.strip() + if status != 'Completado': + logger.debug(u'Wrong subtitle status %s' % status) + continue + path = get_subtitle_path(filepath, language, self.config.multi) + subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords) + subtitles.append(subtitle) + return subtitles + +Service = Subtitulos diff --git a/libs/subliminal/services/thesubdb.py b/libs/subliminal/services/thesubdb.py new file mode 100755 index 00000000..cccddd40 --- /dev/null +++ b/libs/subliminal/services/thesubdb.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin +# +# This file is part of subliminal. +# +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from . import ServiceBase +from ..subtitles import get_subtitle_path, ResultSubtitle +from ..videos import Episode, Movie, UnknownVideo +import logging + + +logger = logging.getLogger(__name__) + + +class TheSubDB(ServiceBase): + server_url = 'http://api.thesubdb.com/' # for testing purpose, use http://sandbox.thesubdb.com/ instead + user_agent = 'SubDB/1.0 (subliminal/0.5; https://github.com/Diaoul/subliminal)' # defined by the API + api_based = True + languages = {'af': 'af', 'cs': 'cs', 'da': 'da', 'de': 'de', 'en': 'en', 'es': 'es', 'fi': 'fi', + 'fr': 'fr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'la': 'la', 'nl': 'nl', 'no': 'no', + 'oc': 'oc', 'pl': 'pl', 'pt': 'pt', 'ro': 'ro', 'ru': 'ru', 'sl': 'sl', 'sr': 'sr', + 'sv': 'sv', 'tr': 'tr'} # list available with the API at http://sandbox.thesubdb.com/?action=languages + reverted_languages = False + videos = [Movie, Episode, UnknownVideo] + require_video = True + + def list(self, video, languages): + if not self.check_validity(video, languages): + return [] + results = self.query(video.path, video.hashes['TheSubDB'], languages) + return results + + def query(self, filepath, moviehash, languages): + r = self.session.get(self.server_url, params={'action': 'search', 'hash': moviehash}) + if r.status_code == 404: + logger.debug(u'Could not find subtitles for hash %s' % moviehash) + return [] + if r.status_code != 200: + logger.error(u'Request %s returned status code %d' % (r.url, r.status_code)) + return [] + available_languages = set([self.get_revert_language(l) for l in r.content.split(',')]) + languages &= available_languages + if not languages: + logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages)) + return [] + subtitles = [] + for language in languages: + path = get_subtitle_path(filepath, language, self.config.multi) + subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, self.get_language(language))) + subtitles.append(subtitle) + return subtitles + +Service = TheSubDB diff --git a/libs/subliminal/subtitles.py b/libs/subliminal/subtitles.py index 479d25c8..355046dc 100755 --- a/libs/subliminal/subtitles.py +++ b/libs/subliminal/subtitles.py @@ -1,50 +1,73 @@ # -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin # -# Subliminal - Subtitles, faster than your thoughts -# Copyright (c) 2011 Antoine Bertin +# This file is part of subliminal. # -# This file is part of Subliminal. -# -# Subliminal is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# Subliminal is distributed in the hope that it will be useful, +# subliminal is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . -# -__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path'] - - -from languages import list_languages, convert_language -import abc +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from .languages import list_languages, convert_language import os.path +__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path'] + + +#: Subtitles extensions EXTENSIONS = ['.srt', '.sub', '.txt'] class Subtitle(object): - __metaclass__ = abc.ABCMeta - """Base class for subtitles""" + """Base class for subtitles + :param string path: path to the subtitle + :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`) + + """ def __init__(self, path, language): self.path = path self.language = language @property def exists(self): + """Whether the subtitle exists or not""" if self.path: return os.path.exists(self.path) return False + +class EmbeddedSubtitle(Subtitle): + """Subtitle embedded in a container + + :param string path: path to the subtitle + :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`) + :param int track_id: id of the subtitle track in the container + + """ + def __init__(self, path, language, track_id): + super(EmbeddedSubtitle, self).__init__(path, language) + self.track_id = track_id + @classmethod - def fromPath(cls, path): + def from_enzyme(cls, path, subtitle): + language = convert_language(subtitle.language, 1, 2) + return cls(path, language, subtitle.trackno) + + +class ExternalSubtitle(Subtitle): + """Subtitle in a file next to the video file""" + @classmethod + def from_path(cls, path): + """Create an :class:`ExternalSubtitle` from path""" extension = '' for e in EXTENSIONS: if path.endswith(e): @@ -58,25 +81,21 @@ class Subtitle(object): return cls(path, language) -class EmbeddedSubtitle(Subtitle): - def __init__(self, path, language, track_id): - super(EmbeddedSubtitle, self).__init__(path, language) - self.track_id = track_id - - @classmethod - def fromEnzyme(cls, path, subtitle): - language = convert_language(subtitle.language, 1, 2) - return cls(path, language, subtitle.trackno) - - -class ExternalSubtitle(Subtitle): - pass - - class ResultSubtitle(ExternalSubtitle): - def __init__(self, path, language, plugin, link, release=None, confidence=1, keywords=set()): + """Subtitle found using :mod:`~subliminal.services` + + :param string path: path to the subtitle + :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`) + :param string service: name of the service + :param string link: download link for the subtitle + :param string release: release name of the video + :param float confidence: confidence that the subtitle matches the video according to the service + :param set keywords: keywords that describe the subtitle + + """ + def __init__(self, path, language, service, link, release=None, confidence=1, keywords=set()): super(ResultSubtitle, self).__init__(path, language) - self.plugin = plugin + self.service = service self.link = link self.release = release self.confidence = confidence @@ -84,19 +103,20 @@ class ResultSubtitle(ExternalSubtitle): @property def single(self): + """Whether this is a single subtitle or not. A single subtitle does not have + a language indicator in its file name + + :rtype: bool + + """ extension = os.path.splitext(self.path)[0] language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:] if not language in list_languages(1): return True return False - def convert(self): - converted = {'path': self.path, 'plugin': self.plugin, 'language': self.language, 'link': self.link, 'release': self.release, - 'confidence': self.confidence, 'keywords': self.keywords} - return converted - - def __str__(self): - return repr(self.convert()) + def __repr__(self): + return 'ResultSubtitle(%s, %s, %.2f, %s)' % (self.language, self.service, self.confidence, self.release) def get_subtitle_path(video_path, language, multi): diff --git a/libs/subliminal/tasks.py b/libs/subliminal/tasks.py index 111a08d3..448a64f1 100755 --- a/libs/subliminal/tasks.py +++ b/libs/subliminal/tasks.py @@ -1,23 +1,20 @@ # -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin # -# Subliminal - Subtitles, faster than your thoughts -# Copyright (c) 2011 Antoine Bertin +# This file is part of subliminal. # -# This file is part of Subliminal. -# -# Subliminal is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# Subliminal is distributed in the hope that it will be useful, +# subliminal is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. -# -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . +# GNU Lesser General Public License for more details. # +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . __all__ = ['Task', 'ListTask', 'DownloadTask', 'StopTask'] @@ -27,21 +24,43 @@ class Task(object): class ListTask(Task): - """List task to list subtitles""" - def __init__(self, video, languages, plugin, config): + """List task used by the worker to search for subtitles + + :param video: video to search subtitles for + :type video: :class:`~subliminal.videos.Video` + :param list languages: languages to search for + :param string service: name of the service to use + :param config: configuration for the service + :type config: :class:`~subliminal.services.ServiceConfig` + + """ + def __init__(self, video, languages, service, config): self.video = video - self.plugin = plugin + self.service = service self.languages = languages self.config = config + def __repr__(self): + return 'ListTask(%r, %r, %s, %r)' % (self.video, self.languages, self.service, self.config) + class DownloadTask(Task): - """Download task to download subtitles""" + """Download task used by the worker to download subtitles + + :param video: video to download subtitles for + :type video: :class:`~subliminal.videos.Video` + :param subtitles: subtitles to download in order of preference + :type subtitles: list of :class:`~subliminal.subtitles.Subtitle` + + """ def __init__(self, video, subtitles): self.video = video self.subtitles = subtitles + def __repr__(self): + return 'DownloadTask(%r, %r)' % (self.video, self.subtitles) + class StopTask(Task): - """Stop task to stop workers""" + """Stop task that will stop the worker""" pass diff --git a/libs/subliminal/utils.py b/libs/subliminal/utils.py index 27dddb16..00cda322 100755 --- a/libs/subliminal/utils.py +++ b/libs/subliminal/utils.py @@ -1,44 +1,35 @@ # -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin # -# Subliminal - Subtitles, faster than your thoughts -# Copyright (c) 2011 Antoine Bertin +# This file is part of subliminal. # -# This file is part of Subliminal. -# -# Subliminal is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# Subliminal is distributed in the hope that it will be useful, +# subliminal is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . -# -__all__ = ['PluginConfig', 'get_keywords', 'split_keyword', 'NullHandler'] - - -import logging +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . import re -try: - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass -class PluginConfig(object): - def __init__(self, multi=None, cache_dir=None, filemode=None): - self.multi = multi - self.cache_dir = cache_dir - self.filemode = filemode +__all__ = ['get_keywords', 'split_keyword', 'to_unicode'] def get_keywords(guess): + """Retrieve keywords from guessed informations + + :param guess: guessed informations + :type guess: :class:`guessit.guess.Guess` + :return: lower case alphanumeric keywords + :rtype: set + + """ keywords = set() for k in ['releaseGroup', 'screenSize', 'videoCodec', 'format']: if k in guess: @@ -47,5 +38,27 @@ def get_keywords(guess): def split_keyword(keyword): + """Split a keyword in multiple ones on any non-alphanumeric character + + :param string keyword: keyword + :return: keywords + :rtype: set + + """ split = set(re.findall(r'\w+', keyword)) return split + + +def to_unicode(data): + """Convert a basestring to unicode + + :param basestring data: data to decode + :return: data as unicode + :rtype: unicode + + """ + if not isinstance(data, basestring): + raise ValueError('Basestring expected') + if isinstance(data, unicode): + return data + return unicode(data, 'utf-8', 'replace') diff --git a/libs/subliminal/videos.py b/libs/subliminal/videos.py index cb5a1520..79276b14 100755 --- a/libs/subliminal/videos.py +++ b/libs/subliminal/videos.py @@ -1,57 +1,73 @@ # -*- coding: utf-8 -*- +# Copyright 2011-2012 Antoine Bertin # -# Subliminal - Subtitles, faster than your thoughts -# Copyright (c) 2011 Antoine Bertin +# This file is part of subliminal. # -# This file is part of Subliminal. -# -# Subliminal is free software; you can redistribute it and/or modify it under -# the terms of the Lesser GNU General Public License as published by +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by # the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# Subliminal is distributed in the hope that it will be useful, +# subliminal is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Lesser GNU General Public License for more details. +# GNU Lesser General Public License for more details. # -# You should have received a copy of the Lesser GNU General Public License -# along with this program. If not, see . -# -__all__ = ['EXTENSIONS', 'MIMETYPES', 'Video', 'Episode', 'Movie', 'UnknownVideo', 'scan'] - - -from languages import list_languages -import abc +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from . import subtitles +from .languages import list_languages import enzyme import guessit import hashlib +import logging import mimetypes import os import struct -import subprocess -import subtitles -EXTENSIONS = ['.avi', '.mkv', '.mpg', '.mp4', '.m4v', '.mov', '.ogm', '.ogv', '.wmv', '.divx', '.asf'] -MIMETYPES = ['video/mpeg', 'video/mp4', 'video/quicktime', 'video/x-ms-wmv', 'video/x-msvideo', 'video/x-flv', 'video/x-matroska', 'video/x-matroska-3d'] +__all__ = ['EXTENSIONS', 'MIMETYPES', 'Video', 'Episode', 'Movie', 'UnknownVideo', + 'scan', 'hash_opensubtitles', 'hash_thesubdb'] +logger = logging.getLogger(__name__) + +#: Video extensions +EXTENSIONS = ['.avi', '.mkv', '.mpg', '.mp4', '.m4v', '.mov', '.ogm', '.ogv', '.wmv', + '.divx', '.asf'] + +#: Video mimetypes +MIMETYPES = ['video/mpeg', 'video/mp4', 'video/quicktime', 'video/x-ms-wmv', 'video/x-msvideo', + 'video/x-flv', 'video/x-matroska', 'video/x-matroska-3d'] class Video(object): - __metaclass__ = abc.ABCMeta - """Base class for videos""" - def __init__(self, release, guess, imdbid=None): - self.release = release + """Base class for videos + + :param string path: path + :param guess: guessed informations + :type guess: :class:`~guessit.guess.Guess` + :param string imdbid: imdbid + + """ + def __init__(self, path, guess, imdbid=None): + self.release = path self.guess = guess self.imdbid = imdbid self._path = None self.hashes = {} - if os.path.exists(release): - self.path = release + if os.path.exists(path): + self._path = path + self.size = os.path.getsize(self._path) + self._compute_hashes() @classmethod - def fromPath(cls, path): - """Create a Video object guessing all informations from the given release/path""" + def from_path(cls, path): + """Create a :class:`Video` subclass guessing all informations from the given path + + :param string path: path + :return: video object + :rtype: :class:`Episode` or :class:`Movie` or :class:`UnknownVideo` + + """ guess = guessit.guess_file_info(path, 'autodetect') result = None if guess['type'] == 'episode' and 'series' in guess and 'season' in guess and 'episodeNumber' in guess: @@ -72,12 +88,14 @@ class Video(object): @property def exists(self): + """Whether the video exists or not""" if self._path: return os.path.exists(self._path) return False @property def path(self): + """Path to the video""" return self._path @path.setter @@ -86,21 +104,137 @@ class Video(object): raise ValueError('Path does not exists') self._path = value self.size = os.path.getsize(self._path) - self._computeHashes() + self._compute_hashes() - def _computeHashes(self): - self.hashes['OpenSubtitles'] = self._computeHashOpenSubtitles() - self.hashes['TheSubDB'] = self._computeHashTheSubDB() + def _compute_hashes(self): + """Compute different hashes""" + self.hashes['OpenSubtitles'] = hash_opensubtitles(self.path) + self.hashes['TheSubDB'] = hash_thesubdb(self.path) - def _computeHashOpenSubtitles(self): - """Hash a file like OpenSubtitles""" - longlongformat = 'q' # long long - bytesize = struct.calcsize(longlongformat) - f = open(self.path, 'rb') - filesize = os.path.getsize(self.path) + def scan(self): + """Scan and return associated subtitles + + :return: associated subtitles + :rtype: list of :class:`~subliminal.subtitles.Subtitle` + + """ + if not self.exists: + return [] + basepath = os.path.splitext(self.path)[0] + results = [] + video_infos = None + try: + video_infos = enzyme.parse(self.path) + logger.debug(u'Succeeded parsing %s with enzyme: %r' % (self.path, video_infos)) + except: + logger.debug(u'Failed parsing %s with enzyme' % self.path) + if isinstance(video_infos, enzyme.core.AVContainer): + results.extend([subtitles.EmbeddedSubtitle.from_enzyme(self.path, s) for s in video_infos.subtitles]) + for l in list_languages(1): + for e in subtitles.EXTENSIONS: + single_path = basepath + '%s' % e + if os.path.exists(single_path): + results.append(subtitles.ExternalSubtitle(single_path, None)) + multi_path = basepath + '.%s%s' % (l, e) + if os.path.exists(multi_path): + results.append(subtitles.ExternalSubtitle(multi_path, l)) + return results + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.release) + + +class Episode(Video): + """Episode :class:`Video` + + :param string path: path + :param string series: series + :param int season: season number + :param int episode: episode number + :param string title: title + :param guess: guessed informations + :type guess: :class:`~guessit.guess.Guess` + :param string tvdbid: tvdbid + :param string imdbid: imdbid + + """ + def __init__(self, path, series, season, episode, title=None, guess=None, tvdbid=None, imdbid=None): + super(Episode, self).__init__(path, guess, imdbid) + self.series = series + self.title = title + self.season = season + self.episode = episode + self.tvdbid = tvdbid + + +class Movie(Video): + """Movie :class:`Video` + + :param string path: path + :param string title: title + :param int year: year + :param guess: guessed informations + :type guess: :class:`~guessit.guess.Guess` + :param string imdbid: imdbid + + """ + def __init__(self, path, title, year=None, guess=None, imdbid=None): + super(Movie, self).__init__(path, guess, imdbid) + self.title = title + self.year = year + + +class UnknownVideo(Video): + """Unknown video""" + pass + + +def scan(entry, max_depth=3, depth=0): + """Scan a path for videos and subtitles + + :param string entry: path + :param int max_depth: maximum folder depth + :param int depth: starting depth + :return: found videos and subtitles + :rtype: list of (:class:`Video`, [:class:`~subliminal.subtitles.Subtitle`]) + + """ + if depth > max_depth and max_depth != 0: # we do not want to search the whole file system except if max_depth = 0 + return [] + if depth == 0: + entry = os.path.abspath(entry) + if os.path.isdir(entry): # a dir? recurse + logger.debug(u'Scanning directory %s with depth %d/%d' % (entry, depth, max_depth)) + result = [] + for e in os.listdir(entry): + result.extend(scan(os.path.join(entry, e), max_depth, depth + 1)) + return result + if os.path.isfile(entry) or depth == 0: + logger.debug(u'Scanning file %s with depth %d/%d' % (entry, depth, max_depth)) + if depth != 0: # trust the user: only check for valid format if recursing + if mimetypes.guess_type(entry)[0] not in MIMETYPES and os.path.splitext(entry)[1] not in EXTENSIONS: + return [] + video = Video.from_path(entry) + return [(video, video.scan())] + logger.warning(u'Scanning entry %s failed with depth %d/%d' % (entry, depth, max_depth)) + return [] # anything else + + +def hash_opensubtitles(path): + """Compute a hash using OpenSubtitles' algorithm + + :param string path: path + :return: hash + :rtype: string + + """ + longlongformat = 'q' # long long + bytesize = struct.calcsize(longlongformat) + with open(path, 'rb') as f: + filesize = os.path.getsize(path) filehash = filesize if filesize < 65536 * 2: - return [] + return None for _ in range(65536 / bytesize): filebuffer = f.read(bytesize) (l_value,) = struct.unpack(longlongformat, filebuffer) @@ -112,103 +246,24 @@ class Video(object): (l_value,) = struct.unpack(longlongformat, filebuffer) filehash += l_value filehash = filehash & 0xFFFFFFFFFFFFFFFF - f.close() - returnedhash = '%016x' % filehash - return returnedhash - - def _computeHashTheSubDB(self): - """Hash a file like TheSubDB""" - readsize = 64 * 1024 - with open(self.path, 'rb') as f: - data = f.read(readsize) - f.seek(-readsize, os.SEEK_END) - data += f.read(readsize) - return hashlib.md5(data).hexdigest() - - def mkvmerge(self, subs, out=None, mkvmerge_bin='mkvmerge', title=None): - """Merge the video with subs""" - if not out: - out = self.path + '.merged.mkv' - args = [mkvmerge_bin, '-o', out, self.path] - if title: - args += ['--title', title] - for sub in subs: - if sub.language: - track_id = 0 - if isinstance(sub, subtitles.EmbeddedSubtitle): - track_id = sub.track_id - args += ['--language', str(track_id) + ':' + sub.language, sub.path] - continue - args += [sub.path] - with open(os.devnull, 'w') as devnull: - p = subprocess.Popen(args, stdout=devnull, stderr=devnull) - p.wait() - - def scan(self): - """Scan and return associated Subtitles""" - if not self.exists: - return [] - basepath = os.path.splitext(self.path)[0] - results = [] - video_infos = None - try: - video_infos = enzyme.parse(self.path) - except enzyme.ParseError: - pass - if isinstance(video_infos, enzyme.core.AVContainer): - results.extend([subtitles.EmbeddedSubtitle.fromEnzyme(self.path, s) for s in video_infos.subtitles]) - for l in list_languages(1): - for e in subtitles.EXTENSIONS: - single_path = basepath + '%s' % e - if os.path.exists(single_path): - results.append(subtitles.ExternalSubtitle(single_path, None)) - multi_path = basepath + '.%s%s' % (l, e) - if os.path.exists(multi_path): - results.append(subtitles.ExternalSubtitle(multi_path, l)) - return results + returnedhash = '%016x' % filehash + logger.debug(u'Computed OpenSubtitle hash %s for %s' % (returnedhash, path)) + return returnedhash -class Episode(Video): - """Episode class""" - def __init__(self, release, series, season, episode, title=None, guess=None, tvdbid=None, imdbid=None): - super(Episode, self).__init__(release, guess, imdbid) - self.series = series - self.title = title - self.season = season - self.episode = episode - self.tvdbid = tvdbid +def hash_thesubdb(path): + """Compute a hash using TheSubDB's algorithm + :param string path: path + :return: hash + :rtype: string -class Movie(Video): - """Movie class""" - def __init__(self, release, title, year=None, guess=None, imdbid=None): - super(Movie, self).__init__(release, guess, imdbid) - self.title = title - self.year = year - - -class UnknownVideo(Video): - """Unknown video""" - def __init__(self, release, guess, imdbid=None): - super(UnknownVideo, self).__init__(release, guess, imdbid) - self.guess = guess - - -def scan(entry, max_depth=3, depth=0): - """Scan a path and return a list of tuples (video, [subtitle])""" - if depth > max_depth and max_depth != 0: # we do not want to search the whole file system except if max_depth = 0 - return [] - if depth == 0: - entry = os.path.abspath(entry) - if os.path.isfile(entry): # a file? scan it - if depth != 0: # trust the user: only check for valid format if recursing - if mimetypes.guess_type(entry)[0] not in MIMETYPES and os.path.splitext(entry)[1] not in EXTENSIONS: - return [] - video = Video.fromPath(entry) - return [(video, video.scan())] - if os.path.isdir(entry): # a dir? recurse - result = [] - for e in os.listdir(entry): - result.extend(scan(os.path.join(entry, e), max_depth, depth + 1)) - return result - return [] # anything else + """ + readsize = 64 * 1024 + with open(path, 'rb') as f: + data = f.read(readsize) + f.seek(-readsize, os.SEEK_END) + data += f.read(readsize) + returnedhash = hashlib.md5(data).hexdigest() + logger.debug(u'Computed TheSubDB hash %s for %s' % (returnedhash, path)) + return returnedhash From be8749e472552994aa376cf1f66f6036a4a9814d Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 9 Apr 2012 20:28:07 +0200 Subject: [PATCH 007/105] Proper subtitle support --- couchpotato/core/plugins/renamer/main.py | 15 ++++++- couchpotato/core/plugins/scanner/main.py | 45 +++++++++++++++++-- couchpotato/core/plugins/subtitle/main.py | 27 ++++++----- .../core/providers/subtitle/__init__.py | 0 couchpotato/core/providers/subtitle/base.py | 13 ------ .../providers/subtitle/subliminal/__init__.py | 6 --- .../providers/subtitle/subliminal/main.py | 22 --------- 7 files changed, 72 insertions(+), 56 deletions(-) delete mode 100644 couchpotato/core/providers/subtitle/__init__.py delete mode 100644 couchpotato/core/providers/subtitle/base.py delete mode 100644 couchpotato/core/providers/subtitle/subliminal/__init__.py delete mode 100644 couchpotato/core/providers/subtitle/subliminal/main.py diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index e8d70188..563604ca 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -192,11 +192,16 @@ class Renamer(Plugin): if file_type is 'leftover': if self.conf('move_leftover'): rename_files[current_file] = os.path.join(destination, final_folder_name, os.path.basename(current_file)) - else: + elif file_type not in ['subtitle']: rename_files[current_file] = os.path.join(destination, final_folder_name, final_file_name) # Check for extra subtitle files if file_type is 'subtitle': + + # rename subtitles with or without language + #rename_files[current_file] = os.path.join(destination, final_folder_name, final_file_name) + sub_langs = group['subtitle_language'].get(current_file) + rename_extras = self.getRenameExtras( extra_type = 'subtitle_extra', replacements = replacements, @@ -206,6 +211,14 @@ class Renamer(Plugin): group = group, current_file = current_file ) + + # Don't add language if multiple languages in 1 file + if len(sub_langs) > 1: + rename_files[current_file] = os.path.join(destination, final_folder_name, final_file_name) + elif len(sub_langs) == 1: + sub_name = final_file_name.replace(replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext'])) + rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name) + rename_files = mergeDicts(rename_files, rename_extras) # Filename without cd etc diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 2d9607cb..927fb127 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -7,6 +7,7 @@ from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import File from couchpotato.environment import Env from enzyme.exceptions import NoParserError, ParseError +from subliminal.videos import scan import enzyme import logging import os @@ -208,7 +209,7 @@ class Scanner(Plugin): for identifier, group in movie_files.iteritems(): if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) - log.debug('Grouping files for: %s' % identifier) + log.debug('Grouping files: %s' % identifier) for file_path in group['unsorted_files']: wo_ext = file_path[:-(len(getExt(file_path)) + 1)] @@ -233,7 +234,7 @@ class Scanner(Plugin): # Group the files based on the identifier for identifier, group in movie_files.iteritems(): - log.debug('Grouping files for: %s' % identifier) + log.debug('Grouping files on identifier: %s' % identifier) found_files = set(self.path_identifiers.get(identifier, [])) group['unsorted_files'].extend(found_files) @@ -262,7 +263,7 @@ class Scanner(Plugin): file_too_new = tryInt(time.time() - file_time) break - if file_too_new: + if file_too_new and not Env.get('dev'): log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s' % (time.ctime(file_time), identifier)) continue @@ -291,6 +292,9 @@ class Scanner(Plugin): log.debug('Getting metadata for %s' % identifier) group['meta_data'] = self.getMetaData(group) + # Subtitle meta + group['subtitle_language'] = self.getSubtitleLanguage(group) + # Get parent dir from movie files for movie_file in group['files']['movie']: group['parentdir'] = os.path.dirname(movie_file) @@ -381,6 +385,41 @@ class Scanner(Plugin): return {} + def getSubtitleLanguage(self, group): + detected_languages = {} + + # Subliminal scanner + try: + paths = group['files']['movie'] + scan_result = [] + for p in paths: + scan_result.extend(scan(p)) + + for video, detected_subtitles in scan_result: + for s in detected_subtitles: + if s.language and s.path not in paths: + detected_languages[s.path] = [s.language] + except: + log.error('Failed parsing subtitle languages for %s: %s' % (paths, traceback.format_exc())) + + # IDX + for extra in group['files']['subtitle_extra']: + try: + if os.path.isfile(extra): + output = open(extra, 'r') + txt = output.read() + output.close() + + idx_langs = re.findall('\nid: (\w+)', txt) + + sub_file = '%s.sub' % os.path.splitext(extra)[0] + if len(idx_langs) > 0 and os.path.isfile(sub_file): + detected_languages[sub_file] = idx_langs + except: + log.error('Failed parsing subtitle idx for %s: %s' % (extra, traceback.format_exc())) + + return detected_languages + def determineMovie(self, group): imdb_id = None diff --git a/couchpotato/core/plugins/subtitle/main.py b/couchpotato/core/plugins/subtitle/main.py index 45f87dbf..da86d493 100644 --- a/couchpotato/core/plugins/subtitle/main.py +++ b/couchpotato/core/plugins/subtitle/main.py @@ -3,15 +3,18 @@ from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, FileType +from couchpotato.environment import Env +import subliminal log = CPLog(__name__) class Subtitle(Plugin): + services = ['opensubtitles', 'thesubdb', 'subswiki'] + def __init__(self): - pass - #addEvent('renamer.before', self.searchSingle) + addEvent('renamer.before', self.searchSingle) def searchLibrary(self): @@ -33,20 +36,22 @@ class Subtitle(Plugin): files.append(file.path); # get subtitles for those files - subtitles = fireEvent('subtitle.search', files = files, languages = self.getLanguages(), merge = True) - - # do something with the returned subtitles - print subtitles - + subliminal.list_subtitles(files, cache_dir = Env.get('cache_dir'), multi = True, languages = self.getLanguages(), services = self.services) def searchSingle(self, group): if self.isDisabled(): return - subtitles = fireEvent('subtitle.search', files = group['files']['movie'], languages = self.getLanguages(), merge = True) + available_languages = sum(group['subtitle_language'].itervalues(), []) + downloaded = [] + for lang in self.getLanguages(): + if lang not in available_languages: + download = subliminal.download_subtitles(group['files']['movie'], multi = True, force = False, languages = [lang], services = self.services, cache_dir = Env.get('cache_dir')) + downloaded.extend(download) - # do something with the returned subtitles - print subtitles + for d_sub in downloaded: + group['files']['subtitle'].add(d_sub.path) + group['subtitle_language'][d_sub.path] = [d_sub.language] def getLanguages(self): - return self.conf('languages').split(',') + return [x.strip() for x in self.conf('languages').split(',')] diff --git a/couchpotato/core/providers/subtitle/__init__.py b/couchpotato/core/providers/subtitle/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/couchpotato/core/providers/subtitle/base.py b/couchpotato/core/providers/subtitle/base.py deleted file mode 100644 index d26a6d1a..00000000 --- a/couchpotato/core/providers/subtitle/base.py +++ /dev/null @@ -1,13 +0,0 @@ -from couchpotato.core.event import addEvent -from couchpotato.core.providers.base import Provider - - -class SubtitleProvider(Provider): - - type = 'subtitle' - - def __init__(self): - addEvent('subtitle.search', self.search) - - def search(self, group): - pass diff --git a/couchpotato/core/providers/subtitle/subliminal/__init__.py b/couchpotato/core/providers/subtitle/subliminal/__init__.py deleted file mode 100644 index bc3cb856..00000000 --- a/couchpotato/core/providers/subtitle/subliminal/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .main import SubliminalProvider - -def start(): - return SubliminalProvider() - -config = [] diff --git a/couchpotato/core/providers/subtitle/subliminal/main.py b/couchpotato/core/providers/subtitle/subliminal/main.py deleted file mode 100644 index 9616aeac..00000000 --- a/couchpotato/core/providers/subtitle/subliminal/main.py +++ /dev/null @@ -1,22 +0,0 @@ -from couchpotato.core.logger import CPLog -from couchpotato.core.providers.subtitle.base import SubtitleProvider -from couchpotato.environment import Env -from libs import subliminal - -log = CPLog(__name__) - - -class SubliminalProvider(SubtitleProvider): - - plugins = ['OpenSubtitles', 'TheSubDB', 'SubsWiki'] - - def search(self, files = [], languages = []): - - # download subtitles - with subliminal.Subliminal(cache_dir = Env.get('cache_dir'), multi = True, - languages = self.getLanguages(), plugins = self.plugins) as subli: - subtitles = subli.downloadSubtitles(files) - - print subtitles - - return subtitles From f283988849b9dae65d782444643a14c91089f6dc Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 9 Apr 2012 20:39:22 +0200 Subject: [PATCH 008/105] Better description --- couchpotato/core/plugins/subtitle/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/plugins/subtitle/__init__.py b/couchpotato/core/plugins/subtitle/__init__.py index 903e934e..858ce9d6 100644 --- a/couchpotato/core/plugins/subtitle/__init__.py +++ b/couchpotato/core/plugins/subtitle/__init__.py @@ -20,14 +20,14 @@ config = [{ }, { 'name': 'languages', - 'description': 'The languages you want to download the sub', - }, - { - 'name': 'automatic', - 'default': True, - 'type': 'bool', - 'description': 'Automaticly search & download for movies in library', + 'description': 'Comma separated, 2 letter country code. Example: en, nl', }, +# { +# 'name': 'automatic', +# 'default': True, +# 'type': 'bool', +# 'description': 'Automaticly search & download for movies in library', +# }, ], }, ], From 2365ccb2e6c11a6c7040140706d83b811d24da9b Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Apr 2012 22:25:48 +0200 Subject: [PATCH 009/105] Don't save faulty year --- couchpotato/core/providers/movie/imdbapi/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/providers/movie/imdbapi/main.py b/couchpotato/core/providers/movie/imdbapi/main.py index 604b37c1..e9077687 100644 --- a/couchpotato/core/providers/movie/imdbapi/main.py +++ b/couchpotato/core/providers/movie/imdbapi/main.py @@ -69,6 +69,8 @@ class IMDBAPI(MovieProvider): if tmp_movie.get(key).lower() == 'n/a': del movie[key] + year = tryInt(movie.get('Year', '')) + movie_data = { 'titles': [movie.get('Title')] if movie.get('Title') else [], 'original_title': movie.get('Title', ''), @@ -82,7 +84,7 @@ class IMDBAPI(MovieProvider): 'imdb': str(movie.get('ID', '')), 'runtime': self.runtimeToMinutes(movie.get('Runtime', '')), 'released': movie.get('Released', ''), - 'year': movie.get('Year', ''), + 'year': year if isinstance(year, (int)) else None, 'plot': movie.get('Plot', ''), 'genres': movie.get('Genre', '').split(','), 'directors': movie.get('Director', '').split(','), From 3d82c4de5aebc6b412d6dc20723346654e3a0239 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Apr 2012 23:05:25 +0200 Subject: [PATCH 010/105] Don't check available when dev --- couchpotato/core/providers/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 0d00179b..64e7011f 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -19,7 +19,7 @@ class Provider(Plugin): def isAvailable(self, test_url): - if Env.get('debug'): return True + if Env.get('dev'): return True now = time.time() host = urlparse(test_url).hostname From a29f936fce3ee2354746ba16c56f445241270fa5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Apr 2012 23:08:10 +0200 Subject: [PATCH 011/105] Disable requests to hosts after 5 fails --- couchpotato/core/plugins/base.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 41802e82..da779bc7 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -29,6 +29,8 @@ class Plugin(object): http_last_use = {} http_time_between_calls = 0 + http_failed_request = {} + http_failed_disabled = {} def registerPlugin(self): addEvent('app.shutdown', self.doShutdown) @@ -101,6 +103,16 @@ class Plugin(object): headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2' host = urlparse(url).hostname + + # Don't try for failed requests + if self.http_failed_disabled.get(host, 0) > 0: + if self.http_failed_disabled[host] > (time.time() - 900): + log.info('Disabled calls to %s for 15 minutes because so many failed requests.' % host) + raise Exception + else: + del self.http_failed_request[host] + del self.http_failed_disabled[host] + self.wait(host) try: @@ -122,6 +134,21 @@ class Plugin(object): except IOError: if show_error: log.error('Failed opening url in %s: %s %s' % (self.getName(), url, traceback.format_exc(1))) + + # Save failed requests by hosts + try: + if not self.http_failed_request.get(host): + self.http_failed_request[host] = 1 + else: + self.http_failed_request[host] += 1 + + # Disable temporarily + if self.http_failed_request[host] > 5: + self.http_failed_disabled[host] = time.time() + + except: + log.debug('Failed logging failed requests for %s: %s' % (url, traceback.format_exc())) + raise self.http_last_use[host] = time.time() From 13873358e10d17db2dbb2817bbb5ea42c7ea965d Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Apr 2012 23:49:17 +0200 Subject: [PATCH 012/105] Delete movies from manage list --- couchpotato/static/scripts/page/wanted.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/static/scripts/page/wanted.js b/couchpotato/static/scripts/page/wanted.js index 8d1dd931..95958510 100644 --- a/couchpotato/static/scripts/page/wanted.js +++ b/couchpotato/static/scripts/page/wanted.js @@ -150,7 +150,7 @@ window.addEvent('domready', function(){ var self = this; self.el = new Element('a.delete', { - 'title': 'Remove the movie from your wanted list', + 'title': 'Remove the movie from this CP list', 'events': { 'click': self.showConfirm.bind(self) } @@ -286,6 +286,7 @@ window.addEvent('domready', function(){ }, }) + ,'Delete': MovieActions.Wanted.Delete }; }) \ No newline at end of file From 00eda676230b8957b37c2adf619ac9e50e346c34 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Apr 2012 23:50:39 +0200 Subject: [PATCH 013/105] Force userscript width popup --- couchpotato/core/plugins/userscript/template.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/userscript/template.js b/couchpotato/core/plugins/userscript/template.js index d2d58adf..0134cd2b 100644 --- a/couchpotato/core/plugins/userscript/template.js +++ b/couchpotato/core/plugins/userscript/template.js @@ -57,7 +57,7 @@ if (typeof GM_addStyle == 'undefined'){ // Styles GM_addStyle('\ - #cp_popup { font-family: "Helvetica Neue", Helvetica, Arial, Geneva, sans-serif; -moz-border-radius: 6px 0px 0px 6px; -webkit-border-radius: 6px 0px 0px 6px; border-radius: 6px 0px 0px 6px; -moz-box-shadow: 0 0 20px rgba(0,0,0,0.5); -webkit-box-shadow: 0 0 20px rgba(0,0,0,0.5); box-shadow: 0 0 20px rgba(0,0,0,0.5); position:fixed; z-index:9999; bottom:0; right:0; font-size:15px; margin: 20px 0; display: block; background:#4E5969; } \ + #cp_popup { width: 492px; font-family: "Helvetica Neue", Helvetica, Arial, Geneva, sans-serif; -moz-border-radius: 6px 0px 0px 6px; -webkit-border-radius: 6px 0px 0px 6px; border-radius: 6px 0px 0px 6px; -moz-box-shadow: 0 0 20px rgba(0,0,0,0.5); -webkit-box-shadow: 0 0 20px rgba(0,0,0,0.5); box-shadow: 0 0 20px rgba(0,0,0,0.5); position:fixed; z-index:9999; bottom:0; right:0; font-size:15px; margin: 20px 0; display: block; background:#4E5969; } \ #cp_popup:hover { } \ #cp_popup a#add_to { cursor:pointer; text-align:center; text-decoration:none; color: #000; display:block; padding:5px 0 5px 5px; } \ #cp_popup a#close_button { cursor:pointer; float: right; padding:120px 10px 10px; } \ From 2a3105772691ebbed7e68aa6b293712270d0ad4e Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 10 Apr 2012 23:55:15 +0200 Subject: [PATCH 014/105] Open userscript width only --- couchpotato/core/plugins/userscript/template.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/userscript/template.js b/couchpotato/core/plugins/userscript/template.js index 0134cd2b..090a62ba 100644 --- a/couchpotato/core/plugins/userscript/template.js +++ b/couchpotato/core/plugins/userscript/template.js @@ -57,8 +57,8 @@ if (typeof GM_addStyle == 'undefined'){ // Styles GM_addStyle('\ - #cp_popup { width: 492px; font-family: "Helvetica Neue", Helvetica, Arial, Geneva, sans-serif; -moz-border-radius: 6px 0px 0px 6px; -webkit-border-radius: 6px 0px 0px 6px; border-radius: 6px 0px 0px 6px; -moz-box-shadow: 0 0 20px rgba(0,0,0,0.5); -webkit-box-shadow: 0 0 20px rgba(0,0,0,0.5); box-shadow: 0 0 20px rgba(0,0,0,0.5); position:fixed; z-index:9999; bottom:0; right:0; font-size:15px; margin: 20px 0; display: block; background:#4E5969; } \ - #cp_popup:hover { } \ + #cp_popup { font-family: "Helvetica Neue", Helvetica, Arial, Geneva, sans-serif; -moz-border-radius: 6px 0px 0px 6px; -webkit-border-radius: 6px 0px 0px 6px; border-radius: 6px 0px 0px 6px; -moz-box-shadow: 0 0 20px rgba(0,0,0,0.5); -webkit-box-shadow: 0 0 20px rgba(0,0,0,0.5); box-shadow: 0 0 20px rgba(0,0,0,0.5); position:fixed; z-index:9999; bottom:0; right:0; font-size:15px; margin: 20px 0; display: block; background:#4E5969; } \ + #cp_popup.opened { width: 492px; } \ #cp_popup a#add_to { cursor:pointer; text-align:center; text-decoration:none; color: #000; display:block; padding:5px 0 5px 5px; } \ #cp_popup a#close_button { cursor:pointer; float: right; padding:120px 10px 10px; } \ #cp_popup a img { vertical-align: middle; } \ @@ -98,12 +98,14 @@ var osd = function(){ catch(e){} popup.innerHTML = ''; + popup.setAttribute('class', 'opened'); popup.appendChild(create('a', { 'innerHTML': '', 'id': 'close_button', 'onclick': function(){ popup.innerHTML = ''; popup.appendChild(add_button); + popup.setAttribute('class', ''); } })); popup.appendChild(iframe) From 105fa6b0772604f1494b117cf432ab17599f435a Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 13 Apr 2012 21:54:10 +0200 Subject: [PATCH 015/105] Remove double param on newznab --- couchpotato/core/providers/nzb/newznab/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index c17e6163..38960cc5 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -85,7 +85,6 @@ class Newznab(NZBProvider, RSS): 'imdbid': movie['library']['identifier'].replace('tt', ''), 'cat': cat_id[0], 'apikey': host['api_key'], - 't': self.urls['search'], 'extended': 1 }) url = "%s&%s" % (self.getUrl(host['host'], self.urls['search']), arguments) @@ -121,8 +120,8 @@ class Newznab(NZBProvider, RSS): elif item.attrib.get('name') == 'usenetdate': date = item.attrib.get('value') - if date is '': log.info('Date not parsed properly or not available for %s: %s' % (host, self.getTextElement(nzb, "title"))) - if size is 0: log.info('Size not parsed properly or not available for %s: %s' % (host, self.getTextElement(nzb, "title"))) + if date is '': log.info('Date not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title"))) + if size is 0: log.info('Size not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title"))) id = self.getTextElement(nzb, "guid").split('/')[-1:].pop() new = { From 98230dee321f15c1f12fcb06fb826a5f5d52c57a Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 13 Apr 2012 23:03:12 +0200 Subject: [PATCH 016/105] Default newznab providers Default retention to 1000 days --- couchpotato/core/plugins/searcher/__init__.py | 2 +- couchpotato/core/providers/nzb/newznab/__init__.py | 6 +++--- couchpotato/core/providers/nzb/newznab/main.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/plugins/searcher/__init__.py b/couchpotato/core/plugins/searcher/__init__.py index 8e0f2cf9..a3f88555 100644 --- a/couchpotato/core/plugins/searcher/__init__.py +++ b/couchpotato/core/plugins/searcher/__init__.py @@ -77,7 +77,7 @@ config = [{ 'options': [ { 'name': 'retention', - 'default': 350, + 'default': 1000, 'type': 'int', 'unit': 'days' }, diff --git a/couchpotato/core/providers/nzb/newznab/__init__.py b/couchpotato/core/providers/nzb/newznab/__init__.py index 212c9847..b637ca8e 100644 --- a/couchpotato/core/providers/nzb/newznab/__init__.py +++ b/couchpotato/core/providers/nzb/newznab/__init__.py @@ -19,16 +19,16 @@ config = [{ }, { 'name': 'use', - 'default': '0' + 'default': '0,0,0' }, { 'name': 'host', - 'default': 'nzb.su', + 'default': 'nzb.su,dognzb.cr,beta.nzbs.org', 'description': 'The hostname of your newznab provider', }, { 'name': 'api_key', - 'default': '', + 'default': ',,', 'label': 'Api Key', 'description': 'Can be found on your profile page', 'type': 'combined', diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index 38960cc5..c3bb6b2b 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -120,8 +120,8 @@ class Newznab(NZBProvider, RSS): elif item.attrib.get('name') == 'usenetdate': date = item.attrib.get('value') - if date is '': log.info('Date not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title"))) - if size is 0: log.info('Size not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title"))) + if date is '': log.debug('Date not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title"))) + if size is 0: log.debug('Size not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title"))) id = self.getTextElement(nzb, "guid").split('/')[-1:].pop() new = { From dac9abc52674721731f50ea0c5a72e43a43f3966 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 15 Apr 2012 10:44:21 +0200 Subject: [PATCH 017/105] Remove duplicates on list merge --- couchpotato/core/helpers/variable.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index f38cf39c..53b71941 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -36,10 +36,18 @@ def mergeDicts(a, b): stack.append((current_dst[key], current_src[key])) elif isinstance(current_src[key], list) and isinstance(current_dst[key], list): current_dst[key].extend(current_src[key]) + current_dst[key] = removeListDuplicates(current_dst[key]) else: current_dst[key] = current_src[key] return dst +def removeListDuplicates(seq): + checked = [] + for e in seq: + if e not in checked: + checked.append(e) + return checked + def flattenList(l): if isinstance(l, list): return sum(map(flattenList, l)) From 14d12d4d5404d850ac41780ce3e386c020e26b48 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 15 Apr 2012 11:52:56 +0200 Subject: [PATCH 018/105] Show quality in manage list --- couchpotato/core/plugins/movie/static/movie.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/movie/static/movie.js b/couchpotato/core/plugins/movie/static/movie.js index dac9bf98..83e6eb94 100644 --- a/couchpotato/core/plugins/movie/static/movie.js +++ b/couchpotato/core/plugins/movie/static/movie.js @@ -79,7 +79,7 @@ var Movie = new Class({ var q = self.quality.getElement('.q_id'+ release.quality_id), status = Status.get(release.status_id); - if(!q && status.identifier == 'snatched') + if(!q && (status.identifier == 'snatched' || status.identifier == 'done')) var q = self.addQuality(release.quality_id) if (q) q.addClass(status.identifier); From e1629cf05aa3e4f760b27cc9ad7a10f18384f15e Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 15 Apr 2012 12:05:55 +0200 Subject: [PATCH 019/105] Change add button text --- couchpotato/static/scripts/page/wanted.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/static/scripts/page/wanted.js b/couchpotato/static/scripts/page/wanted.js index 95958510..af6d774c 100644 --- a/couchpotato/static/scripts/page/wanted.js +++ b/couchpotato/static/scripts/page/wanted.js @@ -61,7 +61,7 @@ window.addEvent('domready', function(){ 'name': 'profile' }), new Element('a.button.edit', { - 'text': 'Save', + 'text': 'Save & Search', 'events': { 'click': self.save.bind(self) } From 20bfb242ad14fc228d5267e15ec110810ae56fb2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 15 Apr 2012 17:27:50 +0200 Subject: [PATCH 020/105] Better quality guessing --- couchpotato/core/plugins/quality/main.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 5736ddd7..70a7b8bc 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -1,7 +1,7 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import addEvent -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, toSafeString from couchpotato.core.helpers.request import jsonified, getParams from couchpotato.core.helpers.variable import mergeDicts, md5, getExt from couchpotato.core.logger import CPLog @@ -16,7 +16,7 @@ log = CPLog(__name__) class QualityPlugin(Plugin): qualities = [ - {'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'width': 1920, 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]}, + {'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]}, {'identifier': '1080p', 'hd': True, 'size': (5000, 20000), 'label': '1080P', 'width': 1920, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts']}, {'identifier': '720p', 'hd': True, 'size': (3500, 10000), 'label': '720P', 'width': 1280, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts']}, {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p'], 'ext':['avi']}, @@ -138,29 +138,31 @@ class QualityPlugin(Plugin): # Create hash for cache hash = md5(str([f.replace('.' + getExt(f), '') for f in files])) cached = self.getCache(hash) - if cached: return cached + if cached and extra is {}: return cached for cur_file in files: size = (os.path.getsize(cur_file) / 1024 / 1024) if os.path.isfile(cur_file) else 0 words = re.split('\W+', cur_file.lower()) + safe_cur_file = toSafeString(cur_file) for quality in self.all(): # Check tags if quality['identifier'] in words: - log.debug('Found via identifier "%s" in %s' % (quality['identifier'], cur_file)) + log.debug('Found via identifier "%s" in %s' % (quality['identifier'], safe_cur_file)) return self.setCache(hash, quality) if list(set(quality.get('alternative', [])) & set(words)): - log.debug('Found %s via alt %s in %s' % (quality['identifier'], quality.get('alternative'), cur_file)) + log.debug('Found %s via alt %s in %s' % (quality['identifier'], quality.get('alternative'), safe_cur_file)) return self.setCache(hash, quality) for tag in quality.get('tags', []): if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words): + log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), safe_cur_file)) return self.setCache(hash, quality) if list(set(quality.get('tags', [])) & set(words)): - log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file)) + log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), safe_cur_file)) return self.setCache(hash, quality) # Check on unreliable stuff From f1cbb265bd3e96aed5b14d0dd484f719beb35529 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 15 Apr 2012 23:17:38 +0200 Subject: [PATCH 021/105] Cascaded deletion --- couchpotato/core/settings/model.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/settings/model.py b/couchpotato/core/settings/model.py index 4dd54d32..bcf5adb8 100644 --- a/couchpotato/core/settings/model.py +++ b/couchpotato/core/settings/model.py @@ -44,8 +44,8 @@ class Movie(Entity): library = ManyToOne('Library') status = ManyToOne('Status') profile = ManyToOne('Profile') - releases = OneToMany('Release') - files = ManyToMany('File') + releases = OneToMany('Release', cascade = 'all, delete-orphan') + files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True) class Library(Entity): @@ -59,9 +59,9 @@ class Library(Entity): info = Field(JsonType) status = ManyToOne('Status') - movies = OneToMany('Movie') - titles = OneToMany('LibraryTitle') - files = ManyToMany('File') + movies = OneToMany('Movie', cascade = 'all, delete-orphan') + titles = OneToMany('LibraryTitle', cascade = 'all, delete-orphan') + files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True) class LibraryTitle(Entity): @@ -94,9 +94,9 @@ class Release(Entity): movie = ManyToOne('Movie') status = ManyToOne('Status') quality = ManyToOne('Quality') - files = ManyToMany('File') - history = OneToMany('History') - info = OneToMany('ReleaseInfo') + files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True) + history = OneToMany('History', cascade = 'all, delete-orphan') + info = OneToMany('ReleaseInfo', cascade = 'all, delete-orphan') class ReleaseInfo(Entity): @@ -229,6 +229,12 @@ class Folder(Entity): label = Field(Unicode(255)) +class Properties(Entity): + + identifier = Field(String(50), index = True) + value = Field(Unicode(255), nullable = False) + + def setup(): """Setup the database and create the tables that don't exists yet""" from elixir import setup_all, create_all From e3a9bb13013aae987e5cf21233f9ad209c501c28 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 15 Apr 2012 23:17:54 +0200 Subject: [PATCH 022/105] Propertie settings in database --- couchpotato/core/settings/__init__.py | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/couchpotato/core/settings/__init__.py b/couchpotato/core/settings/__init__.py index 60c77cb2..9c4c8fff 100644 --- a/couchpotato/core/settings/__init__.py +++ b/couchpotato/core/settings/__init__.py @@ -4,9 +4,11 @@ from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.encoding import isInt, toUnicode from couchpotato.core.helpers.request import getParams, jsonified from couchpotato.core.helpers.variable import mergeDicts, tryInt +from couchpotato.core.settings.model import Properties import ConfigParser import os.path import time +import traceback class Settings(object): @@ -190,3 +192,29 @@ class Settings(object): return jsonified({ 'success': True, }) + + def getProperty(self, identifier): + from couchpotato import get_session + + db = get_session() + try: + prop = db.query(Properties).filter_by(identifier = identifier).first() + return prop.value + except: + print traceback.format_exc() + return None + + def setProperty(self, identifier, value = ''): + from couchpotato import get_session + + db = get_session() + + p = db.query(Properties).filter_by(identifier = identifier).first() + if not p: + p = Properties() + db.add(p) + + p.identifier = identifier + p.value = toUnicode(value) + + db.commit() From 8c922531e2aca0bf1c0b08a754c4ed3863a75250 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 15 Apr 2012 23:18:48 +0200 Subject: [PATCH 023/105] Sort tmdb search results --- couchpotato/core/providers/movie/themoviedb/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/providers/movie/themoviedb/main.py b/couchpotato/core/providers/movie/themoviedb/main.py index 2838c4ce..50aa2392 100644 --- a/couchpotato/core/providers/movie/themoviedb/main.py +++ b/couchpotato/core/providers/movie/themoviedb/main.py @@ -3,6 +3,7 @@ from couchpotato.core.helpers.encoding import simplifyString, toUnicode from couchpotato.core.logger import CPLog from couchpotato.core.providers.movie.base import MovieProvider from libs.themoviedb import tmdb +import re log = CPLog(__name__) @@ -67,8 +68,12 @@ class TheMovieDb(MovieProvider): if raw: try: nr = 0 - for movie in raw: + # Sort on returned score first when year is in q + if re.search('\s\d{4}', q): + movies = sorted(raw, key = lambda k: k['score'], reverse = True) + + for movie in movies: results.append(self.parseMovie(movie)) nr += 1 From 6f21cceb0b3d1595e4ae3ae9f104ea65d103dbb5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:02:19 +0200 Subject: [PATCH 024/105] Scroll mask with searching --- couchpotato/core/plugins/movie/static/search.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/plugins/movie/static/search.js index 53a8d399..d042cab7 100644 --- a/couchpotato/core/plugins/movie/static/search.js +++ b/couchpotato/core/plugins/movie/static/search.js @@ -367,12 +367,13 @@ Block.Search.Item = new Class({ self.mask = new Element('span.mask', { 'styles': { + 'position': 'relative', 'width': s.x, - 'height': s.y + 'height': s.y, + 'top': -s.y, + 'display': 'block' } - }).inject(self.options).fade('hide').position({ - 'relativeTo': self.options - }) + }).inject(self.options).fade('hide') createSpinner(self.mask) self.mask.fade('in') From 600d2b70bd4452f63ba9e19f7684f1976b9ddd88 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:05:46 +0200 Subject: [PATCH 025/105] Fadeout mask, don't hide --- couchpotato/core/plugins/movie/static/search.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/plugins/movie/static/search.js index d042cab7..65c4327d 100644 --- a/couchpotato/core/plugins/movie/static/search.js +++ b/couchpotato/core/plugins/movie/static/search.js @@ -158,7 +158,7 @@ Block.Search = new Class({ rc = self.result_container.getCoordinates(); self.results.setStyle('max-height', (w.y - rc.top - 50) + 'px') - self.mask.hide() + self.mask.fade('out') }, From 339c0146748161806e0cbdf5a0766f5039296bf0 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:06:43 +0200 Subject: [PATCH 026/105] Environment property function --- couchpotato/environment.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/couchpotato/environment.py b/couchpotato/environment.py index ac256c18..ec2fbbf1 100644 --- a/couchpotato/environment.py +++ b/couchpotato/environment.py @@ -61,6 +61,14 @@ class Env(object): return s + @staticmethod + def prop(identifier, value = None): + s = Env.get('settings') + if value == None: + return s.getProperty(identifier) + + s.setProperty(identifier, value) + @staticmethod def getPermission(setting_type): perm = Env.get('settings').get('permission_%s' % setting_type, default = '0777') From a400136e66f74e033b4970eef2bb2494425f4cf5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:06:57 +0200 Subject: [PATCH 027/105] Use custom mask for question --- couchpotato/static/scripts/library/question.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/couchpotato/static/scripts/library/question.js b/couchpotato/static/scripts/library/question.js index b80005e3..cab63465 100644 --- a/couchpotato/static/scripts/library/question.js +++ b/couchpotato/static/scripts/library/question.js @@ -18,13 +18,7 @@ var Question = new Class( { createMask : function() { var self = this - $(document.body).mask( { - 'hideOnClick' : true, - 'destroyOnHide' : true, - 'onHide' : function() { - self.container.destroy(); - } - }).show(); + self.mask = new Element('div.mask').fade('hide').inject(document.body).fade('in'); }, createQuestion : function() { @@ -71,7 +65,11 @@ var Question = new Class( { }, close : function() { - $(document.body).get('mask').destroy(); + var self = this; + self.mask.fade('out'); + (function(){self.mask.destroy()}).delay(1000); + + this.container.destroy(); }, toElement : function() { From ce25e8a9ad97984e2af882fdb8c57939a80c4b7c Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:07:51 +0200 Subject: [PATCH 028/105] Cleanup --- couchpotato/static/scripts/page/manage.js | 1 - couchpotato/static/style/main.css | 1 - 2 files changed, 2 deletions(-) diff --git a/couchpotato/static/scripts/page/manage.js b/couchpotato/static/scripts/page/manage.js index 74dba986..0385ee0f 100644 --- a/couchpotato/static/scripts/page/manage.js +++ b/couchpotato/static/scripts/page/manage.js @@ -38,7 +38,6 @@ Page.Manage = new Class({ refresh: function(full){ var self = this; - p(full) Api.request('manage.update', { 'data': { diff --git a/couchpotato/static/style/main.css b/couchpotato/static/style/main.css index a128ee32..55246c7d 100644 --- a/couchpotato/static/style/main.css +++ b/couchpotato/static/style/main.css @@ -13,7 +13,6 @@ body { background: #4e5969; overflow-y: scroll; height: 100%; - text-align: justify; } body.noscroll { overflow: hidden; } From 7704ff8e755223167fe9368fa805e273c48c0221 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:12:52 +0200 Subject: [PATCH 029/105] Don't log host in javascript error --- couchpotato/templates/_desktop.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/templates/_desktop.html b/couchpotato/templates/_desktop.html index 63aa4dd8..e6d1ccb2 100644 --- a/couchpotato/templates/_desktop.html +++ b/couchpotato/templates/_desktop.html @@ -66,8 +66,8 @@ 'data': { 'type': 'error', 'message': Browser.name + ' ' + Browser.version + ': \n' + message, - 'page': window.location.href, - 'file': file, + 'page': window.location.href.replace(window.location.host, 'HOST'), + 'file': file.replace(window.location.host, 'HOST'), 'line': line } }); From 2beba28e5f26180f040db231257ce043272a2762 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:19:17 +0200 Subject: [PATCH 030/105] Undefined variable fix --- couchpotato/core/providers/movie/themoviedb/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/providers/movie/themoviedb/main.py b/couchpotato/core/providers/movie/themoviedb/main.py index 50aa2392..a84b5568 100644 --- a/couchpotato/core/providers/movie/themoviedb/main.py +++ b/couchpotato/core/providers/movie/themoviedb/main.py @@ -72,6 +72,8 @@ class TheMovieDb(MovieProvider): # Sort on returned score first when year is in q if re.search('\s\d{4}', q): movies = sorted(raw, key = lambda k: k['score'], reverse = True) + else: + movies = raw for movie in movies: results.append(self.parseMovie(movie)) From 24e45b0b345e830c25d68e4bdb5495e3fa5b7a8a Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:20:39 +0200 Subject: [PATCH 031/105] Movie delete event --- couchpotato/core/plugins/movie/main.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 354e64a6..073e2fc7 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -74,7 +74,7 @@ class MoviePlugin(Plugin): 'default_title': {'desc': 'Movie title to use for searches. Has to be one of the titles returned by movie.search.'}, } }) - addApiView('movie.delete', self.delete, docs = { + addApiView('movie.delete', self.deleteView, docs = { 'desc': 'Delete a movie from the wanted list', 'params': { 'id': {'desc': 'Movie ID(s) you want to delete.', 'type': 'int (comma separated)'}, @@ -82,6 +82,7 @@ class MoviePlugin(Plugin): }) addEvent('movie.add', self.add) + addEvent('movie.delete', self.delete) addEvent('movie.get', self.get) addEvent('movie.list', self.list) addEvent('movie.restatus', self.restatus) @@ -347,23 +348,28 @@ class MoviePlugin(Plugin): 'success': True, }) - def delete(self): + def deleteView(self): params = getParams() - db = get_session() - - status = fireEvent('status.add', 'deleted', single = True) ids = params.get('id').split(',') for movie_id in ids: - movie = db.query(Movie).filter_by(id = movie_id).first() - movie.status_id = status.get('id') - db.commit() + self.delete(movie_id) return jsonified({ 'success': True, }) + def delete(self, movie_id): + + db = get_session() + + movie = db.query(Movie).filter_by(id = movie_id).first() + db.delete(movie) + db.commit() + + return True + def restatus(self, movie_id): active_status = fireEvent('status.get', 'active', single = True) From 163ed5bc7567f933ac1e64ae465c07d7b0ab3ce9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:21:08 +0200 Subject: [PATCH 032/105] Restatus, even when no profile is set --- couchpotato/core/plugins/movie/main.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 073e2fc7..50042453 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -379,18 +379,17 @@ class MoviePlugin(Plugin): m = db.query(Movie).filter_by(id = movie_id).first() - if not m.profile: - return - log.debug('Changing status for %s' % (m.library.titles[0].title)) + if not m.profile: + m.status_id = done_status.get('id') + else: + move_to_wanted = True - move_to_wanted = True + for t in m.profile.types: + for release in m.releases: + if t.quality.identifier is release.quality.identifier and (release.status_id is done_status.get('id') and t.finish): + move_to_wanted = False - for t in m.profile.types: - for release in m.releases: - if t.quality.identifier is release.quality.identifier and (release.status_id is done_status.get('id') and t.finish): - move_to_wanted = False - - m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id') + m.status_id = active_status.get('id') if move_to_wanted else done_status.get('id') db.commit() From 1dc5b4bac4f9010d3d497d050cc513d70879062b Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:28:52 +0200 Subject: [PATCH 033/105] Don't fire loading page twice --- couchpotato/static/scripts/couchpotato.js | 1 - 1 file changed, 1 deletion(-) diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index edafdf4c..949d35cd 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -190,7 +190,6 @@ var CouchPotato = new Class({ }, 'onSuccess': function(){ self.unBlockPage(); - self.fireEvent('load'); } }); From 7b95fd1a3c6cf503c7f48e94ec561f15d23b0b49 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 00:37:48 +0200 Subject: [PATCH 034/105] Pretty restart questions --- couchpotato/static/scripts/couchpotato.js | 49 ++++++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index 949d35cd..9809cd67 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -70,13 +70,13 @@ var CouchPotato = new Class({ [new Element('a.orange', { 'text': 'Restart', 'events': { - 'click': self.restart.bind(self) + 'click': self.restartQA.bind(self) } }), new Element('a.red', { 'text': 'Shutdown', 'events': { - 'click': self.shutdown.bind(self) + 'click': self.shutdownQA.bind(self) } }), new Element('a', { @@ -161,6 +161,29 @@ var CouchPotato = new Class({ self.checkAvailable(1000); }, + shutdownQA: function(){ + var self = this; + + new Question('Are you sure you want to shutdown CouchPotato?', '', [{ + 'text': 'Shutdown', + 'class': 'shutdown red', + 'events': { + 'click': function(e){ + (e).preventDefault(); + + self.blockPage('You have shutdown. This is what suppose to happen ;)'); + Api.request('app.shutdown', { + 'onComplete': self.blockPage.bind(self) + }); + self.checkAvailable(1000); + } + } + }, { + 'text': 'No, nevah!', + 'cancel': true + }]); + }, + restart: function(message, title){ var self = this; @@ -169,6 +192,28 @@ var CouchPotato = new Class({ self.checkAvailable(1000); }, + restartQA: function(message, title){ + var self = this; + + new Question('Are you sure you want to restart CouchPotato?', '', [{ + 'text': 'Restart', + 'class': 'restart orange', + 'events': { + 'click': function(e){ + (e).preventDefault(); + + self.blockPage(message || 'Restarting... please wait. If this takes to long, something must have gone wrong.', title); + Api.request('app.restart'); + self.checkAvailable(1000); + + } + } + }, { + 'text': 'No, nevah!', + 'cancel': true + }]); + }, + checkForUpdate: function(func){ var self = this; From 36cff1e1bbc4f8e1c37598eb3ac98e51d58b9e73 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 07:57:40 +0200 Subject: [PATCH 035/105] Use existing functions --- couchpotato/static/scripts/couchpotato.js | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index 9809cd67..0f8fcd99 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -170,12 +170,7 @@ var CouchPotato = new Class({ 'events': { 'click': function(e){ (e).preventDefault(); - - self.blockPage('You have shutdown. This is what suppose to happen ;)'); - Api.request('app.shutdown', { - 'onComplete': self.blockPage.bind(self) - }); - self.checkAvailable(1000); + self.shutdown(); } } }, { @@ -201,11 +196,7 @@ var CouchPotato = new Class({ 'events': { 'click': function(e){ (e).preventDefault(); - - self.blockPage(message || 'Restarting... please wait. If this takes to long, something must have gone wrong.', title); - Api.request('app.restart'); - self.checkAvailable(1000); - + self.restart(message, title); } } }, { From cdc3fa530ef44c2a783d64d0f1075bdd1472c842 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 09:03:30 +0200 Subject: [PATCH 036/105] Allow quality-less release when year is smaller as now --- couchpotato/core/plugins/searcher/main.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index f3dc28fa..f2a85724 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -7,6 +7,7 @@ from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Movie, Release, ReleaseInfo from couchpotato.environment import Env from sqlalchemy.exc import InterfaceError +import datetime import re import traceback @@ -24,6 +25,7 @@ class Searcher(Plugin): # Schedule cronjob fireEvent('schedule.cron', 'searcher.all', self.all_movies, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) + def test(): def all_movies(self): db = get_session() @@ -188,7 +190,7 @@ class Searcher(Plugin): preferred_quality = fireEvent('quality.single', identifier = quality['identifier'], single = True) # Contains lower quality string - if self.containsOtherQuality(nzb['name'], preferred_quality, single_category): + if self.containsOtherQuality(nzb, movie_year = movie['library']['year'], preferred_quality = preferred_quality, single_category = single_category): log.info('Wrong: %s, looking for %s' % (nzb['name'], quality['label'])) return False @@ -230,8 +232,10 @@ class Searcher(Plugin): log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'" % (nzb['name'], movie['library']['titles'][0]['title'], movie['library']['year'])) return False - def containsOtherQuality(self, name, preferred_quality = {}, single_category = False): + def containsOtherQuality(self, nzb, movie_year = None, preferred_quality = {}, single_category = False): + name = nzb['name'] + size = nzb.get('size', 0) nzb_words = re.split('\W+', simplifyString(name)) qualities = fireEvent('quality.all', single = True) @@ -246,6 +250,13 @@ class Searcher(Plugin): if list(set(nzb_words) & set(quality['alternative'])): found[quality['identifier']] = True + # Hack for older movies that don't contain quality tag + if movie_year < datetime.datetime.now().year: + if size > 3000: # Assume dvdr + return 'dvdr' == preferred_quality['identifier'] + else: # Assume dvdrip + return 'dvdrip' == preferred_quality['identifier'] + # Allow other qualities for allowed in preferred_quality.get('allow'): if found.get(allowed): From 8f577fa5f153666e1aaf20578f1b12d1736bc8da Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 10:16:34 +0200 Subject: [PATCH 037/105] Remove leftover testfunction --- couchpotato/core/plugins/searcher/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index f2a85724..b0e5297d 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -25,7 +25,6 @@ class Searcher(Plugin): # Schedule cronjob fireEvent('schedule.cron', 'searcher.all', self.all_movies, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) - def test(): def all_movies(self): db = get_session() From 9ac2cf35eaae1d8302b0f033a6019fe34fb96101 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 23:16:04 +0200 Subject: [PATCH 038/105] Improved folder scanning --- couchpotato/core/plugins/manage/main.py | 31 ++++++++-------- couchpotato/core/plugins/scanner/main.py | 45 +++++++++++++++--------- 2 files changed, 46 insertions(+), 30 deletions(-) diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 20fdd4c3..7ce0ecad 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -14,8 +14,6 @@ log = CPLog(__name__) class Manage(Plugin): - last_update = 0 - def __init__(self): fireEvent('scheduler.interval', identifier = 'manage.update_library', handle = self.updateLibrary, hours = 2) @@ -42,12 +40,14 @@ class Manage(Plugin): }) - def updateLibrary(self, full = False): + def updateLibrary(self, full = True): + last_update = float(Env.prop('manage.last_update')) - if self.isDisabled() or (self.last_update > time.time() - 20): + if self.isDisabled() or (last_update > time.time() - 20): return directories = self.directories() + added_identifiers = [] for directory in directories: @@ -57,21 +57,24 @@ class Manage(Plugin): continue log.info('Updating manage library: %s' % directory) - fireEvent('scanner.folder', folder = directory) - - # If cleanup option is enabled, remove offline files from database - if self.conf('cleanup'): - db = get_session() - files_in_path = db.query(File).filter(File.path.like(directory + '%%')).filter_by(available = 0).all() - [db.delete(x) for x in files_in_path] - db.commit() - db.remove() + identifiers = fireEvent('scanner.folder', folder = directory, newer_than = last_update, single = True) + added_identifiers.extend(identifiers) # Break if CP wants to shut down if self.shuttingDown(): break - self.last_update = time.time() + # If cleanup option is enabled, remove offline files from database + if self.conf('cleanup') and full and not self.shuttingDown(): + + # Get movies with done status + done_movies = fireEvent('movie.list', status = 'done', single = True) + + for done_movie in done_movies: + if done_movie['library']['identifier'] not in added_identifiers: + fireEvent('movie.delete', movie_id = done_movie['id']) + + Env.prop('manage.last_update', time.time()) def directories(self): try: diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 927fb127..bf88e7f0 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -7,6 +7,7 @@ from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import File from couchpotato.environment import Env from enzyme.exceptions import NoParserError, ParseError +from guessit import guess_movie_info from subliminal.videos import scan import enzyme import logging @@ -96,6 +97,10 @@ class Scanner(Plugin): addEvent('rename.after', after_rename) + # Disable lib logging + logging.getLogger('guessit').setLevel(logging.ERROR) + logging.getLogger('subliminal').setLevel(logging.ERROR) + def scanFilesToLibrary(self, folder = None, files = None): groups = self.scan(folder = folder, files = files) @@ -104,21 +109,14 @@ class Scanner(Plugin): if group['library']: fireEvent('release.add', group = group) - def scanFolderToLibrary(self, folder = None, newer_as = None): + def scanFolderToLibrary(self, folder = None, newer_than = None): if not os.path.isdir(folder): return groups = self.scan(folder = folder) - # Open up the db - db = get_session() - - # Mark all files as "offline" before a adding them to the database (again) - files_in_path = db.query(File).filter(File.path.like(toUnicode(folder) + u'%%')) - files_in_path.update({'available': 0}, synchronize_session = False) - db.commit() - + added_identifier = [] while True and not self.shuttingDown(): try: identifier, group = groups.popitem() @@ -130,9 +128,10 @@ class Scanner(Plugin): # Add release fireEvent('release.add', group = group) - fireEvent('library.update', identifier = group['library'].get('identifier')) + library_item = fireEvent('library.update', identifier = group['library'].get('identifier'), single = True) + added_identifier.append(library_item['identifier']) - db.remove() + return added_identifier def scan(self, folder = None, files = []): @@ -179,7 +178,7 @@ class Scanner(Plugin): identifier = self.createStringIdentifier(file_path, folder, exclude_filename = is_dvd_file) # Identifier with quality - quality = fireEvent('quality.guess', [file_path], single = True) + quality = fireEvent('quality.guess', [file_path], single = True) if not is_dvd_file else {'identifier':'dvdr'} identifier_with_quality = '%s %s' % (identifier, quality.get('identifier', '')) if not movie_files.get(identifier): @@ -393,14 +392,15 @@ class Scanner(Plugin): paths = group['files']['movie'] scan_result = [] for p in paths: - scan_result.extend(scan(p)) + if not group['is_dvd']: + scan_result.extend(scan(p)) for video, detected_subtitles in scan_result: for s in detected_subtitles: if s.language and s.path not in paths: detected_languages[s.path] = [s.language] except: - log.error('Failed parsing subtitle languages for %s: %s' % (paths, traceback.format_exc())) + log.debug('Failed parsing subtitle languages for %s: %s' % (paths, traceback.format_exc())) # IDX for extra in group['files']['subtitle_extra']: @@ -472,7 +472,9 @@ class Scanner(Plugin): for identifier in group['identifiers']: if len(identifier) > 2: - movie = fireEvent('movie.search', q = '%(name)s %(year)s' % self.getReleaseNameYear(identifier), merge = True, limit = 1) + try: filename = list(group['files'].get('movie'))[0] + except: filename = None + movie = fireEvent('movie.search', q = '%(name)s %(year)s' % self.getReleaseNameYear(identifier, file_name = filename), merge = True, limit = 1) if len(movie) > 0: imdb_id = movie[0]['imdb'] @@ -690,7 +692,18 @@ class Scanner(Plugin): return '' - def getReleaseNameYear(self, release_name): + def getReleaseNameYear(self, release_name, file_name = None): + + # Use guessit first + if file_name: + guess = guess_movie_info(file_name) + if guess.get('title') and guess.get('year'): + return { + 'name': guess.get('title'), + 'year': guess.get('year'), + } + + # Backup to simple cleaned = ' '.join(re.split('\W+', simplifyString(release_name))) cleaned = re.sub(self.clean, ' ', cleaned) year = self.findYear(cleaned) From 754a49fe2f2de56d7a2a1619be9d92cb0895f571 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 23:46:38 +0200 Subject: [PATCH 039/105] Added profile and quality list to API --- couchpotato/core/plugins/profile/main.py | 14 ++++++++++++++ couchpotato/core/plugins/quality/main.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index 54213e7e..0ba0637a 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -21,9 +21,23 @@ class ProfilePlugin(Plugin): addApiView('profile.save', self.save) addApiView('profile.save_order', self.saveOrder) addApiView('profile.delete', self.delete) + addApiView('profile.list', self.allView, docs = { + 'desc': 'List all available profiles', + 'return': {'type': 'object', 'example': """{ + 'success': True, + 'list': array, profiles +}"""} + }) addEvent('app.initialize', self.fill, priority = 90) + def allView(self): + + return jsonified({ + 'success': True, + 'list': self.all() + }) + def all(self): db = get_session() diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 70a7b8bc..63d37be8 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -36,9 +36,23 @@ class QualityPlugin(Plugin): addEvent('quality.guess', self.guess) addApiView('quality.size.save', self.saveSize) + addApiView('quality.list', self.allView, docs = { + 'desc': 'List all available qualities', + 'return': {'type': 'object', 'example': """{ + 'success': True, + 'list': array, qualities +}"""} + }) addEvent('app.initialize', self.fill, priority = 10) + def allView(self): + + return jsonified({ + 'success': True, + 'list': self.all() + }) + def all(self): db = get_session() From 967aba0f82c5c77839b943b4c81353d3e1a29f88 Mon Sep 17 00:00:00 2001 From: Ruud Date: Mon, 16 Apr 2012 23:51:03 +0200 Subject: [PATCH 040/105] Synology init script --- init/synology | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 init/synology diff --git a/init/synology b/init/synology new file mode 100644 index 00000000..859582bd --- /dev/null +++ b/init/synology @@ -0,0 +1,81 @@ +#!/bin/sh + +# Package +PACKAGE="couchpotato" +DNAME="CouchPotato" + +# Others +INSTALL_DIR="/opt/${PACKAGE}" +PYTHON_DIR="/opt/bin" +PATH="${INSTALL_DIR}/bin:${INSTALL_DIR}/env/bin:${PYTHON_DIR}/bin:/usr/local/bin:/bin:/usr/bin:/usr/syno/bin" +RUNAS="couchpotato" +PYTHON="${PYTHON_DIR}/python2.7" +COUCHPOTATO="${INSTALL_DIR}/CouchPotato.py" +CFG_FILE="${INSTALL_DIR}/var/settings.conf" +PID_FILE="${INSTALL_DIR}/var/couchpotato.pid" +LOG_FILE="${INSTALL_DIR}/var/logs/CouchPotato.log" + + +start_daemon() +{ +su ${RUNAS} -c "PATH=${PATH} ${PYTHON} ${COUCHPOTATO} --daemon --pid_file ${PID_FILE} --config ${CFG_FILE}" +} + +stop_daemon() +{ + kill `cat ${PID_FILE}` + wait_for_status 1 20 + rm -f ${PID_FILE} +} + +daemon_status() +{ + if [ -f ${PID_FILE} ] && [ -d /proc/`cat ${PID_FILE}` ]; then +return 0 + fi +return 1 +} + +wait_for_status() +{ + counter=$2 + while [ ${counter} -gt 0 ]; do +daemon_status + [ $? -eq $1 ] && break +let counter=counter-1 + sleep 1 + done +} +case $1 in + start) + if daemon_status; then +echo ${DNAME} is already running + else +echo Starting ${DNAME} ... + start_daemon + fi + ;; + stop) + if daemon_status; then +echo Stopping ${DNAME} ... + stop_daemon + else +echo ${DNAME} is not running + fi + ;; + status) + if daemon_status; then +echo ${DNAME} is running + exit 0 + else +echo ${DNAME} is not running + exit 1 + fi + ;; + log) + echo ${LOG_FILE} + ;; + *) + exit 1 + ;; +esac \ No newline at end of file From c509aa331fb5c13742de20227b08ef7c218b5bbf Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 17 Apr 2012 08:21:50 +0200 Subject: [PATCH 041/105] Added some 'run from source'-help --- README.md | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f70d091..ea03f1e3 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,37 @@ CouchPotato Server ===== CouchPotato (CP) is an automatic NZB and torrent downloader. You can keep a "movies I want"-list and it will search for NZBs/torrents of these movies every X hours. -Once a movie is found, it will send it to SABnzbd or download the torrent to a specified directory. \ No newline at end of file +Once a movie is found, it will send it to SABnzbd or download the torrent to a specified directory. + + +## Running from Source + +CouchPotatoServer can be run from source. This will use *git* as updater, so make sure that is installed also. + +Windows: + +* Install [PyWin32 2.7](http://sourceforge.net/projects/pywin32/files/pywin32/Build%20217/) and [GIT](http://git-scm.com/) +* If you come and ask on the forums 'why directory selection no work?', I will kill a kitten, also this is because you need PyWin32 +* Open up `Git Bash` (or CMD) and go to the folder you want to install CP. Something like Program Files. +* Run `git clone https://RuudBurger@github.com/RuudBurger/CouchPotatoServer.git`. +* You can now start CP via `CouchPotatoServer\CouchPotato.py` to start + +OSx: + +* If you're on Leopard (10.5) install Python 2.6+: [Python 2.6.5](http://www.python.org/download/releases/2.6.5/) +* Install [GIT](http://git-scm.com/) +* Open up `Terminal` +* Go to your App folder `cd /Applications` +* Run `git clone https://RuudBurger@github.com/RuudBurger/CouchPotatoServer.git` +* Then do `python CouchPotatoServer/CouchPotato.py` + +Linux (ubuntu / debian): + +* Install [GIT](http://git-scm.com/) with `apt-get install git-core` +* 'cd' to the folder of your choosing. +* Run `git clone https://RuudBurger@github.com/RuudBurger/CouchPotatoServer.git` +* Then do `python CouchPotatoServer/CouchPotato.py` to start +* To run on boot copy the init script. `cp CouchPotatoServer/init/ubuntu /etc/init.d/couchpotato` +* Change the paths inside the init script. `nano /etc/init.d/couchpotato` +* Make it executable. `chmod +x /etc/init.d/couchpotato` +* Add it to defaults. `sudo update-rc.d couchpotato defaults` From 7c2233b26616bf6ee979b292c95ea38497c8f449 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 17 Apr 2012 20:27:40 +0200 Subject: [PATCH 042/105] Add name to events --- couchpotato/core/event.py | 4 +++- libs/axl/axel.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/event.py b/couchpotato/core/event.py index 88c15454..7192d39c 100644 --- a/couchpotato/core/event.py +++ b/couchpotato/core/event.py @@ -19,7 +19,7 @@ def addEvent(name, handler, priority = 100): if events.get(name): e = events[name] else: - e = events[name] = Event(threads = 20, exc_info = True, traceback = True, lock = threading.RLock()) + e = events[name] = Event(name = name, threads = 20, exc_info = True, traceback = True, lock = threading.RLock()) def createHandle(*args, **kwargs): @@ -42,6 +42,8 @@ def removeEvent(name, handler): e -= handler def fireEvent(name, *args, **kwargs): + if not events.get(name): return + log.debug('Firing event %s' % name) try: # Fire after event diff --git a/libs/axl/axel.py b/libs/axl/axel.py index 5607450d..0ab8ce32 100644 --- a/libs/axl/axel.py +++ b/libs/axl/axel.py @@ -56,7 +56,7 @@ class Event(object): >> None """ - def __init__(self, sender = None, asynch = False, exc_info = False, + def __init__(self, name = None, sender = None, asynch = False, exc_info = False, lock = None, threads = 3, traceback = False): """ Creates an event @@ -96,6 +96,7 @@ class Event(object): ) """ self.in_order = False + self.name = name self.asynchronous = asynch self.exc_info = exc_info self.lock = lock From 4309d58f9c8beb6a32c1d6d998ad85f4cdf74350 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 18 Apr 2012 01:38:27 +0200 Subject: [PATCH 043/105] Timeout event worker threads --- libs/axl/axel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/axl/axel.py b/libs/axl/axel.py index 0ab8ce32..86921a97 100644 --- a/libs/axl/axel.py +++ b/libs/axl/axel.py @@ -179,7 +179,7 @@ class Event(object): """ Executes all handlers stored in the queue """ while True: try: - h_ = self.queue.get() + h_ = self.queue.get(timeout = 2) handler, memoize, timeout = self.handlers[h_] if self.lock and self.in_order: From 981840573e35405a214ce13cae9d4362cba7fb8b Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 18 Apr 2012 01:39:37 +0200 Subject: [PATCH 044/105] Don't spam the log --- couchpotato/core/event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/event.py b/couchpotato/core/event.py index 7192d39c..82f81e21 100644 --- a/couchpotato/core/event.py +++ b/couchpotato/core/event.py @@ -43,7 +43,7 @@ def removeEvent(name, handler): def fireEvent(name, *args, **kwargs): if not events.get(name): return - log.debug('Firing event %s' % name) + #log.debug('Firing event %s' % name) try: # Fire after event From 187cd09f045012a00b3a42feaaa05756cc1606cd Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 18 Apr 2012 22:19:38 +0200 Subject: [PATCH 045/105] Property fix --- couchpotato/core/plugins/manage/main.py | 2 +- couchpotato/core/settings/__init__.py | 3 +-- couchpotato/environment.py | 5 +++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 7ce0ecad..9db06f52 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -41,7 +41,7 @@ class Manage(Plugin): def updateLibrary(self, full = True): - last_update = float(Env.prop('manage.last_update')) + last_update = float(Env.prop('manage.last_update', default = 0)) if self.isDisabled() or (last_update > time.time() - 20): return diff --git a/couchpotato/core/settings/__init__.py b/couchpotato/core/settings/__init__.py index 9c4c8fff..48c7c329 100644 --- a/couchpotato/core/settings/__init__.py +++ b/couchpotato/core/settings/__init__.py @@ -199,9 +199,8 @@ class Settings(object): db = get_session() try: prop = db.query(Properties).filter_by(identifier = identifier).first() - return prop.value + return prop.value if prop else None except: - print traceback.format_exc() return None def setProperty(self, identifier, value = ''): diff --git a/couchpotato/environment.py b/couchpotato/environment.py index ec2fbbf1..1ac184f0 100644 --- a/couchpotato/environment.py +++ b/couchpotato/environment.py @@ -62,10 +62,11 @@ class Env(object): return s @staticmethod - def prop(identifier, value = None): + def prop(identifier, value = None, default = None): s = Env.get('settings') if value == None: - return s.getProperty(identifier) + v = s.getProperty(identifier) + return v if v else default s.setProperty(identifier, value) From b808637fcf8bf3c45596f1af3f066aa60f0a087d Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 18 Apr 2012 22:28:21 +0200 Subject: [PATCH 046/105] Length of nonetype error. fixes #105 --- couchpotato/core/plugins/renamer/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 563604ca..50966417 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -200,7 +200,7 @@ class Renamer(Plugin): # rename subtitles with or without language #rename_files[current_file] = os.path.join(destination, final_folder_name, final_file_name) - sub_langs = group['subtitle_language'].get(current_file) + sub_langs = group['subtitle_language'].get(current_file, []) rename_extras = self.getRenameExtras( extra_type = 'subtitle_extra', From 10b9205e05790e16d8f9021962f6115aaa28d7f5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 18 Apr 2012 22:37:51 +0200 Subject: [PATCH 047/105] Cleanup import --- couchpotato/core/plugins/manage/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 9db06f52..d4bae232 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -1,10 +1,8 @@ -from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent, fireEventAsync from couchpotato.core.helpers.request import jsonified, getParams from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin -from couchpotato.core.settings.model import File from couchpotato.environment import Env import os import time From 994b19aa13c14b2c63a00c2f60ddf049a146bce2 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 18 Apr 2012 22:42:16 +0200 Subject: [PATCH 048/105] Quality not guessed. Don't error out, just debug log --- couchpotato/core/plugins/quality/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 63d37be8..ae33cafa 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -198,5 +198,5 @@ class QualityPlugin(Plugin): if quality: return self.setCache(hash, quality) - log.error('Could not identify quality for: %s' % files) + log.debug('Could not identify quality for: %s' % files) return self.setCache(hash, self.single('dvdrip')) From a42f3e4d9c48532bb6cae53e218bf633116b9e5f Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 18 Apr 2012 22:54:50 +0200 Subject: [PATCH 049/105] ref before assigned. fixes #106 --- couchpotato/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/runner.py b/couchpotato/runner.py index 217f52ba..66d9d7d0 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -52,7 +52,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En locale.setlocale(locale.LC_ALL, "") encoding = locale.getpreferredencoding() except (locale.Error, IOError): - pass + encoding = None # for OSes that are poorly configured I'll just force UTF-8 if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'): From c6e5dafdfce67334a309c459ce8524d9d4534568 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 19 Apr 2012 00:05:04 +0200 Subject: [PATCH 050/105] Reset failed request counter on success --- couchpotato/core/plugins/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index da779bc7..5f47a8b6 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -131,6 +131,8 @@ class Plugin(object): request = urllib2.Request(url, data, headers) data = urllib2.urlopen(request, timeout = timeout).read() + + self.http_failed_request[host] = 0 except IOError: if show_error: log.error('Failed opening url in %s: %s %s' % (self.getName(), url, traceback.format_exc(1))) From ab7bdf6d19302d3f2cdc79905a17ae1a12194b62 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 19 Apr 2012 20:51:44 +0200 Subject: [PATCH 051/105] Better movie matching quality --- couchpotato/core/plugins/scanner/main.py | 6 +++--- couchpotato/core/plugins/searcher/main.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index bf88e7f0..f7869905 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -686,7 +686,7 @@ class Scanner(Plugin): return None def findYear(self, text): - matches = re.search('(?P[0-9]{4})', text) + matches = re.search('(?P[12]{1}[0-9]{3})', text) if matches: return matches.group('year') @@ -713,7 +713,7 @@ class Scanner(Plugin): movie_name = cleaned.split(year).pop(0).strip() return { 'name': movie_name, - 'year': year, + 'year': int(year), } except: pass @@ -722,7 +722,7 @@ class Scanner(Plugin): movie_name = cleaned.split(' ').pop(0).strip() return { 'name': movie_name, - 'year': year, + 'year': int(year), } except: pass diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index b0e5297d..4e3273d6 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -250,7 +250,8 @@ class Searcher(Plugin): found[quality['identifier']] = True # Hack for older movies that don't contain quality tag - if movie_year < datetime.datetime.now().year: + year_name = fireEvent('scanner.name_year', name, single = True) + if movie_year < datetime.datetime.now().year - 3 and not year_name.get('year', None): if size > 3000: # Assume dvdr return 'dvdr' == preferred_quality['identifier'] else: # Assume dvdrip From 86dda9c30ea071084de6566374ab0c454923b9d3 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 19 Apr 2012 21:18:26 +0200 Subject: [PATCH 052/105] Add link to details in release list --- couchpotato/core/plugins/movie/static/movie.css | 2 +- couchpotato/core/plugins/movie/static/movie.js | 11 ++++++++++- couchpotato/static/images/icon.info.png | Bin 0 -> 778 bytes couchpotato/static/style/main.css | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 couchpotato/static/images/icon.info.png diff --git a/couchpotato/core/plugins/movie/static/movie.css b/couchpotato/core/plugins/movie/static/movie.css index 55dce2e3..9b82598e 100644 --- a/couchpotato/core/plugins/movie/static/movie.css +++ b/couchpotato/core/plugins/movie/static/movie.css @@ -282,7 +282,7 @@ width: 130px; } .movies .options .table .name { - width: 370px; + width: 350px; overflow: hidden; text-align: left; padding: 0 10px; diff --git a/couchpotato/core/plugins/movie/static/movie.js b/couchpotato/core/plugins/movie/static/movie.js index 83e6eb94..bbe526df 100644 --- a/couchpotato/core/plugins/movie/static/movie.js +++ b/couchpotato/core/plugins/movie/static/movie.js @@ -280,7 +280,12 @@ var ReleaseAction = new Class({ Array.each(self.movie.data.releases, function(release){ var status = Status.get(release.status_id), - quality = Quality.getProfile(release.quality_id) + quality = Quality.getProfile(release.quality_id), + info = release.info; + + try { + var details_url = info.filter(function(item){ return item.identifier == 'detail_url' }).pick().value; + } catch(e){} new Element('div', { 'class': 'item ' + status.identifier @@ -291,6 +296,10 @@ var ReleaseAction = new Class({ new Element('span.age', {'text': self.get(release, 'age')}), new Element('span.score', {'text': self.get(release, 'score')}), new Element('span.provider', {'text': self.get(release, 'provider')}), + details_url ? new Element('a.info.icon', { + 'href': details_url, + 'target': '_blank' + }) : null, new Element('a.download.icon', { 'events': { 'click': function(e){ diff --git a/couchpotato/static/images/icon.info.png b/couchpotato/static/images/icon.info.png new file mode 100644 index 0000000000000000000000000000000000000000..12cd1aef900803abba99b26920337ec01ad5c267 GIT binary patch literal 778 zcmV+l1NHogP)BVme|mWaqy4$_pJm?y9KM{-*hp?1+Ey3e-CEDooTa!B;e(Q>TSF?bj>5At13y1p zriN3w3x~5SfZj{@J4M{kp{?=M_Lh2bV+5LH)Q)5W!-ePA$RgE1@5f1cyHki0Y}JyVEYZF(LD$xXlt$7A5CgE@ zpV-&l%vf;=5kZ2-2gi@Y6J&=cuwt>!vJ^#(&n|LcZyUzi6Duj$$hJ1s*HD-#;k-w@ zpdrwAuoDG_N2bvb07G$Zk*?Hc)JLtW4yqOnic_$zO7NZ#l>Fm){;fE?b$IbOaX2fe z0la4g0Dfw2xk7Wi7NapVD8YMPCZu?A1QCK*67dgsvRKBLFtrM>?$%&_lD1882mzdO zWPdw5KWw6IT`m1b_8=lS5jt8D3=RDa=&jWzR-)S@56WMslZ~mKu1)-wpXB>rNBQ>N zU#K`#1B&v|_AQK;7I~B}OdGiUT9LX>f0xm6<;LeP!=vFjPsUQF*wCJ*dO)4YBypgdiuF!=i@6Zyi7F|q#K zz?tlSZULa@t1D?$e;f@b36&N!V2mjOHw|* .spinner, .mask{ .icon.refresh { background-image: url('../images/icon.refresh.png'); } .icon.rating { background-image: url('../images/icon.rating.png'); } .icon.files { background-image: url('../images/icon.files.png'); } +.icon.info { background-image: url('../images/icon.info.png'); } /*** Navigation ***/ .header { From 05fa253fc7771e6c28c61b93b2230c58d195e90c Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 19 Apr 2012 21:29:59 +0200 Subject: [PATCH 053/105] Only delete movie if it exists. fixes #107 --- couchpotato/core/plugins/movie/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 50042453..9d0964f5 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -365,8 +365,9 @@ class MoviePlugin(Plugin): db = get_session() movie = db.query(Movie).filter_by(id = movie_id).first() - db.delete(movie) - db.commit() + if movie: + db.delete(movie) + db.commit() return True From 20b0334b8e94f906d5ba0ef5e2190597d48c0584 Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 19 Apr 2012 21:38:35 +0200 Subject: [PATCH 054/105] Catch guessit error. fixes #108 --- couchpotato/core/plugins/scanner/main.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index f7869905..a89b980a 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -696,12 +696,15 @@ class Scanner(Plugin): # Use guessit first if file_name: - guess = guess_movie_info(file_name) - if guess.get('title') and guess.get('year'): - return { - 'name': guess.get('title'), - 'year': guess.get('year'), - } + try: + guess = guess_movie_info(file_name) + if guess.get('title') and guess.get('year'): + return { + 'name': guess.get('title'), + 'year': guess.get('year'), + } + except: + log.debug('Could not detect via guessit "%s": %s' % (file_name, traceback.format_exc())) # Backup to simple cleaned = ' '.join(re.split('\W+', simplifyString(release_name))) From 4ae91fa17a31cddd072f7e007f10e37f67871c6e Mon Sep 17 00:00:00 2001 From: Ruud Date: Thu, 19 Apr 2012 22:18:10 +0200 Subject: [PATCH 055/105] Just print error when loader didn't finish --- CouchPotato.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CouchPotato.py b/CouchPotato.py index 57f6461c..6c3a8dee 100755 --- a/CouchPotato.py +++ b/CouchPotato.py @@ -114,6 +114,7 @@ class Loader(object): if __name__ == '__main__': + l = None try: l = Loader() l.daemonize() @@ -135,7 +136,10 @@ if __name__ == '__main__': try: # if this fails we will have two tracebacks # one for failing to log, and one for the exception that got us here. - l.log.critical(traceback.format_exc()) + if l: + l.log.critical(traceback.format_exc()) + else: + print traceback.format_exc() except: print traceback.format_exc() - raise \ No newline at end of file + raise From 984ffe38bf106bb7fc83c70546f7a3a8889bba89 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 20 Apr 2012 20:56:52 +0200 Subject: [PATCH 056/105] NZBClub returning wrong url. fixes #116 --- couchpotato/core/providers/nzb/nzbclub/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/providers/nzb/nzbclub/main.py b/couchpotato/core/providers/nzb/nzbclub/main.py index 07ae2c2d..1e2b7d98 100644 --- a/couchpotato/core/providers/nzb/nzbclub/main.py +++ b/couchpotato/core/providers/nzb/nzbclub/main.py @@ -68,8 +68,8 @@ class NZBClub(NZBProvider, RSS): 'name': toUnicode(self.getTextElement(nzb, "title")), 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))), 'size': tryInt(size) / 1024 / 1024, - 'url': enclosure['url'], - 'download': enclosure['url'].replace(' ', '_'), + 'url': enclosure['url'].replace(' ', '_'), + 'download': self.download, 'detail_url': self.getTextElement(nzb, "link"), 'description': description, } From 7dfee80afc5d2d030497046d16b3fc4156c89e09 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 20 Apr 2012 21:25:56 +0200 Subject: [PATCH 057/105] Add some logging --- couchpotato/core/plugins/searcher/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 4e3273d6..030e5459 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -73,6 +73,8 @@ class Searcher(Plugin): quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True) results = fireEvent('yarr.search', movie, quality, merge = True) sorted_results = sorted(results, key = lambda k: k['score'], reverse = True) + if len(sorted_results) == 0: + log.debug('Nothing found for %s in %s' % (default_title, quality_type['quality']['label'])) # Add them to this movie releases list for nzb in sorted_results: From cb774bc31bc1fde67fe217a13bab3a612a3a9366 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 21 Apr 2012 16:39:34 +0200 Subject: [PATCH 058/105] Refactored updater to make git-less updating possible. closes #78 --- couchpotato/core/_base/updater/main.py | 316 ++++++++++++++++++------- 1 file changed, 237 insertions(+), 79 deletions(-) diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index 8bf26359..97210123 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -4,9 +4,13 @@ from couchpotato.core.helpers.request import jsonified from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env -from git.repository import LocalRepository from datetime import datetime +from dateutil.parser import parse +from git.repository import LocalRepository +import json import os +import shutil +import tarfile import time import traceback @@ -15,19 +19,14 @@ log = CPLog(__name__) class Updater(Plugin): - repo_name = 'RuudBurger/CouchPotatoServer' - - version = None - update_failed = False - update_version = None - last_check = 0 - def __init__(self): - self.repo = LocalRepository(Env.get('app_dir'), command = self.conf('git_command', default = 'git')) + if os.path.isdir(os.path.join(Env.get('app_dir'), '.git')): + self.updater = GitUpdater(self.conf('git_command', default = 'git')) + else: + self.updater = SourceUpdater() fireEvent('schedule.interval', 'updater.check', self.check, hours = 6) - addEvent('app.load', self.check) addApiView('updater.info', self.getInfo, docs = { @@ -35,7 +34,6 @@ class Updater(Plugin): 'return': { 'type': 'object', 'example': """{ - 'repo_name': "Name of used repository", 'last_check': "last checked for update", 'update_version': "available update version or empty", 'version': current_cp_version @@ -47,75 +45,88 @@ class Updater(Plugin): 'return': {'type': 'see updater.info'} }) - def getInfo(self): + def check(self): + if self.isDisabled(): + return + if self.updater.check(): + if self.conf('automatic') and not self.updater.update_failed: + if self.updater.doUpdate(): + fireEventAsync('app.crappy_restart') + else: + if self.conf('notification'): + fireEvent('updater.available', message = 'A new update is available', data = self.updater.getVersion()) + + def getInfo(self): + return jsonified(self.updater.info()) + + def checkView(self): + self.check() + return self.updater.getInfo() + + def doUpdateView(self): + return jsonified({ + 'success': self.updater.doUpdate() + }) + + +class BaseUpdater(Plugin): + + repo_user = 'RuudBurger' + repo_name = 'CouchPotatoServer' + branch = 'develop' + + version = None + update_failed = False + update_version = None + last_check = 0 + + def doUpdate(self): + pass + + def getInfo(self): return jsonified(self.info()) def info(self): return { - 'repo_name': self.repo_name, 'last_check': self.last_check, 'update_version': self.update_version, 'version': self.getVersion() } - def getVersion(self): - - if not self.version: - try: - output = self.repo.getHead() # Yes, please - log.debug('Git version output: %s' % output.hash) - self.version = { - 'hash': output.hash[:8], - 'date': output.getDate(), - } - except Exception, e: - log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s' % e) - return 'No GIT' - - return self.version - def check(self): + pass - if self.update_version or self.isDisabled(): - return + def deletePyc(self, only_excess = True): - log.info('Checking for new version on github for %s' % self.repo_name) - if not Env.get('dev'): - self.repo.fetch() + for root, dirs, files in os.walk(Env.get('app_dir')): - current_branch = self.repo.getCurrentBranch().name + pyc_files = filter(lambda filename: filename.endswith('.pyc'), files) + py_files = set(filter(lambda filename: filename.endswith('.py'), files)) + excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files - for branch in self.repo.getRemoteByName('origin').getBranches(): - if current_branch == branch.name: + for excess_pyc_file in excess_pyc_files: + full_path = os.path.join(root, excess_pyc_file) + log.debug('Removing old PYC file: %s' % full_path) + try: + os.remove(full_path) + except: + log.error('Couldn\'t remove %s: %s' % (full_path, traceback.format_exc())) - local = self.repo.getHead() - remote = branch.getHead() + for dir_name in dirs: + full_path = os.path.join(root, dir_name) + if len(os.listdir(full_path)) == 0: + try: + os.rmdir(full_path) + except: + log.error('Couldn\'t remove empty directory %s: %s' % (full_path, traceback.format_exc())) - log.info('Versions, local:%s, remote:%s' % (local.hash[:8], remote.hash[:8])) - if local.getDate() < remote.getDate(): - self.update_version = { - 'hash': remote.hash[:8], - 'date': remote.getDate(), - } - if self.conf('automatic') and not self.update_failed: - if self.doUpdate(): - fireEventAsync('app.crappy_restart') - else: - if self.conf('notification'): - fireEvent('updater.available', message = 'A new update is available', data = self.getVersion()) - self.last_check = time.time() +class GitUpdater(BaseUpdater): - def checkView(self): - self.check() - return self.getInfo() - - def doUpdateView(self): - return jsonified({ - 'success': self.doUpdate() - }) + def __init__(self, git_command): + self.repo = LocalRepository(Env.get('app_dir'), command = git_command) def doUpdate(self): try: @@ -141,29 +152,176 @@ class Updater(Plugin): return False - def deletePyc(self): + def getVersion(self): - for root, dirs, files in os.walk(Env.get('app_dir')): + if not self.version: + try: + output = self.repo.getHead() # Yes, please + log.debug('Git version output: %s' % output.hash) + self.version = { + 'hash': output.hash[:8], + 'date': output.getDate(), + } + except Exception, e: + log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s' % e) + return 'No GIT' - pyc_files = filter(lambda filename: filename.endswith('.pyc'), files) - py_files = set(filter(lambda filename: filename.endswith('.py'), files)) - excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) + return self.version - for excess_pyc_file in excess_pyc_files: - full_path = os.path.join(root, excess_pyc_file) - log.debug('Removing old PYC file: %s' % full_path) - try: - os.remove(full_path) - except: - log.error('Couldn\'t remove %s: %s' % (full_path, traceback.format_exc())) + def check(self): - for dir_name in dirs: - full_path = os.path.join(root, dir_name) - if len(os.listdir(full_path)) == 0: + if self.update_version: + return + + log.info('Checking for new version on github for %s' % self.repo_name) + if not Env.get('dev'): + self.repo.fetch() + + current_branch = self.repo.getCurrentBranch().name + + for branch in self.repo.getRemoteByName('origin').getBranches(): + if current_branch == branch.name: + + local = self.repo.getHead() + remote = branch.getHead() + + log.info('Versions, local:%s, remote:%s' % (local.hash[:8], remote.hash[:8])) + + if local.getDate() < remote.getDate(): + self.update_version = { + 'hash': remote.hash[:8], + 'date': remote.getDate(), + } + return True + + self.last_check = time.time() + return False + + + +class SourceUpdater(BaseUpdater): + + def __init__(self): + + # Create version file in cache + self.version_file = os.path.join(Env.get('cache_dir'), 'version') + if not os.path.isfile(self.version_file): + self.createFile(self.version_file, json.dumps(self.latestCommit())) + + def doUpdate(self): + + try: + url = 'https://github.com/%s/%s/tarball/%s' % (self.repo_user, self.repo_name, self.branch) + destination = os.path.join(Env.get('cache_dir'), self.update_version.get('hash') + '.tar.gz') + extracted_path = os.path.join(Env.get('cache_dir'), 'temp_updater') + + destination = fireEvent('file.download', url = url, dest = destination, single = True) + + # Cleanup leftover from last time + if os.path.isdir(extracted_path): + self.removeDir(extracted_path) + self.makeDir(extracted_path) + + # Extract + tar = tarfile.open(destination) + tar.extractall(path = extracted_path) + os.remove(destination) + + self.replaceWith(os.path.join(extracted_path, os.listdir(extracted_path)[0])) + self.removeDir(extracted_path) + + # Write update version to file + self.createFile(self.version_file, json.dumps(self.update_version)) + + return True + except: + log.error('Failed updating: %s' % traceback.format_exc()) + + self.update_failed = True + return False + + def replaceWith(self, path): + app_dir = Env.get('app_dir') + + # Get list of files we want to overwrite + self.deletePyc(only_excess = False) + existing_files = [] + for root, subfiles, filenames in os.walk(app_dir): + for filename in filenames: + existing_files.append(os.path.join(root, filename)) + + for root, subfiles, filenames in os.walk(path): + for filename in filenames: + fromfile = os.path.join(root, filename) + tofile = os.path.join(app_dir, fromfile.replace(path + os.path.sep, '')) + + if not Env.get('dev'): try: - os.rmdir(full_path) + os.remove(tofile) except: - log.error('Couldn\'t remove empty directory %s: %s' % (full_path, traceback.format_exc())) + pass - def isEnabled(self): - return super(Updater, self).isEnabled() and Env.get('uses_git') + try: + os.renames(fromfile, tofile) + try: + existing_files.remove(tofile) + except ValueError: + pass + except Exception, e: + log.error('Failed overwriting file: %s' % e) + + + def removeDir(self, path): + try: + if os.path.isdir(path): + shutil.rmtree(path) + except OSError, inst: + os.chmod(inst.filename, 0777) + self.removeDir(path) + + def getVersion(self): + + if not self.version: + try: + f = open(self.version_file, 'r') + output = json.loads(f.read()) + f.close() + + log.debug('Source version output: %s' % output) + self.version = output + except Exception, e: + log.error('Failed using source updater. %s' % e) + return {} + + return self.version + + def check(self): + + current_version = self.getVersion() + + try: + latest = self.latestCommit() + + if latest.get('hash') != current_version.get('hash') and latest.get('date') >= current_version.get('date'): + self.update_version = latest + + self.last_check = time.time() + except: + log.error('Failed updating via source: %s' % traceback.format_exc()) + + return self.update_version is not None + + def latestCommit(self): + try: + url = 'https://api.github.com/repos/%s/%s/commits?per_page=1&sha=%s' % (self.repo_user, self.repo_name, self.branch) + data = self.getCache('github.commit', url = url) + commit = json.loads(data)[0] + + return { + 'hash': commit['sha'], + 'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())), + } + except: + log.error('Failed getting latest request from github: %s' % traceback.format_exc()) + + return {} From 1718d3e706c1ba3863bf164fd1759b01b98c3824 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 21 Apr 2012 16:43:41 +0200 Subject: [PATCH 059/105] Don't search when name and year can't be found. fixes #125 --- couchpotato/core/plugins/scanner/main.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index a89b980a..7bc726c0 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -474,12 +474,15 @@ class Scanner(Plugin): if len(identifier) > 2: try: filename = list(group['files'].get('movie'))[0] except: filename = None - movie = fireEvent('movie.search', q = '%(name)s %(year)s' % self.getReleaseNameYear(identifier, file_name = filename), merge = True, limit = 1) - if len(movie) > 0: - imdb_id = movie[0]['imdb'] - log.debug('Found movie via search: %s' % cur_file) - if imdb_id: break + name_year = self.getReleaseNameYear(identifier, file_name = filename) + if name_year.get('name') and name_year.get('year'): + movie = fireEvent('movie.search', q = '%(name)s %(year)s' % name_year, merge = True, limit = 1) + + if len(movie) > 0: + imdb_id = movie[0]['imdb'] + log.debug('Found movie via search: %s' % cur_file) + if imdb_id: break else: log.debug('Identifier to short to use for search: %s' % identifier) From 8b80c50d894dbb7831d0ee60088a4c698d8b5e84 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 21 Apr 2012 16:50:46 +0200 Subject: [PATCH 060/105] Remove empty metadata creators for now. closes #122 --- .../metadata/mediabrowser/__init__.py | 41 ------------------- .../providers/metadata/mediabrowser/main.py | 5 --- .../providers/metadata/sonyps3/__init__.py | 31 -------------- .../core/providers/metadata/sonyps3/main.py | 5 --- .../core/providers/metadata/wdtv/__init__.py | 30 -------------- .../core/providers/metadata/wdtv/main.py | 5 --- 6 files changed, 117 deletions(-) delete mode 100644 couchpotato/core/providers/metadata/mediabrowser/__init__.py delete mode 100644 couchpotato/core/providers/metadata/mediabrowser/main.py delete mode 100644 couchpotato/core/providers/metadata/sonyps3/__init__.py delete mode 100644 couchpotato/core/providers/metadata/sonyps3/main.py delete mode 100644 couchpotato/core/providers/metadata/wdtv/__init__.py delete mode 100644 couchpotato/core/providers/metadata/wdtv/main.py diff --git a/couchpotato/core/providers/metadata/mediabrowser/__init__.py b/couchpotato/core/providers/metadata/mediabrowser/__init__.py deleted file mode 100644 index 2e3cc202..00000000 --- a/couchpotato/core/providers/metadata/mediabrowser/__init__.py +++ /dev/null @@ -1,41 +0,0 @@ -from .main import MediaBrowser - -def start(): - return MediaBrowser() - -""" -config = [{ - 'name': 'mediabrowser', - 'groups': [ - { - 'tab': 'renamer', - 'subtab': 'metadata', - 'name': 'mediabrowser_metadata', - 'label': 'MediaBrowser', - 'description': 'Enable metadata MediaBrowser can understand', - 'options': [ - { - 'name': 'meta_enabled', - 'default': False, - 'type': 'enabler', - }, - { - 'name': 'meta_nfo', - 'default': True, - 'type': 'bool', - }, - { - 'name': 'meta_fanart', - 'default': True, - 'type': 'bool', - }, - { - 'name': 'meta_thumbnail', - 'default': True, - 'type': 'bool', - }, - ], - }, - ], -}] -""" diff --git a/couchpotato/core/providers/metadata/mediabrowser/main.py b/couchpotato/core/providers/metadata/mediabrowser/main.py deleted file mode 100644 index 52a83d00..00000000 --- a/couchpotato/core/providers/metadata/mediabrowser/main.py +++ /dev/null @@ -1,5 +0,0 @@ -from couchpotato.core.providers.metadata.base import MetaDataBase - -class MediaBrowser(MetaDataBase): - - pass diff --git a/couchpotato/core/providers/metadata/sonyps3/__init__.py b/couchpotato/core/providers/metadata/sonyps3/__init__.py deleted file mode 100644 index c6a7b49e..00000000 --- a/couchpotato/core/providers/metadata/sonyps3/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -from .main import SonyPS3 - -def start(): - return SonyPS3() - -""" -config = [{ - 'name': 'sonyps3', - 'groups': [ - { - 'tab': 'renamer', - 'subtab': 'metadata', - 'name': 'sonyps3_metadata', - 'label': 'Sony PS3', - 'description': 'Enable metadata your Playstation 3 can understand', - 'options': [ - { - 'name': 'meta_enabled', - 'default': False, - 'type': 'enabler', - }, - { - 'name': 'meta_thumbnail', - 'default': True, - 'type': 'bool', - }, - ], - }, - ], -}] -""" diff --git a/couchpotato/core/providers/metadata/sonyps3/main.py b/couchpotato/core/providers/metadata/sonyps3/main.py deleted file mode 100644 index 58fdd7fd..00000000 --- a/couchpotato/core/providers/metadata/sonyps3/main.py +++ /dev/null @@ -1,5 +0,0 @@ -from couchpotato.core.providers.metadata.base import MetaDataBase - -class SonyPS3(MetaDataBase): - - pass diff --git a/couchpotato/core/providers/metadata/wdtv/__init__.py b/couchpotato/core/providers/metadata/wdtv/__init__.py deleted file mode 100644 index 9be6c750..00000000 --- a/couchpotato/core/providers/metadata/wdtv/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -from .main import WDTV - -def start(): - return WDTV() -""" -config = [{ - 'name': 'wdtv', - 'groups': [ - { - 'tab': 'renamer', - 'subtab': 'metadata', - 'name': 'wdtv_metadata', - 'label': 'WDTV', - 'description': 'Enable metadata WDTV can understand', - 'options': [ - { - 'name': 'meta_enabled', - 'default': False, - 'type': 'enabler', - }, - { - 'name': 'meta_thumbnail', - 'default': True, - 'type': 'bool', - }, - ], - }, - ], -}] -""" diff --git a/couchpotato/core/providers/metadata/wdtv/main.py b/couchpotato/core/providers/metadata/wdtv/main.py deleted file mode 100644 index 74322a0b..00000000 --- a/couchpotato/core/providers/metadata/wdtv/main.py +++ /dev/null @@ -1,5 +0,0 @@ -from couchpotato.core.providers.metadata.base import MetaDataBase - -class WDTV(MetaDataBase): - - pass From fd462514c3723bbd9f0d6b1e6b4b0f4aaca1bc92 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 21 Apr 2012 20:55:21 +0200 Subject: [PATCH 061/105] Use custom urlencode function. fixes #121 --- couchpotato/core/downloaders/sabnzbd/main.py | 4 ++-- couchpotato/core/helpers/encoding.py | 21 +++++++++++++++++-- couchpotato/core/helpers/request.py | 7 ++++--- couchpotato/core/notifications/nmj/main.py | 5 ++--- couchpotato/core/notifications/prowl/main.py | 5 ++--- .../core/notifications/twitter/main.py | 4 ++-- couchpotato/core/notifications/xbmc/main.py | 4 ++-- couchpotato/core/plugins/base.py | 6 ++---- couchpotato/core/plugins/library/main.py | 10 +++++---- couchpotato/core/plugins/movie/main.py | 15 +++++++------ .../core/providers/movie/imdbapi/main.py | 7 +++++-- couchpotato/core/providers/nzb/moovee/main.py | 4 ++-- .../core/providers/nzb/mysterbin/main.py | 7 +++---- .../core/providers/nzb/newzbin/main.py | 4 ++-- .../core/providers/nzb/newznab/main.py | 6 +++--- .../core/providers/nzb/nzbclub/main.py | 8 +++---- .../core/providers/nzb/nzbindex/main.py | 7 +++---- .../core/providers/nzb/nzbmatrix/main.py | 4 ++-- couchpotato/core/providers/nzb/nzbs/main.py | 5 ++--- couchpotato/core/providers/nzb/x264/main.py | 6 +++--- .../providers/torrent/kickasstorrents/main.py | 2 +- .../core/providers/trailer/hdtrailers/main.py | 4 ++-- 22 files changed, 79 insertions(+), 66 deletions(-) diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 41ac9206..0319f68d 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -1,9 +1,9 @@ from couchpotato.core.downloaders.base import Downloader +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog from inspect import isfunction from tempfile import mkstemp -from urllib import urlencode import base64 import os import re @@ -58,7 +58,7 @@ class Sabnzbd(Downloader): if pp: params['script'] = pp_script_fn - url = cleanHost(self.conf('host')) + "api?" + urlencode(params) + url = cleanHost(self.conf('host')) + "api?" + tryUrlencode(params) try: if params.get('mode') is 'addfile': diff --git a/couchpotato/core/helpers/encoding.py b/couchpotato/core/helpers/encoding.py index d252364f..31a85db0 100644 --- a/couchpotato/core/helpers/encoding.py +++ b/couchpotato/core/helpers/encoding.py @@ -1,5 +1,6 @@ from couchpotato.core.logger import CPLog from string import ascii_letters, digits +from urllib import quote_plus import re import unicodedata @@ -19,7 +20,7 @@ def simplifyString(original): def toUnicode(original, *args): try: - if type(original) is unicode: + if isinstance(original, unicode): return original else: try: @@ -35,7 +36,7 @@ def toUnicode(original, *args): return toUnicode(ascii_text) def ek(original, *args): - if type(original) in [str, unicode]: + if isinstance(original, (str, unicode)): try: from couchpotato.environment import Env return original.decode(Env.get('encoding')) @@ -53,3 +54,19 @@ def isInt(value): def stripAccents(s): return ''.join((c for c in unicodedata.normalize('NFD', toUnicode(s)) if unicodedata.category(c) != 'Mn')) + +def tryUrlencode(s): + new = u'' + if isinstance(s, (dict)): + for key, value in s.iteritems(): + new += u'&%s=%s' % (key, tryUrlencode(value)) + + return new[1:] + else: + for letter in toUnicode(s): + try: + new += quote_plus(letter) + except: + new += letter + + return new diff --git a/couchpotato/core/helpers/request.py b/couchpotato/core/helpers/request.py index a9dff599..72c9b9ed 100644 --- a/couchpotato/core/helpers/request.py +++ b/couchpotato/core/helpers/request.py @@ -1,3 +1,4 @@ +from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.variable import natcmp from flask.globals import current_app from flask.helpers import json @@ -25,7 +26,7 @@ def getParams(): for item in nested: if item is nested[-1]: - current[item] = unquote_plus(value) + current[item] = toUnicode(unquote_plus(value)).encode('utf-8') else: try: current[item] @@ -34,7 +35,7 @@ def getParams(): current = current[item] else: - temp[param] = unquote_plus(value) + temp[param] = toUnicode(unquote_plus(value)).encode('utf-8') return dictToList(temp) @@ -56,7 +57,7 @@ def dictToList(params): def getParam(attr, default = None): try: - return unquote_plus(getattr(flask.request, 'args').get(attr, default)) + return toUnicode(unquote_plus(getattr(flask.request, 'args').get(attr, default))).encode('utf-8') except: return None diff --git a/couchpotato/core/notifications/nmj/main.py b/couchpotato/core/notifications/nmj/main.py index 1ac8d70a..89aa8729 100644 --- a/couchpotato/core/notifications/nmj/main.py +++ b/couchpotato/core/notifications/nmj/main.py @@ -1,12 +1,11 @@ from couchpotato.api import addApiView from couchpotato.core.event import addEvent +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.request import getParams, jsonified from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification -from couchpotato.environment import Env import re import telnetlib -import urllib try: import xml.etree.cElementTree as etree @@ -90,7 +89,7 @@ class NMJ(Notification): 'arg2': 'background', 'arg3': '', } - params = urllib.urlencode(params) + params = tryUrlencode(params) UPDATE_URL = 'http://%(host)s:8008/metadata_database?%(params)s' updateUrl = UPDATE_URL % {'host': host, 'params': params} diff --git a/couchpotato/core/notifications/prowl/main.py b/couchpotato/core/notifications/prowl/main.py index 966ffae2..44aaa9c5 100644 --- a/couchpotato/core/notifications/prowl/main.py +++ b/couchpotato/core/notifications/prowl/main.py @@ -1,8 +1,7 @@ -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from httplib import HTTPSConnection -from urllib import urlencode log = CPLog(__name__) @@ -24,7 +23,7 @@ class Prowl(Notification): http_handler.request('POST', '/publicapi/add', headers = {'Content-type': 'application/x-www-form-urlencoded'}, - body = urlencode(data) + body = tryUrlencode(data) ) response = http_handler.getresponse() request_status = response.status diff --git a/couchpotato/core/notifications/twitter/main.py b/couchpotato/core/notifications/twitter/main.py index dae428e9..700f7fa3 100644 --- a/couchpotato/core/notifications/twitter/main.py +++ b/couchpotato/core/notifications/twitter/main.py @@ -1,4 +1,5 @@ from couchpotato.api import addApiView +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.request import jsonified, getParam from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog @@ -7,7 +8,6 @@ from flask.helpers import url_for from pytwitter import Api, parse_qsl from werkzeug.utils import redirect import oauth2 -import urllib log = CPLog(__name__) @@ -56,7 +56,7 @@ class Twitter(Notification): oauth_consumer = oauth2.Consumer(self.consumer_key, self.consumer_secret) oauth_client = oauth2.Client(oauth_consumer) - resp, content = oauth_client.request(self.urls['request'], 'POST', body = urllib.urlencode({'oauth_callback': callback_url})) + resp, content = oauth_client.request(self.urls['request'], 'POST', body = tryUrlencode({'oauth_callback': callback_url})) if resp['status'] != '200': log.error('Invalid response from Twitter requesting temp token: %s' % resp['status']) diff --git a/couchpotato/core/notifications/xbmc/main.py b/couchpotato/core/notifications/xbmc/main.py index 2651caec..cbc81df2 100644 --- a/couchpotato/core/notifications/xbmc/main.py +++ b/couchpotato/core/notifications/xbmc/main.py @@ -1,7 +1,7 @@ +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification import base64 -import urllib log = CPLog(__name__) @@ -21,7 +21,7 @@ class XBMC(Notification): def send(self, command, host): - url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, urllib.urlencode(command)) + url = 'http://%s/xbmcCmds/xbmcHttp/?%s' % (host, tryUrlencode(command)) headers = {} if self.conf('password'): diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 5f47a8b6..bd3e066a 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -1,5 +1,6 @@ from couchpotato import addView from couchpotato.core.event import fireEvent, addEvent +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import getExt from couchpotato.core.logger import CPLog from couchpotato.environment import Env @@ -11,10 +12,8 @@ import glob import math import os.path import re -import socket import time import traceback -import urllib import urllib2 log = CPLog(__name__) @@ -114,7 +113,6 @@ class Plugin(object): del self.http_failed_disabled[host] self.wait(host) - try: if multipart: @@ -127,7 +125,7 @@ class Plugin(object): data = opener.open(request, timeout = timeout).read() else: log.info('Opening url: %s, params: %s' % (url, [x for x in params.iterkeys()])) - data = urllib.urlencode(params) if len(params) > 0 else None + data = tryUrlencode(params) if len(params) > 0 else None request = urllib2.Request(url, data, headers) data = urllib2.urlopen(request, timeout = timeout).read() diff --git a/couchpotato/core/plugins/library/main.py b/couchpotato/core/plugins/library/main.py index 9a4cde04..8d6c3192 100644 --- a/couchpotato/core/plugins/library/main.py +++ b/couchpotato/core/plugins/library/main.py @@ -1,6 +1,7 @@ from couchpotato import get_session from couchpotato.core.event import addEvent, fireEventAsync, fireEvent -from couchpotato.core.helpers.encoding import toUnicode, simplifyString +from couchpotato.core.helpers.encoding import toUnicode, simplifyString, \ + tryUrlencode from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, LibraryTitle, File @@ -45,7 +46,7 @@ class LibraryPlugin(Plugin): # Update library info if update_after is not False: handle = fireEventAsync if update_after is 'async' else fireEvent - handle('library.update', identifier = l.identifier, default_title = attrs.get('title', '')) + handle('library.update', identifier = l.identifier, default_title = toUnicode(attrs.get('title', ''))) return l.to_dict(self.default_dict) @@ -86,10 +87,11 @@ class LibraryPlugin(Plugin): for title in titles: if not title: continue + title = toUnicode(title) t = LibraryTitle( - title = toUnicode(title), + title = title, simple_title = self.simplifyTitle(title), - default = title.lower() == default_title.lower() or (default_title is '' and titles[0] == title) + default = title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == u'' and toUnicode(titles[0]) == title) ) library.titles.append(t) diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 9d0964f5..28332dca 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -1,8 +1,9 @@ from couchpotato import get_session from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, fireEventAsync, addEvent -from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.helpers.request import getParams, jsonified +from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \ + simplifyString +from couchpotato.core.helpers.request import getParams, jsonified, getParam from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Movie, Library, LibraryTitle @@ -10,7 +11,6 @@ from couchpotato.environment import Env from sqlalchemy.orm import joinedload_all from sqlalchemy.sql.expression import or_, asc, not_ from string import ascii_lowercase -from urllib import urlencode log = CPLog(__name__) @@ -225,7 +225,6 @@ class MoviePlugin(Plugin): if title.default: default_title = title.title if movie: - #addEvent('library.update.after', ) fireEventAsync('library.update', identifier = movie.library.identifier, default_title = default_title, force = True) fireEventAsync('searcher.single', movie.to_dict(self.default_dict)) @@ -235,12 +234,12 @@ class MoviePlugin(Plugin): def search(self): - params = getParams() - cache_key = '%s/%s' % (__name__, urlencode(params)) + q = getParam('q') + cache_key = u'%s/%s' % (__name__, simplifyString(q)) movies = Env.get('cache').get(cache_key) if not movies: - movies = fireEvent('movie.search', q = params.get('q'), merge = True) + movies = fireEvent('movie.search', q = q, merge = True) Env.get('cache').set(cache_key, movies) return jsonified({ @@ -335,7 +334,7 @@ class MoviePlugin(Plugin): # Default title if params.get('default_title'): for title in m.library.titles: - title.default = params.get('default_title').lower() == title.title.lower() + title.default = toUnicode(params.get('default_title', '')).lower() == toUnicode(title.title).lower() db.commit() diff --git a/couchpotato/core/providers/movie/imdbapi/main.py b/couchpotato/core/providers/movie/imdbapi/main.py index e9077687..bde256a9 100644 --- a/couchpotato/core/providers/movie/imdbapi/main.py +++ b/couchpotato/core/providers/movie/imdbapi/main.py @@ -1,8 +1,8 @@ from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import tryInt, tryFloat from couchpotato.core.logger import CPLog from couchpotato.core.providers.movie.base import MovieProvider -from urllib import urlencode import json import re import traceback @@ -27,8 +27,11 @@ class IMDBAPI(MovieProvider): name_year = fireEvent('scanner.name_year', q, single = True) + if not name_year.get('name'): + return [] + cache_key = 'imdbapi.cache.%s' % q - cached = self.getCache(cache_key, self.urls['search'] % urlencode({'t': name_year.get('name'), 'y': name_year.get('year')})) + cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')})) if cached: result = self.parseMovie(cached) diff --git a/couchpotato/core/providers/nzb/moovee/main.py b/couchpotato/core/providers/nzb/moovee/main.py index a6353bc9..a6aeb32a 100644 --- a/couchpotato/core/providers/nzb/moovee/main.py +++ b/couchpotato/core/providers/nzb/moovee/main.py @@ -1,8 +1,8 @@ from couchpotato.core.event import fireEvent +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider from dateutil.parser import parse -from urllib import quote_plus import re import time @@ -27,7 +27,7 @@ class Moovee(NZBProvider): return results q = '%s %s' % (movie['library']['titles'][0]['title'], quality.get('identifier')) - url = self.urls['search'] % quote_plus(q) + url = self.urls['search'] % tryUrlencode(q) cache_key = 'moovee.%s' % q data = self.getCache(cache_key, url) diff --git a/couchpotato/core/providers/nzb/mysterbin/main.py b/couchpotato/core/providers/nzb/mysterbin/main.py index d92595f6..7c3f56bc 100644 --- a/couchpotato/core/providers/nzb/mysterbin/main.py +++ b/couchpotato/core/providers/nzb/mysterbin/main.py @@ -1,11 +1,10 @@ from BeautifulSoup import BeautifulSoup from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env -import urllib log = CPLog(__name__) @@ -40,8 +39,8 @@ class Mysterbin(NZBProvider): 'nopasswd': 'on', } - cache_key = 'mysterbin.%s' % q - data = self.getCache(cache_key, self.urls['search'] % urllib.urlencode(params)) + cache_key = 'mysterbin.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) + data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params)) if data: try: diff --git a/couchpotato/core/providers/nzb/newzbin/main.py b/couchpotato/core/providers/nzb/newzbin/main.py index 96a34a3f..73b4a733 100644 --- a/couchpotato/core/providers/nzb/newzbin/main.py +++ b/couchpotato/core/providers/nzb/newzbin/main.py @@ -1,9 +1,9 @@ from couchpotato.core.event import fireEvent +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider from dateutil.parser import parse -from urllib import urlencode import base64 import time import xml.etree.ElementTree as XMLTree @@ -45,7 +45,7 @@ class Newzbin(NZBProvider, RSS): format_id = self.getFormatId(type) cat_id = self.getCatId(type) - arguments = urlencode({ + arguments = tryUrlencode({ 'searchaction': 'Search', 'u_url_posts_only': '0', 'u_show_passworded': '0', diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index c3bb6b2b..5ec67432 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -1,10 +1,10 @@ from couchpotato.core.event import fireEvent +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import cleanHost from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider from dateutil.parser import parse -from urllib import urlencode import time import xml.etree.ElementTree as XMLTree @@ -48,7 +48,7 @@ class Newznab(NZBProvider, RSS): if self.isDisabled(host) or not self.isAvailable(self.getUrl(host['host'], self.urls['search'])): return results - arguments = urlencode({ + arguments = tryUrlencode({ 't': self.cat_backup_id, 'r': host['api_key'], 'i': 58, @@ -81,7 +81,7 @@ class Newznab(NZBProvider, RSS): return results cat_id = self.getCatId(quality['identifier']) - arguments = urlencode({ + arguments = tryUrlencode({ 'imdbid': movie['library']['identifier'].replace('tt', ''), 'cat': cat_id[0], 'apikey': host['api_key'], diff --git a/couchpotato/core/providers/nzb/nzbclub/main.py b/couchpotato/core/providers/nzb/nzbclub/main.py index 1e2b7d98..283b29fc 100644 --- a/couchpotato/core/providers/nzb/nzbclub/main.py +++ b/couchpotato/core/providers/nzb/nzbclub/main.py @@ -1,6 +1,5 @@ -from BeautifulSoup import BeautifulSoup, SoupStrainer from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog @@ -8,7 +7,6 @@ from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env from dateutil.parser import parse import time -import urllib import xml.etree.ElementTree as XMLTree log = CPLog(__name__) @@ -41,8 +39,8 @@ class NZBClub(NZBProvider, RSS): 'ns': 1, } - cache_key = 'nzbclub.%s' % q - data = self.getCache(cache_key, self.urls['search'] % urllib.urlencode(params)) + cache_key = 'nzbclub.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) + data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params)) if data: try: try: diff --git a/couchpotato/core/providers/nzb/nzbindex/main.py b/couchpotato/core/providers/nzb/nzbindex/main.py index 1f56a299..22e202b6 100644 --- a/couchpotato/core/providers/nzb/nzbindex/main.py +++ b/couchpotato/core/providers/nzb/nzbindex/main.py @@ -1,12 +1,11 @@ from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env from dateutil.parser import parse -from urllib import urlencode import re import time import xml.etree.ElementTree as XMLTree @@ -30,7 +29,7 @@ class NzbIndex(NZBProvider, RSS): return results q = '%s %s %s' % (movie['library']['titles'][0]['title'], movie['library']['year'], quality.get('identifier')) - arguments = urlencode({ + arguments = tryUrlencode({ 'q': q, 'age': Env.setting('retention', 'nzb'), 'sort': 'agedesc', @@ -43,7 +42,7 @@ class NzbIndex(NZBProvider, RSS): }) url = "%s?%s" % (self.urls['api'], arguments) - cache_key = 'nzbindex.%s' % q + cache_key = 'nzbindex.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) data = self.getCache(cache_key, url) if data: diff --git a/couchpotato/core/providers/nzb/nzbmatrix/main.py b/couchpotato/core/providers/nzb/nzbmatrix/main.py index 43b95873..4e26488b 100644 --- a/couchpotato/core/providers/nzb/nzbmatrix/main.py +++ b/couchpotato/core/providers/nzb/nzbmatrix/main.py @@ -1,10 +1,10 @@ from couchpotato.core.event import fireEvent +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider from couchpotato.environment import Env from dateutil.parser import parse -from urllib import urlencode import time import xml.etree.ElementTree as XMLTree @@ -37,7 +37,7 @@ class NZBMatrix(NZBProvider, RSS): cat_ids = ','.join(['%s' % x for x in self.getCatId(quality.get('identifier'))]) - arguments = urlencode({ + arguments = tryUrlencode({ 'term': movie['library']['identifier'], 'subcat': cat_ids, 'username': self.conf('username'), diff --git a/couchpotato/core/providers/nzb/nzbs/main.py b/couchpotato/core/providers/nzb/nzbs/main.py index a7e8fc66..4a8d9a06 100644 --- a/couchpotato/core/providers/nzb/nzbs/main.py +++ b/couchpotato/core/providers/nzb/nzbs/main.py @@ -1,10 +1,9 @@ from couchpotato.core.event import fireEvent -from couchpotato.core.helpers.encoding import simplifyString +from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode from couchpotato.core.helpers.rss import RSS from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider from dateutil.parser import parse -from urllib import urlencode import time import xml.etree.ElementTree as XMLTree @@ -36,7 +35,7 @@ class Nzbs(NZBProvider, RSS): return results cat_id = self.getCatId(quality.get('identifier')) - arguments = urlencode({ + arguments = tryUrlencode({ 'action':'search', 'q': simplifyString(movie['library']['titles'][0]['title']), 'catid': cat_id[0], diff --git a/couchpotato/core/providers/nzb/x264/main.py b/couchpotato/core/providers/nzb/x264/main.py index 0db9eabc..d1b9f727 100644 --- a/couchpotato/core/providers/nzb/x264/main.py +++ b/couchpotato/core/providers/nzb/x264/main.py @@ -1,8 +1,8 @@ from couchpotato.core.event import fireEvent +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.providers.nzb.base import NZBProvider -from urllib import quote_plus import re log = CPLog(__name__) @@ -26,9 +26,9 @@ class X264(NZBProvider): return results q = '%s %s %s' % (movie['library']['titles'][0]['title'], movie['library']['year'], quality.get('identifier')) - url = self.urls['search'] % quote_plus(q) + url = self.urls['search'] % tryUrlencode(q) - cache_key = 'x264.%s' % q + cache_key = 'x264.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) data = self.getCache(cache_key, url) if data: match = re.compile(self.regex, re.DOTALL).finditer(data) diff --git a/couchpotato/core/providers/torrent/kickasstorrents/main.py b/couchpotato/core/providers/torrent/kickasstorrents/main.py index aff0508f..8a2fe76c 100644 --- a/couchpotato/core/providers/torrent/kickasstorrents/main.py +++ b/couchpotato/core/providers/torrent/kickasstorrents/main.py @@ -36,7 +36,7 @@ class KickAssTorrents(TorrentProvider): if self.isDisabled() or not self.isAvailable(self.urls['test']): return results - cache_key = 'kickasstorrents.%s' % movie['library']['identifier'] + cache_key = 'kickasstorrents.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) data = self.getCache(cache_key, self.urls['search'] % (movie['library']['titles'][0]['title'], movie['library']['identifier'].replace('tt', ''))) if data: diff --git a/couchpotato/core/providers/trailer/hdtrailers/main.py b/couchpotato/core/providers/trailer/hdtrailers/main.py index e6628760..26980702 100644 --- a/couchpotato/core/providers/trailer/hdtrailers/main.py +++ b/couchpotato/core/providers/trailer/hdtrailers/main.py @@ -1,9 +1,9 @@ from BeautifulSoup import SoupStrainer, BeautifulSoup +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import mergeDicts from couchpotato.core.logger import CPLog from couchpotato.core.providers.trailer.base import TrailerProvider from string import letters, digits -from urllib import urlencode import re log = CPLog(__name__) @@ -44,7 +44,7 @@ class HDTrailers(TrailerProvider): movie_name = group['library']['titles'][0]['title'] - url = "%s?%s" % (self.url['backup'], urlencode({'s':movie_name})) + url = "%s?%s" % (self.url['backup'], tryUrlencode({'s':movie_name})) data = self.getCache('hdtrailers.alt.%s' % group['library']['identifier'], url) try: From 6220d3e63dd6cf133b9a0f48adb40f8e50fa0545 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 21 Apr 2012 21:13:18 +0200 Subject: [PATCH 062/105] Get proper userdir on some platforms --- couchpotato/core/helpers/variable.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index 53b71941..d23e563d 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -1,6 +1,7 @@ import hashlib import os.path import platform +import pwd import re def getDataDir(): @@ -9,6 +10,7 @@ def getDataDir(): if os.name == 'nt': return os.path.join(os.environ['APPDATA'], 'CouchPotato') + os.environ['HOME'] = pwd.getpwuid(os.geteuid()).pw_dir user_dir = os.path.expanduser('~') # OSX From a9830ab0a6f7a134e47e830e48013613ed8f94fa Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 21 Apr 2012 21:13:55 +0200 Subject: [PATCH 063/105] Check for proper returned value. fixes #126 --- couchpotato/core/plugins/manage/main.py | 3 ++- couchpotato/core/plugins/scanner/main.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index d4bae232..52b42271 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -56,7 +56,8 @@ class Manage(Plugin): log.info('Updating manage library: %s' % directory) identifiers = fireEvent('scanner.folder', folder = directory, newer_than = last_update, single = True) - added_identifiers.extend(identifiers) + if identifiers: + added_identifiers.extend(identifiers) # Break if CP wants to shut down if self.shuttingDown(): diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 7bc726c0..a36742de 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -129,7 +129,8 @@ class Scanner(Plugin): # Add release fireEvent('release.add', group = group) library_item = fireEvent('library.update', identifier = group['library'].get('identifier'), single = True) - added_identifier.append(library_item['identifier']) + if library_item: + added_identifier.append(library_item['identifier']) return added_identifier From dcc61f740b00d79b2f47c56e149fd56d40352c71 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 21 Apr 2012 21:46:45 +0200 Subject: [PATCH 064/105] Newzbin don't fail when no attributes found --- couchpotato/core/providers/nzb/newzbin/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/providers/nzb/newzbin/main.py b/couchpotato/core/providers/nzb/newzbin/main.py index 73b4a733..34fd89ec 100644 --- a/couchpotato/core/providers/nzb/newzbin/main.py +++ b/couchpotato/core/providers/nzb/newzbin/main.py @@ -92,8 +92,11 @@ class Newzbin(NZBProvider, RSS): REPORT_NS = 'http://www.newzbin.com/DTD/2007/feeds/report/'; # Add attributes to name - for attr in nzb.find('{%s}attributes' % REPORT_NS): - title += ' ' + attr.text + try: + for attr in nzb.find('{%s}attributes' % REPORT_NS): + title += ' ' + attr.text + except: + pass id = int(self.getTextElement(nzb, '{%s}id' % REPORT_NS)) size = str(int(self.getTextElement(nzb, '{%s}size' % REPORT_NS)) / 1024 / 1024) + ' mb' From 00175cd28378acc844bdc072f7c393893ebcae5d Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 00:03:29 +0200 Subject: [PATCH 065/105] Shutdown/restart question fixes --- couchpotato/static/scripts/couchpotato.js | 10 ++++++---- couchpotato/static/style/main.css | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index 0f8fcd99..8f412eea 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -161,16 +161,17 @@ var CouchPotato = new Class({ self.checkAvailable(1000); }, - shutdownQA: function(){ + shutdownQA: function(e){ var self = this; - new Question('Are you sure you want to shutdown CouchPotato?', '', [{ + var q = new Question('Are you sure you want to shutdown CouchPotato?', '', [{ 'text': 'Shutdown', 'class': 'shutdown red', 'events': { 'click': function(e){ (e).preventDefault(); self.shutdown(); + q.close.delay(100, q); } } }, { @@ -187,16 +188,17 @@ var CouchPotato = new Class({ self.checkAvailable(1000); }, - restartQA: function(message, title){ + restartQA: function(e, message, title){ var self = this; - new Question('Are you sure you want to restart CouchPotato?', '', [{ + var q = new Question('Are you sure you want to restart CouchPotato?', '', [{ 'text': 'Restart', 'class': 'restart orange', 'events': { 'click': function(e){ (e).preventDefault(); self.restart(message, title); + q.close.delay(100, q); } } }, { diff --git a/couchpotato/static/style/main.css b/couchpotato/static/style/main.css index 467e1f8b..dfc3878d 100644 --- a/couchpotato/static/style/main.css +++ b/couchpotato/static/style/main.css @@ -491,7 +491,7 @@ body > .spinner, .mask{ width: auto; } .question .answer:hover { - background: #f1f1f1; + background: #000; } .question .answer.delete { From 7454eb5747de6fa8ea0012e799b0126f360a3e18 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 02:13:14 +0200 Subject: [PATCH 066/105] Import error pwd on Windows --- couchpotato/core/helpers/variable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/helpers/variable.py b/couchpotato/core/helpers/variable.py index d23e563d..1f5f76a2 100644 --- a/couchpotato/core/helpers/variable.py +++ b/couchpotato/core/helpers/variable.py @@ -1,7 +1,6 @@ import hashlib import os.path import platform -import pwd import re def getDataDir(): @@ -10,6 +9,7 @@ def getDataDir(): if os.name == 'nt': return os.path.join(os.environ['APPDATA'], 'CouchPotato') + import pwd os.environ['HOME'] = pwd.getpwuid(os.geteuid()).pw_dir user_dir = os.path.expanduser('~') From d3febf0f7e51d61511212a7602fbc96e73d25e5e Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 13:41:13 +0200 Subject: [PATCH 067/105] Https for providers --- couchpotato/core/providers/nzb/mysterbin/main.py | 6 +++--- couchpotato/core/providers/nzb/newzbin/main.py | 2 +- couchpotato/core/providers/nzb/nzbclub/main.py | 2 +- couchpotato/core/providers/nzb/nzbmatrix/main.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/couchpotato/core/providers/nzb/mysterbin/main.py b/couchpotato/core/providers/nzb/mysterbin/main.py index 7c3f56bc..06936f9a 100644 --- a/couchpotato/core/providers/nzb/mysterbin/main.py +++ b/couchpotato/core/providers/nzb/mysterbin/main.py @@ -12,9 +12,9 @@ log = CPLog(__name__) class Mysterbin(NZBProvider): urls = { - 'search': 'http://www.mysterbin.com/advsearch?%s', - 'download': 'http://www.mysterbin.com/nzb?c=%s', - 'nfo': 'http://www.mysterbin.com/nfo?c=%s', + 'search': 'https://www.mysterbin.com/advsearch?%s', + 'download': 'https://www.mysterbin.com/nzb?c=%s', + 'nfo': 'https://www.mysterbin.com/nfo?c=%s', } http_time_between_calls = 1 #seconds diff --git a/couchpotato/core/providers/nzb/newzbin/main.py b/couchpotato/core/providers/nzb/newzbin/main.py index 34fd89ec..c25c2664 100644 --- a/couchpotato/core/providers/nzb/newzbin/main.py +++ b/couchpotato/core/providers/nzb/newzbin/main.py @@ -14,7 +14,7 @@ log = CPLog(__name__) class Newzbin(NZBProvider, RSS): urls = { - 'download': 'http://www.newzbin2.es/api/dnzb/', + 'download': 'https://www.newzbin2.es/api/dnzb/', 'search': 'https://www.newzbin2.es/search/', } diff --git a/couchpotato/core/providers/nzb/nzbclub/main.py b/couchpotato/core/providers/nzb/nzbclub/main.py index 283b29fc..eb33fe18 100644 --- a/couchpotato/core/providers/nzb/nzbclub/main.py +++ b/couchpotato/core/providers/nzb/nzbclub/main.py @@ -15,7 +15,7 @@ log = CPLog(__name__) class NZBClub(NZBProvider, RSS): urls = { - 'search': 'http://www.nzbclub.com/nzbfeed.aspx?%s', + 'search': 'https://www.nzbclub.com/nzbfeed.aspx?%s', } http_time_between_calls = 3 #seconds diff --git a/couchpotato/core/providers/nzb/nzbmatrix/main.py b/couchpotato/core/providers/nzb/nzbmatrix/main.py index 4e26488b..8e82133b 100644 --- a/couchpotato/core/providers/nzb/nzbmatrix/main.py +++ b/couchpotato/core/providers/nzb/nzbmatrix/main.py @@ -16,7 +16,7 @@ class NZBMatrix(NZBProvider, RSS): urls = { 'download': 'https://api.nzbmatrix.com/v1.1/download.php?id=%s', 'detail': 'https://nzbmatrix.com/nzb-details.php?id=%s&hit=1', - 'search': 'http://rss.nzbmatrix.com/rss.php', + 'search': 'https://rss.nzbmatrix.com/rss.php', } cat_ids = [ From d627662038dca8281bcb25e52f884f83a7d40f12 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 16:18:23 +0200 Subject: [PATCH 068/105] Don't run multiple searches at once --- couchpotato/core/plugins/searcher/main.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 030e5459..d0f44202 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -16,6 +16,8 @@ log = CPLog(__name__) class Searcher(Plugin): + in_progress = False + def __init__(self): addEvent('searcher.all', self.all_movies) addEvent('searcher.single', self.single) @@ -27,6 +29,12 @@ class Searcher(Plugin): def all_movies(self): + if self.in_progress: + log.info('Search already in progress') + return + + self.in_progress = True + db = get_session() movies = db.query(Movie).filter( @@ -52,6 +60,8 @@ class Searcher(Plugin): if self.shuttingDown(): break + self.in_progress = False + def single(self, movie): available_status = fireEvent('status.get', 'available', single = True) From b944cc610d94e3dffc4ed987196ee70d0a0c13ad Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 16:56:04 +0200 Subject: [PATCH 069/105] Updater version --- couchpotato/core/_base/_core/main.py | 19 +++++++++++++++++++ couchpotato/core/_base/updater/main.py | 6 ++++++ 2 files changed, 25 insertions(+) diff --git a/couchpotato/core/_base/_core/main.py b/couchpotato/core/_base/_core/main.py index b22f9ba5..81ed9449 100644 --- a/couchpotato/core/_base/_core/main.py +++ b/couchpotato/core/_base/_core/main.py @@ -8,6 +8,7 @@ from couchpotato.environment import Env from flask import request from uuid import uuid4 import os +import platform import time import traceback import webbrowser @@ -32,12 +33,16 @@ class Core(Plugin): addApiView('app.available', self.available, docs = { 'desc': 'Check if app available.' }) + addApiView('app.version', self.versionView, docs = { + 'desc': 'Get version.' + }) addEvent('app.crappy_shutdown', self.crappyShutdown) addEvent('app.crappy_restart', self.crappyRestart) addEvent('app.load', self.launchBrowser, priority = 1) addEvent('app.base_url', self.createBaseUrl) addEvent('app.api_url', self.createApiUrl) + addEvent('app.version', self.version) addEvent('setting.save.core.password', self.md5Password) addEvent('setting.save.core.api_key', self.checkApikey) @@ -161,3 +166,17 @@ class Core(Plugin): def createApiUrl(self): return '%s/%s' % (self.createBaseUrl(), Env.setting('api_key')) + + def version(self): + ver = fireEvent('updater.info', single = True) + + if os.name == 'nt': platf = 'windows' + elif 'Darwin' in platform.platform(): platf = 'osx' + else: platf = 'linux' + + return '%s - %s-%s - v2' % (platf, ver.get('version')['type'], ver.get('version')['hash']) + + def versionView(self): + return jsonified({ + 'version': self.version() + }) diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index 97210123..fdd306c0 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -28,6 +28,7 @@ class Updater(Plugin): fireEvent('schedule.interval', 'updater.check', self.check, hours = 6) addEvent('app.load', self.check) + addEvent('updater.info', self.info) addApiView('updater.info', self.getInfo, docs = { 'desc': 'Get updater information', @@ -57,6 +58,9 @@ class Updater(Plugin): if self.conf('notification'): fireEvent('updater.available', message = 'A new update is available', data = self.updater.getVersion()) + def info(self): + return self.updater.info() + def getInfo(self): return jsonified(self.updater.info()) @@ -161,6 +165,7 @@ class GitUpdater(BaseUpdater): self.version = { 'hash': output.hash[:8], 'date': output.getDate(), + 'type': 'git', } except Exception, e: log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s' % e) @@ -320,6 +325,7 @@ class SourceUpdater(BaseUpdater): return { 'hash': commit['sha'], 'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())), + 'type': 'source', } except: log.error('Failed getting latest request from github: %s' % traceback.format_exc()) From 60439d45bf1cd5fcd6cc1cfaa297af58c55478a1 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 16:56:45 +0200 Subject: [PATCH 070/105] Don't look for unreleased movies. closes #123 --- couchpotato/core/plugins/library/main.py | 20 +++++++++++ couchpotato/core/plugins/quality/main.py | 4 +++ couchpotato/core/plugins/searcher/main.py | 35 +++++++++++++++++++ .../providers/movie/couchpotatoapi/main.py | 25 ++++++++----- 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/couchpotato/core/plugins/library/main.py b/couchpotato/core/plugins/library/main.py index 8d6c3192..af77a5b3 100644 --- a/couchpotato/core/plugins/library/main.py +++ b/couchpotato/core/plugins/library/main.py @@ -2,10 +2,12 @@ from couchpotato import get_session from couchpotato.core.event import addEvent, fireEventAsync, fireEvent from couchpotato.core.helpers.encoding import toUnicode, simplifyString, \ tryUrlencode +from couchpotato.core.helpers.variable import mergeDicts from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Library, LibraryTitle, File from string import ascii_letters +import time import traceback log = CPLog(__name__) @@ -17,6 +19,8 @@ class LibraryPlugin(Plugin): def __init__(self): addEvent('library.add', self.add) addEvent('library.update', self.update) + addEvent('library.update_release_date', self.updateReleaseDates) + def add(self, attrs = {}, update_after = True): @@ -119,6 +123,22 @@ class LibraryPlugin(Plugin): return library_dict + def updateReleaseDates(self, identifier): + + db = get_session() + library = db.query(Library).filter_by(identifier = identifier).first() + + if library.info.get('release_date', {}).get('expires', 0) < time.time(): + dates = fireEvent('movie.release_date', identifier = identifier, merge = True) + library.info = mergeDicts(library.info, {'release_date': dates}) + db.commit() + + dates = library.info.get('release_date', {}) + db.remove() + + return dates + + def simplifyTitle(self, title): title = toUnicode(title) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index ae33cafa..e6fa1b63 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -34,6 +34,7 @@ class QualityPlugin(Plugin): addEvent('quality.all', self.all) addEvent('quality.single', self.single) addEvent('quality.guess', self.guess) + addEvent('quality.pre_releases', self.preReleases) addApiView('quality.size.save', self.saveSize) addApiView('quality.list', self.allView, docs = { @@ -46,6 +47,9 @@ class QualityPlugin(Plugin): addEvent('app.initialize', self.fill, priority = 10) + def preReleases(self): + return self.pre_releases + def allView(self): return jsonified({ diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index d0f44202..a4050b05 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -9,6 +9,7 @@ from couchpotato.environment import Env from sqlalchemy.exc import InterfaceError import datetime import re +import time import traceback log = CPLog(__name__) @@ -27,6 +28,8 @@ class Searcher(Plugin): # Schedule cronjob fireEvent('schedule.cron', 'searcher.all', self.all_movies, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) + addEvent('app.load', self.all_movies) + def all_movies(self): if self.in_progress: @@ -64,10 +67,15 @@ class Searcher(Plugin): def single(self, movie): + pre_releases = fireEvent('quality.pre_releases', single = True) + release_dates = fireEvent('library.update_release_date', identifier = movie['library']['identifier'], merge = True) available_status = fireEvent('status.get', 'available', single = True) default_title = movie['library']['titles'][0]['title'] for quality_type in movie['profile']['types']: + if not self.couldBeReleased(quality_type['quality']['identifier'], release_dates, pre_releases): + log.info('To early to search for %s, %s' % (quality_type['quality']['identifier'], default_title)) + continue has_better_quality = 0 @@ -329,3 +337,30 @@ class Searcher(Plugin): pass return nfo and getImdb(nfo) == imdb_id + + def couldBeReleased(self, wanted_quality, dates, pre_releases): + + now = int(time.time()) + + if not dates or (dates.get('theater', 0) == 0 and dates.get('dvd', 0) == 0): + return True + else: + if wanted_quality in pre_releases: + # Prerelease 1 week before theaters + if dates.get('theater') >= now - 604800 and wanted_quality in pre_releases: + return True + else: + # 6 weeks after theater release + if dates.get('theater') < now - 3628800: + return True + + # 6 weeks before dvd release + if dates.get('dvd') > now - 3628800: + return True + + # Dvd should be released + if dates.get('dvd') < now: + return True + + + return False diff --git a/couchpotato/core/providers/movie/couchpotatoapi/main.py b/couchpotato/core/providers/movie/couchpotatoapi/main.py index aca80ccc..dd8d36df 100644 --- a/couchpotato/core/providers/movie/couchpotatoapi/main.py +++ b/couchpotato/core/providers/movie/couchpotatoapi/main.py @@ -1,6 +1,6 @@ from couchpotato import get_session from couchpotato.api import addApiView -from couchpotato.core.event import addEvent +from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.request import jsonified, getParams from couchpotato.core.logger import CPLog from couchpotato.core.providers.movie.base import MovieProvider @@ -15,19 +15,28 @@ class CouchPotatoApi(MovieProvider): apiUrl = 'http://couchpota.to/api/%s/' def __init__(self): - addEvent('movie.release_date', self.releaseDate) - addApiView('movie.suggest', self.suggestView) + #addApiView('movie.suggest', self.suggestView) - def releaseDate(self, imdb_id): + addEvent('movie.info', self.getInfo) + addEvent('movie.release_date', self.getReleaseDate) + + def getInfo(self, identifier = None): + return { + 'release_date': self.getReleaseDate(identifier) + } + + def getReleaseDate(self, identifier = None): try: - data = self.urlopen((self.apiUrl % ('eta')) + (id + '/')) + headers = {'X-CP-Version': fireEvent('app.version', single = True)} + data = self.urlopen((self.apiUrl % ('eta')) + (identifier + '/'), headers = headers) dates = json.loads(data) - log.info('Found ETA for %s: %s' % (imdb_id, dates)) + log.debug('Found ETA for %s: %s' % (identifier, dates)) + return dates except Exception, e: - log.error('Error getting ETA for %s: %s' % (imdb_id, e)) + log.error('Error getting ETA for %s: %s' % (identifier, e)) - return dates + return {} def suggest(self, movies = [], ignore = []): try: From 5fa6e3d960c55374496666a1c24fe938363d2949 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 18:02:15 +0200 Subject: [PATCH 071/105] Prefixfree update. closes #128 --- .../static/scripts/library/prefix_free.js | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/couchpotato/static/scripts/library/prefix_free.js b/couchpotato/static/scripts/library/prefix_free.js index 1ca634ee..4876187b 100644 --- a/couchpotato/static/scripts/library/prefix_free.js +++ b/couchpotato/static/scripts/library/prefix_free.js @@ -25,20 +25,24 @@ var self = window.StyleFix = { var url = link.href || link.getAttribute('data-href'), base = url.replace(/[^\/]+$/, ''), parent = link.parentNode, - xhr = new XMLHttpRequest(); + xhr = new XMLHttpRequest(), + process; - xhr.open('GET', url); - xhr.onreadystatechange = function() { if(xhr.readyState === 4) { + process(); + } + }; + + process = function() { var css = xhr.responseText; - if(css && link.parentNode) { + if(css && link.parentNode && (!xhr.status || xhr.status < 400 || xhr.status > 600)) { css = self.fix(css, true, link); // Convert relative URLs to absolute, if needed if(base) { - css = css.replace(/url\(((?:"|')?)(.+?)\1\)/gi, function($0, quote, url) { + css = css.replace(/url\(\s*?((?:"|')?)(.+?)\1\s*?\)/gi, function($0, quote, url) { if(!/^([a-z]{3,10}:|\/|#)/i.test(url)) { // If url not absolute & not a hash // May contain sequences like /../ and /./ but those DO work return 'url("' + base + url + '")'; @@ -60,10 +64,21 @@ var self = window.StyleFix = { parent.insertBefore(style, link); parent.removeChild(link); } - } }; - - xhr.send(null); + + try { + xhr.open('GET', url); + xhr.send(null); + } catch (e) { + // Fallback to XDomainRequest if available + if (typeof XDomainRequest != "undefined") { + xhr = new XDomainRequest(); + xhr.onerror = xhr.onprogress = function() {}; + xhr.onload = process; + xhr.open("GET", url); + xhr.send(null); + } + } link.setAttribute('data-inprogress', ''); }, @@ -135,7 +150,7 @@ function $(expr, con) { })(); /** - * PrefixFree 1.0.4 + * PrefixFree 1.0.5 * @author Lea Verou * MIT license */ @@ -300,6 +315,10 @@ var functions = { 'element': { property: 'backgroundImage', params: '#foo' + }, + 'cross-fade': { + property: 'backgroundImage', + params: 'url(a.png), url(b.png), 50%' } }; From aae2b53b2d4195d6cc514f60bb26a27543e7a9c4 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 19:38:34 +0200 Subject: [PATCH 072/105] Add notification plugin for pushover.net --- .../core/notifications/pushover/__init__.py | 43 +++++++++++++++++++ .../core/notifications/pushover/main.py | 41 ++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 couchpotato/core/notifications/pushover/__init__.py create mode 100644 couchpotato/core/notifications/pushover/main.py diff --git a/couchpotato/core/notifications/pushover/__init__.py b/couchpotato/core/notifications/pushover/__init__.py new file mode 100644 index 00000000..949acc2b --- /dev/null +++ b/couchpotato/core/notifications/pushover/__init__.py @@ -0,0 +1,43 @@ +from .main import Pushover + +def start(): + return Pushover() + +config = [{ + 'name': 'pushover', + 'groups': [ + { + 'tab': 'notifications', + 'name': 'pushover', + 'label': 'Pushover', + 'options': [ + { + 'name': 'enabled', + 'default': 0, + 'type': 'enabler', + }, + { + 'name': 'user_key', + 'description': 'Register on pushover.net to get one.' + }, + { + 'name': 'app_token', + 'description': 'You can get one here.' + }, + { + 'name': 'priority', + 'default': 0, + 'type': 'dropdown', + 'values': [('Normal', 0), ('High', 1)], + }, + { + 'name': 'on_snatch', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Also send message when movie is snatched.', + }, + ], + } + ], +}] diff --git a/couchpotato/core/notifications/pushover/main.py b/couchpotato/core/notifications/pushover/main.py new file mode 100644 index 00000000..6c406185 --- /dev/null +++ b/couchpotato/core/notifications/pushover/main.py @@ -0,0 +1,41 @@ +from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.logger import CPLog +from couchpotato.core.notifications.base import Notification +from httplib import HTTPSConnection +from urllib import urlencode + +log = CPLog(__name__) + + +class Pushover(Notification): + + def notify(self, message = '', data = {}): + if self.isDisabled(): return + + http_handler = HTTPSConnection("api.pushover.net:443") + + data = { + 'user': self.conf('user_key'), + 'token': self.conf('app_token'), + 'message': toUnicode(message), + 'priority': self.conf('priority') + } + + http_handler.request('POST', + "/1/messages.json", + headers = {'Content-type': 'application/x-www-form-urlencoded'}, + body = urlencode(data) + ) + + response = http_handler.getresponse() + request_status = response.status + + if request_status == 200: + log.info('Pushover notifications sent.') + return True + elif request_status == 401: + log.error('Pushover auth failed: %s' % response.reason) + return False + else: + log.error('Pushover notification failed.') + return False From 98176ea259274bcb9e2fae03b9f3271d71061473 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 19:56:35 +0200 Subject: [PATCH 073/105] Add apptoken for Pushover --- couchpotato/core/notifications/pushover/__init__.py | 5 ----- couchpotato/core/notifications/pushover/main.py | 9 +++++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/couchpotato/core/notifications/pushover/__init__.py b/couchpotato/core/notifications/pushover/__init__.py index 949acc2b..d8d76b78 100644 --- a/couchpotato/core/notifications/pushover/__init__.py +++ b/couchpotato/core/notifications/pushover/__init__.py @@ -9,7 +9,6 @@ config = [{ { 'tab': 'notifications', 'name': 'pushover', - 'label': 'Pushover', 'options': [ { 'name': 'enabled', @@ -20,10 +19,6 @@ config = [{ 'name': 'user_key', 'description': 'Register on pushover.net to get one.' }, - { - 'name': 'app_token', - 'description': 'You can get one here.' - }, { 'name': 'priority', 'default': 0, diff --git a/couchpotato/core/notifications/pushover/main.py b/couchpotato/core/notifications/pushover/main.py index 6c406185..d531f2b6 100644 --- a/couchpotato/core/notifications/pushover/main.py +++ b/couchpotato/core/notifications/pushover/main.py @@ -1,14 +1,15 @@ -from couchpotato.core.helpers.encoding import toUnicode +from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification from httplib import HTTPSConnection -from urllib import urlencode log = CPLog(__name__) class Pushover(Notification): + app_token = 'YkxHMYDZp285L265L3IwH3LmzkTaCy' + def notify(self, message = '', data = {}): if self.isDisabled(): return @@ -16,7 +17,7 @@ class Pushover(Notification): data = { 'user': self.conf('user_key'), - 'token': self.conf('app_token'), + 'token': self.app_token, 'message': toUnicode(message), 'priority': self.conf('priority') } @@ -24,7 +25,7 @@ class Pushover(Notification): http_handler.request('POST', "/1/messages.json", headers = {'Content-type': 'application/x-www-form-urlencoded'}, - body = urlencode(data) + body = tryUrlencode(data) ) response = http_handler.getresponse() From 76e29e6a7a8fae9b6800b6023d0812040cb48e6e Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 20:10:05 +0200 Subject: [PATCH 074/105] Don't search on start. --- couchpotato/core/plugins/searcher/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index a4050b05..ce675228 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -28,8 +28,6 @@ class Searcher(Plugin): # Schedule cronjob fireEvent('schedule.cron', 'searcher.all', self.all_movies, day = self.conf('cron_day'), hour = self.conf('cron_hour'), minute = self.conf('cron_minute')) - addEvent('app.load', self.all_movies) - def all_movies(self): if self.in_progress: From 7c25f406b0ef887f3ce810f975707706477f50bc Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 20:11:59 +0200 Subject: [PATCH 075/105] Improved couldBeReleased check. fixes #123 --- couchpotato/core/plugins/searcher/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index ce675228..0159744a 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -357,7 +357,7 @@ class Searcher(Plugin): return True # Dvd should be released - if dates.get('dvd') < now: + if dates.get('dvd') > 0 and dates.get('dvd') < now: return True From 64ad5fb6b433be4ac2a569a906b64e4b398638fb Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 20:25:12 +0200 Subject: [PATCH 076/105] Don't limit CouchPotato api requests --- couchpotato/core/providers/movie/couchpotatoapi/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/providers/movie/couchpotatoapi/main.py b/couchpotato/core/providers/movie/couchpotatoapi/main.py index dd8d36df..17b58c0f 100644 --- a/couchpotato/core/providers/movie/couchpotatoapi/main.py +++ b/couchpotato/core/providers/movie/couchpotatoapi/main.py @@ -12,7 +12,8 @@ log = CPLog(__name__) class CouchPotatoApi(MovieProvider): - apiUrl = 'http://couchpota.to/api/%s/' + api_url = 'http://couchpota.to/api/%s/' + http_time_between_calls = 0 def __init__(self): #addApiView('movie.suggest', self.suggestView) @@ -29,7 +30,7 @@ class CouchPotatoApi(MovieProvider): try: headers = {'X-CP-Version': fireEvent('app.version', single = True)} - data = self.urlopen((self.apiUrl % ('eta')) + (identifier + '/'), headers = headers) + data = self.urlopen((self.api_url % ('eta')) + (identifier + '/'), headers = headers) dates = json.loads(data) log.debug('Found ETA for %s: %s' % (identifier, dates)) return dates @@ -40,7 +41,7 @@ class CouchPotatoApi(MovieProvider): def suggest(self, movies = [], ignore = []): try: - data = self.urlopen((self.apiUrl % ('suggest')) + ','.join(movies) + '/' + ','.join(ignore) + '/') + data = self.urlopen((self.api_url % ('suggest')) + ','.join(movies) + '/' + ','.join(ignore) + '/') suggestions = json.loads(data) log.info('Found Suggestions for %s' % (suggestions)) except Exception, e: From 27f711df03d2aa85c9e7c8fb9f7ad341ad161097 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 21:33:55 +0200 Subject: [PATCH 077/105] Update bar blocks header. fixes #127 --- couchpotato/core/_base/updater/main.py | 3 ++- couchpotato/static/style/main.css | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index fdd306c0..ecc0292d 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -95,7 +95,8 @@ class BaseUpdater(Plugin): return { 'last_check': self.last_check, 'update_version': self.update_version, - 'version': self.getVersion() + 'version': self.getVersion(), + 'repo_name': '%s/%s' % (self.repo_user, self.repo_name) } def check(self): diff --git a/couchpotato/static/style/main.css b/couchpotato/static/style/main.css index dfc3878d..d580368c 100644 --- a/couchpotato/static/style/main.css +++ b/couchpotato/static/style/main.css @@ -305,9 +305,9 @@ body > .spinner, .mask{ text-align: center; position: relative; top: -70px; - padding: 15px 0 20px; + padding: 2px 0; background: #ff6134; - font-size: 26px; + font-size: 12px; border-radius: 0 0 5px 5px; box-shadow: 0 2px 1px rgba(0,0,0, 0.3); } From 8069350b43f79bd5e7da00e0be7bd249cc9910ce Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 21:34:14 +0200 Subject: [PATCH 078/105] Import cleaning --- couchpotato/core/providers/movie/couchpotatoapi/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/couchpotato/core/providers/movie/couchpotatoapi/main.py b/couchpotato/core/providers/movie/couchpotatoapi/main.py index 17b58c0f..040bc2e2 100644 --- a/couchpotato/core/providers/movie/couchpotatoapi/main.py +++ b/couchpotato/core/providers/movie/couchpotatoapi/main.py @@ -1,5 +1,4 @@ from couchpotato import get_session -from couchpotato.api import addApiView from couchpotato.core.event import addEvent, fireEvent from couchpotato.core.helpers.request import jsonified, getParams from couchpotato.core.logger import CPLog From d7d8d7acce7e7f18268487e31f21531824cca012 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sun, 22 Apr 2012 22:03:00 +0200 Subject: [PATCH 079/105] Autoupdater broken --- couchpotato/core/plugins/automation/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/automation/main.py b/couchpotato/core/plugins/automation/main.py index 6acf26d8..a29d8810 100644 --- a/couchpotato/core/plugins/automation/main.py +++ b/couchpotato/core/plugins/automation/main.py @@ -10,7 +10,7 @@ class Automation(Plugin): def __init__(self): - fireEvent('schedule.interval', 'updater.check', self.addMovies, hours = self.conf('hour', default = 12)) + fireEvent('schedule.interval', 'automation.add_movies', self.addMovies, hours = self.conf('hour', default = 12)) if not Env.get('dev'): addEvent('app.load', self.addMovies) From b924e9aa343040e3c35deae726233ccb648ff546 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 24 Apr 2012 00:06:39 +0200 Subject: [PATCH 080/105] Search with imdbid. closes #133 --- couchpotato/core/plugins/movie/main.py | 7 ++++++- couchpotato/core/plugins/movie/static/search.js | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 28332dca..06394bcb 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -4,6 +4,7 @@ from couchpotato.core.event import fireEvent, fireEventAsync, addEvent from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode, \ simplifyString from couchpotato.core.helpers.request import getParams, jsonified, getParam +from couchpotato.core.helpers.variable import getImdb from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.core.settings.model import Movie, Library, LibraryTitle @@ -239,7 +240,11 @@ class MoviePlugin(Plugin): movies = Env.get('cache').get(cache_key) if not movies: - movies = fireEvent('movie.search', q = q, merge = True) + + if getImdb(q): + movies = [fireEvent('movie.info', identifier = q, merge = True)] + else: + movies = fireEvent('movie.search', q = q, merge = True) Env.get('cache').set(cache_key, movies) return jsonified({ diff --git a/couchpotato/core/plugins/movie/static/search.js b/couchpotato/core/plugins/movie/static/search.js index 65c4327d..10381b56 100644 --- a/couchpotato/core/plugins/movie/static/search.js +++ b/couchpotato/core/plugins/movie/static/search.js @@ -148,6 +148,9 @@ Block.Search = new Class({ $(m).inject(self.results) self.movies[movie.imdb || 'r-'+Math.floor(Math.random()*10000)] = m + if(q == movie.imdb) + m.showOptions() + }); if(q != self.q()) From 796bb3987d51eca47615d990798ad75b1242fd23 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 24 Apr 2012 00:23:26 +0200 Subject: [PATCH 081/105] Always return version type. fixes #134 --- couchpotato/core/_base/updater/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index ecc0292d..f5182228 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -295,6 +295,7 @@ class SourceUpdater(BaseUpdater): log.debug('Source version output: %s' % output) self.version = output + self.version['type'] = 'source' except Exception, e: log.error('Failed using source updater. %s' % e) return {} @@ -326,7 +327,6 @@ class SourceUpdater(BaseUpdater): return { 'hash': commit['sha'], 'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())), - 'type': 'source', } except: log.error('Failed getting latest request from github: %s' % traceback.format_exc()) From 7b716f3b9d53fd6b47c185aef80684a1c73d219a Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 24 Apr 2012 08:23:00 +0200 Subject: [PATCH 082/105] Don't download automatically but enable provider. #135 --- couchpotato/core/downloaders/base.py | 6 ++++++ couchpotato/core/downloaders/blackhole/__init__.py | 7 +++++++ couchpotato/core/downloaders/blackhole/main.py | 4 ++-- couchpotato/core/downloaders/nzbget/__init__.py | 7 +++++++ couchpotato/core/downloaders/nzbget/main.py | 4 ++-- couchpotato/core/downloaders/sabnzbd/__init__.py | 7 +++++++ couchpotato/core/downloaders/sabnzbd/main.py | 4 ++-- couchpotato/core/downloaders/transmission/__init__.py | 7 +++++++ couchpotato/core/downloaders/transmission/main.py | 4 ++-- couchpotato/core/plugins/release/main.py | 2 +- couchpotato/core/plugins/searcher/main.py | 4 ++-- 11 files changed, 45 insertions(+), 11 deletions(-) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 39ee6129..5c4e1020 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -40,3 +40,9 @@ class Downloader(Plugin): log.debug("Downloader doesn't support this type") return is_correct + + def isDisabled(self, manual): + return not self.isEnabled(manual) + + def isEnabled(self, manual): + return super(Downloader, self).isEnabled(), ((self.conf('manual', default = False) and manual) or manual is True) diff --git a/couchpotato/core/downloaders/blackhole/__init__.py b/couchpotato/core/downloaders/blackhole/__init__.py index 3231c48d..b9796051 100644 --- a/couchpotato/core/downloaders/blackhole/__init__.py +++ b/couchpotato/core/downloaders/blackhole/__init__.py @@ -32,6 +32,13 @@ config = [{ 'type': 'dropdown', 'values': [('nzbs & torrents', 'both'), ('nzb', 'nzb'), ('torrent', 'torrent')], }, + { + 'name': 'manual', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', + }, ], } ], diff --git a/couchpotato/core/downloaders/blackhole/main.py b/couchpotato/core/downloaders/blackhole/main.py index 9408de18..a24e71b1 100644 --- a/couchpotato/core/downloaders/blackhole/main.py +++ b/couchpotato/core/downloaders/blackhole/main.py @@ -10,8 +10,8 @@ class Blackhole(Downloader): type = ['nzb', 'torrent'] - def download(self, data = {}, movie = {}): - if self.isDisabled() or (not self.isCorrectType(data.get('type')) or (not self.conf('use_for') in ['both', data.get('type')])): + def download(self, data = {}, movie = {}, manual = False): + if self.isDisabled(manual) or (not self.isCorrectType(data.get('type')) or (not self.conf('use_for') in ['both', data.get('type')])): return directory = self.conf('directory') diff --git a/couchpotato/core/downloaders/nzbget/__init__.py b/couchpotato/core/downloaders/nzbget/__init__.py index 00fa3f0f..5015437e 100644 --- a/couchpotato/core/downloaders/nzbget/__init__.py +++ b/couchpotato/core/downloaders/nzbget/__init__.py @@ -33,6 +33,13 @@ config = [{ 'default': 'Movies', 'description': 'The category CP places the nzb in. Like movies or couchpotato', }, + { + 'name': 'manual', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', + }, ], } ], diff --git a/couchpotato/core/downloaders/nzbget/main.py b/couchpotato/core/downloaders/nzbget/main.py index b65e2511..15243ca4 100644 --- a/couchpotato/core/downloaders/nzbget/main.py +++ b/couchpotato/core/downloaders/nzbget/main.py @@ -14,9 +14,9 @@ class NZBGet(Downloader): url = 'http://nzbget:%(password)s@%(host)s/xmlrpc' - def download(self, data = {}, movie = {}): + def download(self, data = {}, movie = {}, manual = False): - if self.isDisabled() or not self.isCorrectType(data.get('type')): + if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): return log.info('Sending "%s" to NZBGet.' % data.get('name')) diff --git a/couchpotato/core/downloaders/sabnzbd/__init__.py b/couchpotato/core/downloaders/sabnzbd/__init__.py index 20af743b..23918fe2 100644 --- a/couchpotato/core/downloaders/sabnzbd/__init__.py +++ b/couchpotato/core/downloaders/sabnzbd/__init__.py @@ -39,6 +39,13 @@ config = [{ 'type': 'directory', 'description': 'Your Post-Processing Script directory, set in Sabnzbd > Config > Directories.', }, + { + 'name': 'manual', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', + }, ], } ], diff --git a/couchpotato/core/downloaders/sabnzbd/main.py b/couchpotato/core/downloaders/sabnzbd/main.py index 0319f68d..837ab063 100644 --- a/couchpotato/core/downloaders/sabnzbd/main.py +++ b/couchpotato/core/downloaders/sabnzbd/main.py @@ -15,9 +15,9 @@ class Sabnzbd(Downloader): type = ['nzb'] - def download(self, data = {}, movie = {}): + def download(self, data = {}, movie = {}, manual = False): - if self.isDisabled() or not self.isCorrectType(data.get('type')): + if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): return log.info("Sending '%s' to SABnzbd." % data.get('name')) diff --git a/couchpotato/core/downloaders/transmission/__init__.py b/couchpotato/core/downloaders/transmission/__init__.py index 34bd3c8f..f72dea6b 100644 --- a/couchpotato/core/downloaders/transmission/__init__.py +++ b/couchpotato/core/downloaders/transmission/__init__.py @@ -48,6 +48,13 @@ config = [{ 'advanced': True, 'description': 'Stop transfer when reaching ratio', }, + { + 'name': 'manual', + 'default': 0, + 'type': 'bool', + 'advanced': True, + 'description': 'Disable this downloader for automated searches, but use it when I manually send a release.', + }, ], } ], diff --git a/couchpotato/core/downloaders/transmission/main.py b/couchpotato/core/downloaders/transmission/main.py index 7677ad02..4ed3e583 100644 --- a/couchpotato/core/downloaders/transmission/main.py +++ b/couchpotato/core/downloaders/transmission/main.py @@ -11,9 +11,9 @@ class Transmission(Downloader): type = ['torrent'] - def download(self, data = {}, movie = {}): + def download(self, data = {}, movie = {}, manual = False): - if self.isDisabled() or not self.isCorrectType(data.get('type')): + if self.isDisabled(manual) or not self.isCorrectType(data.get('type')): return log.info('Sending "%s" to Transmission.' % data.get('name')) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index f83b322f..3a0f2047 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -147,7 +147,7 @@ class Release(Plugin): 'releases': {'status': {}, 'quality': {}}, 'library': {'titles': {}, 'files':{}}, 'files': {} - })) + }), manual = True) return jsonified({ 'success': True diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 0159744a..72350d81 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -135,10 +135,10 @@ class Searcher(Plugin): return False - def download(self, data, movie): + def download(self, data, movie, manual = False): snatched_status = fireEvent('status.get', 'snatched', single = True) - successful = fireEvent('download', data = data, movie = movie, single = True) + successful = fireEvent('download', data = data, movie = movie, manual = manual, single = True) if successful: From 442e2233997cf0ff19cac936f0c4f8ebc242e033 Mon Sep 17 00:00:00 2001 From: Ruud Date: Tue, 24 Apr 2012 17:22:03 +0200 Subject: [PATCH 083/105] Cleanup --- couchpotato/core/plugins/library/main.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/library/main.py b/couchpotato/core/plugins/library/main.py index af77a5b3..11c212ab 100644 --- a/couchpotato/core/plugins/library/main.py +++ b/couchpotato/core/plugins/library/main.py @@ -1,7 +1,6 @@ from couchpotato import get_session from couchpotato.core.event import addEvent, fireEventAsync, fireEvent -from couchpotato.core.helpers.encoding import toUnicode, simplifyString, \ - tryUrlencode +from couchpotato.core.helpers.encoding import toUnicode, simplifyString from couchpotato.core.helpers.variable import mergeDicts from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin @@ -19,7 +18,7 @@ class LibraryPlugin(Plugin): def __init__(self): addEvent('library.add', self.add) addEvent('library.update', self.update) - addEvent('library.update_release_date', self.updateReleaseDates) + addEvent('library.update_release_date', self.updateReleaseDate) def add(self, attrs = {}, update_after = True): @@ -123,7 +122,7 @@ class LibraryPlugin(Plugin): return library_dict - def updateReleaseDates(self, identifier): + def updateReleaseDate(self, identifier): db = get_session() library = db.query(Library).filter_by(identifier = identifier).first() From 5eb59636e9889e615225213811692ce423d79daf Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Apr 2012 20:50:05 +0200 Subject: [PATCH 084/105] Downloaded isEnabled error --- couchpotato/core/downloaders/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 5c4e1020..31eecfd7 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -45,4 +45,4 @@ class Downloader(Plugin): return not self.isEnabled(manual) def isEnabled(self, manual): - return super(Downloader, self).isEnabled(), ((self.conf('manual', default = False) and manual) or manual is True) + return super(Downloader, self).isEnabled() and ((self.conf('manual', default = False) and manual) or manual is True) From ee3c8d7110509c22efb614f3d3c87a72cbf5db3d Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Apr 2012 20:58:19 +0200 Subject: [PATCH 085/105] Manual download fix --- couchpotato/core/downloaders/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/downloaders/base.py b/couchpotato/core/downloaders/base.py index 31eecfd7..710251aa 100644 --- a/couchpotato/core/downloaders/base.py +++ b/couchpotato/core/downloaders/base.py @@ -45,4 +45,5 @@ class Downloader(Plugin): return not self.isEnabled(manual) def isEnabled(self, manual): - return super(Downloader, self).isEnabled() and ((self.conf('manual', default = False) and manual) or manual is True) + d_manual = self.conf('manual', default = False) + return super(Downloader, self).isEnabled() and ((d_manual and manual) or (d_manual is False)) From ed7b9e6564988f0baf7e018cac7052ff2de4fd7e Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Apr 2012 21:34:31 +0200 Subject: [PATCH 086/105] Improved searching --- couchpotato/core/plugins/searcher/main.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 72350d81..5a3aaf80 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -190,7 +190,8 @@ class Searcher(Plugin): log.info('Wrong: Outside retention, age is %s, needs %s or lower: %s' % (nzb['age'], retention, nzb['name'])) return False - nzb_words = re.split('\W+', simplifyString(nzb['name'])) + movie_name = simplifyString(nzb['name']) + nzb_words = re.split('\W+', movie_name) required_words = self.conf('required_words').split(',') if self.conf('required_words') and not list(set(nzb_words) & set(required_words)): @@ -203,6 +204,12 @@ class Searcher(Plugin): log.info("Wrong: '%s' blacklisted words: %s" % (nzb['name'], ", ".join(blacklisted))) return False + pron_tags = ['xxx', 'sex', 'anal', 'tits', 'fuck', 'porn', 'orgy', 'milf', 'boobs'] + for p_tag in pron_tags: + if p_tag in movie_name: + log.info('Wrong: %s, probably pr0n' % (nzb['name'])) + return False + #qualities = fireEvent('quality.all', single = True) preferred_quality = fireEvent('quality.single', identifier = quality['identifier'], single = True) @@ -313,10 +320,10 @@ class Searcher(Plugin): check_movie = fireEvent('scanner.name_year', check_name, single = True) try: - check_words = re.split('\W+', check_movie.get('name', '')) - movie_words = re.split('\W+', simplifyString(movie_name)) + check_words = filter(None, re.split('\W+', check_movie.get('name', ''))) + movie_words = filter(None, re.split('\W+', simplifyString(movie_name))) - if len(list(set(check_words) - set(movie_words))) == 0: + if len(check_words) > 0 and len(movie_words) > 0 and len(list(set(check_words) - set(movie_words))) == 0: return True except: pass From 269fc66e4c34d334276ee13b7b285b583fe99d1a Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Apr 2012 21:41:45 +0200 Subject: [PATCH 087/105] Added .ts to searcher --- couchpotato/core/plugins/quality/main.py | 2 +- couchpotato/core/plugins/scanner/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index e6fa1b63..3c5ad7be 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -18,7 +18,7 @@ class QualityPlugin(Plugin): qualities = [ {'identifier': 'bd50', 'hd': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]}, {'identifier': '1080p', 'hd': True, 'size': (5000, 20000), 'label': '1080P', 'width': 1920, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts']}, - {'identifier': '720p', 'hd': True, 'size': (3500, 10000), 'label': '720P', 'width': 1280, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts']}, + {'identifier': '720p', 'hd': True, 'size': (3500, 10000), 'label': '720P', 'width': 1280, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts', 'ts']}, {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p'], 'ext':['avi']}, {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': [], 'allow': [], 'ext':['iso', 'img'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'alternative': ['dvdrip'], 'allow': [], 'ext':['avi', 'mpg', 'mpeg']}, diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index a36742de..469642ce 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -31,7 +31,7 @@ class Scanner(Plugin): ignored_in_path = ['_unpack', '_failed_', '_unknown_', '_exists_', '.appledouble', '.appledb', '.appledesktop', os.path.sep + '._', '.ds_store', 'cp.cpnfo'] #unpacking, smb-crap, hidden files ignore_names = ['extract', 'extracting', 'extracted', 'movie', 'movies', 'film', 'films', 'download', 'downloads', 'video_ts', 'audio_ts', 'bdmv', 'certificate'] extensions = { - 'movie': ['mkv', 'wmv', 'avi', 'mpg', 'mpeg', 'mp4', 'm2ts', 'iso', 'img', 'mdf'], + 'movie': ['mkv', 'wmv', 'avi', 'mpg', 'mpeg', 'mp4', 'm2ts', 'iso', 'img', 'mdf', 'ts'], 'movie_extra': ['mds'], 'dvd': ['vts_*', 'vob'], 'nfo': ['nfo', 'txt', 'tag'], From ac15df31d803746e5d98f1273bb8db736c4ef60c Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Apr 2012 21:57:47 +0200 Subject: [PATCH 088/105] Properly split and strip words. fixes #137 --- couchpotato/core/notifications/core/main.py | 4 ++-- couchpotato/core/notifications/notifymyandroid/main.py | 2 +- couchpotato/core/notifications/notifymywp/main.py | 2 +- couchpotato/core/plugins/manage/main.py | 2 +- couchpotato/core/plugins/movie/main.py | 6 +++--- couchpotato/core/plugins/score/scores.py | 2 +- couchpotato/core/plugins/searcher/main.py | 4 ++-- couchpotato/core/providers/nzb/newznab/main.py | 6 +++--- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/couchpotato/core/notifications/core/main.py b/couchpotato/core/notifications/core/main.py index c3a16214..4c20094a 100644 --- a/couchpotato/core/notifications/core/main.py +++ b/couchpotato/core/notifications/core/main.py @@ -56,7 +56,7 @@ class CoreNotifier(Notification): addEvent('library.update_finish', lambda data: fireEvent('notify.frontend', type = 'library.update', data = data)) def markAsRead(self): - ids = getParam('ids').split(',') + ids = [x.strip() for x in getParam('ids').split(',')] db = get_session() @@ -78,7 +78,7 @@ class CoreNotifier(Notification): q = db.query(Notif) if limit_offset: - splt = limit_offset.split(',') + splt = [x.strip() for x in limit_offset.split(',')] limit = splt[0] offset = 0 if len(splt) is 1 else splt[1] q = q.limit(limit).offset(offset) diff --git a/couchpotato/core/notifications/notifymyandroid/main.py b/couchpotato/core/notifications/notifymyandroid/main.py index 45b07cde..2f843174 100644 --- a/couchpotato/core/notifications/notifymyandroid/main.py +++ b/couchpotato/core/notifications/notifymyandroid/main.py @@ -11,7 +11,7 @@ class NotifyMyAndroid(Notification): if self.isDisabled(): return nma = pynma.PyNMA() - keys = self.conf('api_key').split(',') + keys = [x.strip() for x in self.conf('api_key').split(',')] nma.addkey(keys) nma.developerkey(self.conf('dev_key')) diff --git a/couchpotato/core/notifications/notifymywp/main.py b/couchpotato/core/notifications/notifymywp/main.py index 7c294bfc..203fdad5 100644 --- a/couchpotato/core/notifications/notifymywp/main.py +++ b/couchpotato/core/notifications/notifymywp/main.py @@ -10,7 +10,7 @@ class NotifyMyWP(Notification): def notify(self, message = '', data = {}): if self.isDisabled(): return - keys = self.conf('api_key').split(',') + keys = [x.strip() for x in self.conf('api_key').split(',')] p = PyNMWP(keys, self.conf('dev_key')) response = p.push(application = self.default_title, event = message, description = message, priority = self.conf('priority'), batch_mode = len(keys) > 1) diff --git a/couchpotato/core/plugins/manage/main.py b/couchpotato/core/plugins/manage/main.py index 52b42271..73e10693 100644 --- a/couchpotato/core/plugins/manage/main.py +++ b/couchpotato/core/plugins/manage/main.py @@ -77,6 +77,6 @@ class Manage(Plugin): def directories(self): try: - return self.conf('library', default = '').split('::') + return [x.strip() for x in self.conf('library', default = '').split('::')] except: return [] diff --git a/couchpotato/core/plugins/movie/main.py b/couchpotato/core/plugins/movie/main.py index 06394bcb..e18d000d 100644 --- a/couchpotato/core/plugins/movie/main.py +++ b/couchpotato/core/plugins/movie/main.py @@ -139,7 +139,7 @@ class MoviePlugin(Plugin): if limit_offset: - splt = limit_offset.split(',') + splt = [x.strip() for x in limit_offset.split(',')] limit = splt[0] offset = 0 if len(splt) is 1 else splt[1] q2 = q2.limit(limit).offset(offset) @@ -324,7 +324,7 @@ class MoviePlugin(Plugin): available_status = fireEvent('status.get', 'available', single = True) - ids = params.get('id').split(',') + ids = [x.strip() for x in params.get('id').split(',')] for movie_id in ids: m = db.query(Movie).filter_by(id = movie_id).first() @@ -356,7 +356,7 @@ class MoviePlugin(Plugin): params = getParams() - ids = params.get('id').split(',') + ids = [x.strip() for x in params.get('id').split(',')] for movie_id in ids: self.delete(movie_id) diff --git a/couchpotato/core/plugins/score/scores.py b/couchpotato/core/plugins/score/scores.py index 1a2f2b88..2cace048 100644 --- a/couchpotato/core/plugins/score/scores.py +++ b/couchpotato/core/plugins/score/scores.py @@ -40,7 +40,7 @@ def nameScore(name, year): # Contains preferred word nzb_words = re.split('\W+', simplifyString(name)) - preferred_words = Env.setting('preferred_words', section = 'searcher').split(',') + preferred_words = [x.strip() for x in Env.setting('preferred_words', section = 'searcher').split(',')] for word in preferred_words: if word.strip() and word.strip().lower() in nzb_words: score = score + 100 diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 5a3aaf80..4c26f4ce 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -192,13 +192,13 @@ class Searcher(Plugin): movie_name = simplifyString(nzb['name']) nzb_words = re.split('\W+', movie_name) - required_words = self.conf('required_words').split(',') + required_words = [x.strip() for x in self.conf('required_words').split(',')] if self.conf('required_words') and not list(set(nzb_words) & set(required_words)): log.info("NZB doesn't contain any of the required words.") return False - ignored_words = self.conf('ignored_words').split(',') + ignored_words = [x.strip() for x in self.conf('ignored_words').split(',')] blacklisted = list(set(nzb_words) & set(ignored_words)) if self.conf('ignored_words') and blacklisted: log.info("Wrong: '%s' blacklisted words: %s" % (nzb['name'], ", ".join(blacklisted))) diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py index 5ec67432..99fb771d 100644 --- a/couchpotato/core/providers/nzb/newznab/main.py +++ b/couchpotato/core/providers/nzb/newznab/main.py @@ -157,9 +157,9 @@ class Newznab(NZBProvider, RSS): def getHosts(self): - uses = str(self.conf('use')).split(',') - hosts = self.conf('host').split(',') - api_keys = self.conf('api_key').split(',') + uses = [x.strip() for x in str(self.conf('use')).split(',')] + hosts = [x.strip() for x in self.conf('host').split(',')] + api_keys = [x.strip() for x in self.conf('api_key').split(',')] list = [] for nr in range(len(hosts)): From b936576d6d2f2ff164f372b94d20eb9557ded6b9 Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Apr 2012 22:31:22 +0200 Subject: [PATCH 089/105] Get apikey with username and password. closes #142 --- couchpotato/__init__.py | 18 ++++++++++++++++++ couchpotato/templates/api.html | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/couchpotato/__init__.py b/couchpotato/__init__.py index 799a2984..8386ed9f 100644 --- a/couchpotato/__init__.py +++ b/couchpotato/__init__.py @@ -1,6 +1,8 @@ from couchpotato.api import api_docs, api_docs_missing from couchpotato.core.auth import requires_auth from couchpotato.core.event import fireEvent +from couchpotato.core.helpers.request import getParams, jsonified +from couchpotato.core.helpers.variable import md5 from couchpotato.core.logger import CPLog from couchpotato.environment import Env from flask.app import Flask @@ -51,6 +53,22 @@ def apiDocs(): del api_docs_missing[''] return render_template('api.html', fireEvent = fireEvent, routes = sorted(routes), api_docs = api_docs, api_docs_missing = sorted(api_docs_missing)) +@web.route('getkey/') +def getApiKey(): + + api = None + params = getParams() + username = Env.setting('username') + password = Env.setting('password') + + if (params.get('u') == md5(username) or not username) and (params.get('p') == password or not password): + api = Env.setting('api_key') + + return jsonified({ + 'success': api is not None, + 'api_key': api + }) + @app.errorhandler(404) def page_not_found(error): index_url = url_for('web.index') diff --git a/couchpotato/templates/api.html b/couchpotato/templates/api.html index ec067f95..f0e76e36 100644 --- a/couchpotato/templates/api.html +++ b/couchpotato/templates/api.html @@ -18,6 +18,12 @@
You can also use the API over another domain using JSONP, the callback function should be in 'callback_func' 
{{ fireEvent('app.api_url', single = True)|safe }}/updater.info/?callback_func=myfunction
+
+
+ Get the API key: + 
/getkey/?p=md5(password)&u=md5(username)
+ Will return {"api_key": "XXXXXXXXXX", "success": true}. When username or password is empty you don't need to md5 it. +
{% for route in routes %} From 263a972b88f0281c2556bb1d0eae4588fe41f3ec Mon Sep 17 00:00:00 2001 From: Ruud Date: Wed, 25 Apr 2012 22:50:05 +0200 Subject: [PATCH 090/105] If downloaders are disabled, keep searching for all qualities. closes #135 --- couchpotato/core/plugins/searcher/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py index 4c26f4ce..530c4ce1 100644 --- a/couchpotato/core/plugins/searcher/main.py +++ b/couchpotato/core/plugins/searcher/main.py @@ -123,7 +123,11 @@ class Searcher(Plugin): for nzb in sorted_results: - return self.download(data = nzb, movie = movie) + downloaded = self.download(data = nzb, movie = movie) + if downloaded: + return True + else: + break else: log.info('Better quality (%s) already available or snatched for %s' % (quality_type['quality']['label'], default_title)) fireEvent('movie.restatus', movie['id']) From d489b574f0b75be03516914aa89668a509e67c08 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 27 Apr 2012 10:05:59 +0200 Subject: [PATCH 091/105] Remove dev check --- couchpotato/core/plugins/scanner/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 469642ce..8b183f3c 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -263,7 +263,7 @@ class Scanner(Plugin): file_too_new = tryInt(time.time() - file_time) break - if file_too_new and not Env.get('dev'): + if file_too_new: log.info('Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s' % (time.ctime(file_time), identifier)) continue From c2cb188d445ee71764ad372c00e1aab2b7a9441c Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 27 Apr 2012 10:23:05 +0200 Subject: [PATCH 092/105] HTTPS for nzbs.org --- couchpotato/core/providers/nzb/nzbs/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/couchpotato/core/providers/nzb/nzbs/main.py b/couchpotato/core/providers/nzb/nzbs/main.py index 4a8d9a06..4281f819 100644 --- a/couchpotato/core/providers/nzb/nzbs/main.py +++ b/couchpotato/core/providers/nzb/nzbs/main.py @@ -13,10 +13,10 @@ log = CPLog(__name__) class Nzbs(NZBProvider, RSS): urls = { - 'download': 'http://nzbs.org/index.php?action=getnzb&nzbid=%s%s', - 'nfo': 'http://nzbs.org/index.php?action=view&nzbid=%s&nfo=1', - 'detail': 'http://nzbs.org/index.php?action=view&nzbid=%s', - 'api': 'http://nzbs.org/rss.php', + 'download': 'https://nzbs.org/index.php?action=getnzb&nzbid=%s%s', + 'nfo': 'https://nzbs.org/index.php?action=view&nzbid=%s&nfo=1', + 'detail': 'https://nzbs.org/index.php?action=view&nzbid=%s', + 'api': 'https://nzbs.org/rss.php', } cat_ids = [ From 7b1ef3c99408a48af74035447a79b93000bfcf14 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 27 Apr 2012 10:53:45 +0200 Subject: [PATCH 093/105] Don't search for empty movies. fixes #153 --- couchpotato/core/providers/movie/couchpotatoapi/main.py | 1 + couchpotato/core/providers/movie/imdbapi/main.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/providers/movie/couchpotatoapi/main.py b/couchpotato/core/providers/movie/couchpotatoapi/main.py index 040bc2e2..26e42df3 100644 --- a/couchpotato/core/providers/movie/couchpotatoapi/main.py +++ b/couchpotato/core/providers/movie/couchpotatoapi/main.py @@ -27,6 +27,7 @@ class CouchPotatoApi(MovieProvider): def getReleaseDate(self, identifier = None): + if identifier is None: return {} try: headers = {'X-CP-Version': fireEvent('app.version', single = True)} data = self.urlopen((self.api_url % ('eta')) + (identifier + '/'), headers = headers) diff --git a/couchpotato/core/providers/movie/imdbapi/main.py b/couchpotato/core/providers/movie/imdbapi/main.py index bde256a9..568b413e 100644 --- a/couchpotato/core/providers/movie/imdbapi/main.py +++ b/couchpotato/core/providers/movie/imdbapi/main.py @@ -27,7 +27,7 @@ class IMDBAPI(MovieProvider): name_year = fireEvent('scanner.name_year', q, single = True) - if not name_year.get('name'): + if not q or not name_year.get('name'): return [] cache_key = 'imdbapi.cache.%s' % q @@ -45,6 +45,9 @@ class IMDBAPI(MovieProvider): def getInfo(self, identifier = None): + if not identifier: + return {} + cache_key = 'imdbapi.cache.%s' % identifier cached = self.getCache(cache_key, self.urls['info'] % identifier) From aed6e4ab06c11b0d042129bb2fc5c5a301a26e1d Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 27 Apr 2012 11:07:59 +0200 Subject: [PATCH 094/105] Merge pull request #152 from garlandkr/master Removed group setting for init --- init/ubuntu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/ubuntu b/init/ubuntu index 8abc9e79..d6af148a 100644 --- a/init/ubuntu +++ b/init/ubuntu @@ -45,7 +45,7 @@ case "$1" in start) echo "Starting $DESC" rm -rf $PID_PATH || return 1 - install -d --mode=0755 -o $RUN_AS -g $RUN_AS $PID_PATH || return 1 + install -d --mode=0755 -o $RUN_AS $PID_PATH || return 1 start-stop-daemon -d $APP_PATH -c $RUN_AS --start --background --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS ;; stop) From af6026666218de02bd69ca51beca62955098be82 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 27 Apr 2012 11:18:43 +0200 Subject: [PATCH 095/105] Check if file exists before adding it to movie_files. fixes #146 --- couchpotato/core/plugins/scanner/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/couchpotato/core/plugins/scanner/main.py b/couchpotato/core/plugins/scanner/main.py index 8b183f3c..e71d5c20 100644 --- a/couchpotato/core/plugins/scanner/main.py +++ b/couchpotato/core/plugins/scanner/main.py @@ -165,6 +165,9 @@ class Scanner(Plugin): for file_path in files: + if not os.path.exists(file_path): + continue + # Remove ignored files if self.isSampleFile(file_path): leftovers.append(file_path) From b2ad1f12b5b0f0938e2d7444ae16ba0a1ec88b66 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 27 Apr 2012 13:00:46 +0200 Subject: [PATCH 096/105] Wrong logging format --- couchpotato/core/plugins/renamer/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/renamer/main.py b/couchpotato/core/plugins/renamer/main.py index 50966417..672ad8ea 100644 --- a/couchpotato/core/plugins/renamer/main.py +++ b/couchpotato/core/plugins/renamer/main.py @@ -328,7 +328,7 @@ class Renamer(Plugin): try: os.remove(src) except: - log.error('Failed removing %s: %s', (src, traceback.format_exc())) + log.error('Failed removing %s: %s' % (src, traceback.format_exc())) # Remove matching releases for release in remove_releases: @@ -336,14 +336,14 @@ class Renamer(Plugin): try: db.delete(release) except: - log.error('Failed removing %s: %s', (release.identifier, traceback.format_exc())) + log.error('Failed removing %s: %s' % (release.identifier, traceback.format_exc())) if group['dirname'] and group['parentdir']: try: log.info('Deleting folder: %s' % group['parentdir']) self.deleteEmptyFolder(group['parentdir']) except: - log.error('Failed removing %s: %s', (group['parentdir'], traceback.format_exc())) + log.error('Failed removing %s: %s' % (group['parentdir'], traceback.format_exc())) # Search for trailers etc fireEventAsync('renamer.after', group) From 7df659a3029d50a92a2f1500f4a80710d1e66344 Mon Sep 17 00:00:00 2001 From: Ruud Date: Fri, 27 Apr 2012 13:26:12 +0200 Subject: [PATCH 097/105] Kickasstorrent fix #156 --- couchpotato/core/providers/torrent/kickasstorrents/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/torrent/kickasstorrents/main.py b/couchpotato/core/providers/torrent/kickasstorrents/main.py index 8a2fe76c..dd5e375a 100644 --- a/couchpotato/core/providers/torrent/kickasstorrents/main.py +++ b/couchpotato/core/providers/torrent/kickasstorrents/main.py @@ -127,7 +127,7 @@ class KickAssTorrents(TorrentProvider): return tryInt(age) def download(self, url = '', nzb_id = ''): - compressed_data = super(KickAssTorrents, self).download(url = url, nzb_id = nzb_id) + compressed_data = self.urlopen(url = url, headers = {'Referer': 'http://kat.ph/'}) compressedstream = StringIO.StringIO(compressed_data) gzipper = gzip.GzipFile(fileobj = compressedstream) From 0e9c086b2fcc51afe133d7bd28cb25ce7bf96a81 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Apr 2012 19:52:07 +0200 Subject: [PATCH 098/105] Show status of release. fixes #159 #165 --- couchpotato/core/plugins/movie/static/movie.css | 5 ++++- couchpotato/core/plugins/movie/static/movie.js | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/plugins/movie/static/movie.css b/couchpotato/core/plugins/movie/static/movie.css index 9b82598e..56521c26 100644 --- a/couchpotato/core/plugins/movie/static/movie.css +++ b/couchpotato/core/plugins/movie/static/movie.css @@ -279,7 +279,8 @@ border: 0; } .movies .options .table .provider { - width: 130px; + width: 120px; + text-overflow: ellipsis; } .movies .options .table .name { width: 350px; @@ -290,6 +291,8 @@ .movies .options .table.files .name { width: 605px; } .movies .options .table .type { width: 130px; } .movies .options .table .is_available { width: 90px; } + .movies .options .table .age, + .movies .options .table .size { width: 40px; } .movies .options .table a { width: 30px !important; diff --git a/couchpotato/core/plugins/movie/static/movie.js b/couchpotato/core/plugins/movie/static/movie.js index bbe526df..5544ed4e 100644 --- a/couchpotato/core/plugins/movie/static/movie.js +++ b/couchpotato/core/plugins/movie/static/movie.js @@ -270,8 +270,9 @@ var ReleaseAction = new Class({ // Header new Element('div.item.head').adopt( new Element('span.name', {'text': 'Release name'}), + new Element('span.status', {'text': 'Status'}), new Element('span.quality', {'text': 'Quality'}), - new Element('span.size', {'text': 'Size (MB)'}), + new Element('span.size', {'text': 'Size'}), new Element('span.age', {'text': 'Age'}), new Element('span.score', {'text': 'Score'}), new Element('span.provider', {'text': 'Provider'}) @@ -288,9 +289,10 @@ var ReleaseAction = new Class({ } catch(e){} new Element('div', { - 'class': 'item ' + status.identifier + 'class': 'item' }).adopt( new Element('span.name', {'text': self.get(release, 'name'), 'title': self.get(release, 'name')}), + new Element('span.status', {'text': status.identifier, 'class': 'release_status '+status.identifier}), new Element('span.quality', {'text': quality.get('label')}), new Element('span.size', {'text': (self.get(release, 'size') || 'unknown')}), new Element('span.age', {'text': self.get(release, 'age')}), From 7d68c5ad89404cb25a4ab9036d31943e670274c5 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Apr 2012 19:59:30 +0200 Subject: [PATCH 099/105] Return branch in update.info. closes #161 --- couchpotato/core/_base/updater/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index f5182228..eb34906c 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -96,7 +96,8 @@ class BaseUpdater(Plugin): 'last_check': self.last_check, 'update_version': self.update_version, 'version': self.getVersion(), - 'repo_name': '%s/%s' % (self.repo_user, self.repo_name) + 'repo_name': '%s/%s' % (self.repo_user, self.repo_name), + 'branch': self.branch, } def check(self): From ab5bfd704b32521cbd46519567a2f144d63ec35d Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Apr 2012 20:04:14 +0200 Subject: [PATCH 100/105] Relative docs url --- couchpotato/core/_base/_core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/_base/_core/__init__.py b/couchpotato/core/_base/_core/__init__.py index d90fa59b..2aca8e2b 100644 --- a/couchpotato/core/_base/_core/__init__.py +++ b/couchpotato/core/_base/_core/__init__.py @@ -55,7 +55,7 @@ config = [{ 'name': 'api_key', 'default': uuid4().hex, 'readonly': 1, - 'description': 'Let 3rd party app do stuff. Docs', + 'description': 'Let 3rd party app do stuff. Docs', }, { 'name': 'debug', From a2e364a7d7334047dbbd0bc18aec48002730357e Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Apr 2012 20:13:17 +0200 Subject: [PATCH 101/105] Show normal file path --- couchpotato/core/plugins/quality/main.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index 3c5ad7be..e38b8182 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -161,26 +161,25 @@ class QualityPlugin(Plugin): for cur_file in files: size = (os.path.getsize(cur_file) / 1024 / 1024) if os.path.isfile(cur_file) else 0 words = re.split('\W+', cur_file.lower()) - safe_cur_file = toSafeString(cur_file) for quality in self.all(): # Check tags if quality['identifier'] in words: - log.debug('Found via identifier "%s" in %s' % (quality['identifier'], safe_cur_file)) + log.debug('Found via identifier "%s" in %s' % (quality['identifier'], cur_file)) return self.setCache(hash, quality) if list(set(quality.get('alternative', [])) & set(words)): - log.debug('Found %s via alt %s in %s' % (quality['identifier'], quality.get('alternative'), safe_cur_file)) + log.debug('Found %s via alt %s in %s' % (quality['identifier'], quality.get('alternative'), cur_file)) return self.setCache(hash, quality) for tag in quality.get('tags', []): if isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words): - log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), safe_cur_file)) + log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file)) return self.setCache(hash, quality) if list(set(quality.get('tags', [])) & set(words)): - log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), safe_cur_file)) + log.debug('Found %s via tag %s in %s' % (quality['identifier'], quality.get('tags'), cur_file)) return self.setCache(hash, quality) # Check on unreliable stuff From 121cb3c7b40bac4295fbbf115edfd322fb9d608b Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Apr 2012 20:23:20 +0200 Subject: [PATCH 102/105] Remove update message after update. fixes #163 --- couchpotato/core/_base/updater/static/updater.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/couchpotato/core/_base/updater/static/updater.js b/couchpotato/core/_base/updater/static/updater.js index 202540d3..899f8796 100644 --- a/couchpotato/core/_base/updater/static/updater.js +++ b/couchpotato/core/_base/updater/static/updater.js @@ -79,6 +79,8 @@ var UpdaterBase = new Class({ if(json.success){ App.restart('Please wait while CouchPotato is being updated with more awesome stuff.', 'Updating'); App.checkAvailable.delay(500, App); + if(self.message) + self.message.destroy(); } } }); From 05b294c2312a0b6bf08356de81f7aa8f30ca61e7 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Apr 2012 22:14:51 +0200 Subject: [PATCH 103/105] Make sure KAT downloads the correct torrent. fixes #166 --- couchpotato/core/providers/base.py | 8 ++++++-- .../core/providers/torrent/kickasstorrents/main.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py index 64e7011f..826c7d96 100644 --- a/couchpotato/core/providers/base.py +++ b/couchpotato/core/providers/base.py @@ -65,9 +65,13 @@ class YarrProvider(Provider): def belongsTo(self, url, host = None): try: hostname = urlparse(url).hostname - download_url = host if host else self.urls['download'] - if hostname in download_url: + if host and hostname in host: return self + else: + for url_type in self.urls: + download_url = self.urls[url_type] + if hostname in download_url: + return self except: log.debug('Url % s doesn\'t belong to %s' % (url, self.getName())) diff --git a/couchpotato/core/providers/torrent/kickasstorrents/main.py b/couchpotato/core/providers/torrent/kickasstorrents/main.py index dd5e375a..6a039c71 100644 --- a/couchpotato/core/providers/torrent/kickasstorrents/main.py +++ b/couchpotato/core/providers/torrent/kickasstorrents/main.py @@ -17,6 +17,7 @@ class KickAssTorrents(TorrentProvider): 'test': 'http://www.kat.ph/', 'detail': 'http://www.kat.ph/%s-t%s.html', 'search': 'http://www.kat.ph/%s-i%s/', + 'download': 'http://torcache.net/', } cat_ids = [ From cdfad20cb280500b3fcc860b9a7704f89e941aac Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Apr 2012 22:40:37 +0200 Subject: [PATCH 104/105] Newzbin DTD update. fixes #157 --- couchpotato/core/providers/nzb/newzbin/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/couchpotato/core/providers/nzb/newzbin/main.py b/couchpotato/core/providers/nzb/newzbin/main.py index c25c2664..d68bbae3 100644 --- a/couchpotato/core/providers/nzb/newzbin/main.py +++ b/couchpotato/core/providers/nzb/newzbin/main.py @@ -89,7 +89,7 @@ class Newzbin(NZBProvider, RSS): title = self.getTextElement(nzb, "title") if 'error' in title.lower(): continue - REPORT_NS = 'http://www.newzbin.com/DTD/2007/feeds/report/'; + REPORT_NS = 'http://www.newzbin2.es/DTD/2007/feeds/report/'; # Add attributes to name try: From c7bc0f43790916b9ec8f9ae20a4cafdfc1ad861e Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 28 Apr 2012 22:57:26 +0200 Subject: [PATCH 105/105] Customize meta renaming. closes #158 --- .../core/providers/metadata/xbmc/__init__.py | 22 +++++++++++++++++++ .../core/providers/metadata/xbmc/main.py | 6 ++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/couchpotato/core/providers/metadata/xbmc/__init__.py b/couchpotato/core/providers/metadata/xbmc/__init__.py index 2a9510e5..71de827a 100644 --- a/couchpotato/core/providers/metadata/xbmc/__init__.py +++ b/couchpotato/core/providers/metadata/xbmc/__init__.py @@ -20,19 +20,41 @@ config = [{ }, { 'name': 'meta_nfo', + 'label': 'NFO', 'default': True, 'type': 'bool', }, + { + 'name': 'meta_nfo_name', + 'label': 'NFO filename', + 'default': '%s.nfo', + 'advanced': True, + 'description': '%s is the rootname of the movie. For example "/path/to/movie cd1.mkv" will be "/path/to/movie"' + }, { 'name': 'meta_fanart', + 'label': 'Fanart', 'default': True, 'type': 'bool', }, + { + 'name': 'meta_fanart_name', + 'label': 'Fanart filename', + 'default': '%s-fanart.jpg', + 'advanced': True, + }, { 'name': 'meta_thumbnail', + 'label': 'Thumbnail', 'default': True, 'type': 'bool', }, + { + 'name': 'meta_thumbnail_name', + 'label': 'Thumbnail filename', + 'default': '%s.tbn', + 'advanced': True, + }, ], }, ], diff --git a/couchpotato/core/providers/metadata/xbmc/main.py b/couchpotato/core/providers/metadata/xbmc/main.py index 2908c6f6..7797cce8 100644 --- a/couchpotato/core/providers/metadata/xbmc/main.py +++ b/couchpotato/core/providers/metadata/xbmc/main.py @@ -15,13 +15,13 @@ class XBMC(MetaDataBase): return os.path.join(data['destination_dir'], data['filename']) def getFanartName(self, root): - return '%s-fanart.jpg' % root + return self.conf('meta_fanart_name') % root def getThumbnailName(self, root): - return '%s.tbn' % root + return self.conf('meta_thumbnail_name') % root def getNfoName(self, root): - return '%s.nfo' % root + return self.conf('meta_nfo_name') % root def getNfo(self, movie_info = {}, data = {}): nfoxml = Element('movie')