[12]{1}[0-9]{3})', text)
if matches:
return matches.group('year')
return ''
- def getReleaseNameYear(self, release_name):
+ def getReleaseNameYear(self, release_name, file_name = None):
+
+ # Use guessit first
+ if file_name:
+ try:
+ guess = guess_movie_info(file_name)
+ if guess.get('title') and guess.get('year'):
+ return {
+ 'name': guess.get('title'),
+ 'year': guess.get('year'),
+ }
+ except:
+ log.debug('Could not detect via guessit "%s": %s' % (file_name, traceback.format_exc()))
+
+ # Backup to simple
cleaned = ' '.join(re.split('\W+', simplifyString(release_name)))
cleaned = re.sub(self.clean, ' ', cleaned)
year = self.findYear(cleaned)
@@ -661,7 +723,7 @@ class Scanner(Plugin):
movie_name = cleaned.split(year).pop(0).strip()
return {
'name': movie_name,
- 'year': year,
+ 'year': int(year),
}
except:
pass
@@ -670,7 +732,7 @@ class Scanner(Plugin):
movie_name = cleaned.split(' ').pop(0).strip()
return {
'name': movie_name,
- 'year': year,
+ 'year': int(year),
}
except:
pass
diff --git a/couchpotato/core/plugins/score/scores.py b/couchpotato/core/plugins/score/scores.py
index 1a2f2b88..2cace048 100644
--- a/couchpotato/core/plugins/score/scores.py
+++ b/couchpotato/core/plugins/score/scores.py
@@ -40,7 +40,7 @@ def nameScore(name, year):
# Contains preferred word
nzb_words = re.split('\W+', simplifyString(name))
- preferred_words = Env.setting('preferred_words', section = 'searcher').split(',')
+ preferred_words = [x.strip() for x in Env.setting('preferred_words', section = 'searcher').split(',')]
for word in preferred_words:
if word.strip() and word.strip().lower() in nzb_words:
score = score + 100
diff --git a/couchpotato/core/plugins/searcher/__init__.py b/couchpotato/core/plugins/searcher/__init__.py
index 8e0f2cf9..a3f88555 100644
--- a/couchpotato/core/plugins/searcher/__init__.py
+++ b/couchpotato/core/plugins/searcher/__init__.py
@@ -77,7 +77,7 @@ config = [{
'options': [
{
'name': 'retention',
- 'default': 350,
+ 'default': 1000,
'type': 'int',
'unit': 'days'
},
diff --git a/couchpotato/core/plugins/searcher/main.py b/couchpotato/core/plugins/searcher/main.py
index f3dc28fa..530c4ce1 100644
--- a/couchpotato/core/plugins/searcher/main.py
+++ b/couchpotato/core/plugins/searcher/main.py
@@ -7,7 +7,9 @@ from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Movie, Release, ReleaseInfo
from couchpotato.environment import Env
from sqlalchemy.exc import InterfaceError
+import datetime
import re
+import time
import traceback
log = CPLog(__name__)
@@ -15,6 +17,8 @@ log = CPLog(__name__)
class Searcher(Plugin):
+ in_progress = False
+
def __init__(self):
addEvent('searcher.all', self.all_movies)
addEvent('searcher.single', self.single)
@@ -26,6 +30,12 @@ class Searcher(Plugin):
def all_movies(self):
+ if self.in_progress:
+ log.info('Search already in progress')
+ return
+
+ self.in_progress = True
+
db = get_session()
movies = db.query(Movie).filter(
@@ -51,12 +61,19 @@ class Searcher(Plugin):
if self.shuttingDown():
break
+ self.in_progress = False
+
def single(self, movie):
+ pre_releases = fireEvent('quality.pre_releases', single = True)
+ release_dates = fireEvent('library.update_release_date', identifier = movie['library']['identifier'], merge = True)
available_status = fireEvent('status.get', 'available', single = True)
default_title = movie['library']['titles'][0]['title']
for quality_type in movie['profile']['types']:
+ if not self.couldBeReleased(quality_type['quality']['identifier'], release_dates, pre_releases):
+ log.info('To early to search for %s, %s' % (quality_type['quality']['identifier'], default_title))
+ continue
has_better_quality = 0
@@ -72,6 +89,8 @@ class Searcher(Plugin):
quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True)
results = fireEvent('yarr.search', movie, quality, merge = True)
sorted_results = sorted(results, key = lambda k: k['score'], reverse = True)
+ if len(sorted_results) == 0:
+ log.debug('Nothing found for %s in %s' % (default_title, quality_type['quality']['label']))
# Add them to this movie releases list
for nzb in sorted_results:
@@ -104,7 +123,11 @@ class Searcher(Plugin):
for nzb in sorted_results:
- return self.download(data = nzb, movie = movie)
+ downloaded = self.download(data = nzb, movie = movie)
+ if downloaded:
+ return True
+ else:
+ break
else:
log.info('Better quality (%s) already available or snatched for %s' % (quality_type['quality']['label'], default_title))
fireEvent('movie.restatus', movie['id'])
@@ -116,10 +139,10 @@ class Searcher(Plugin):
return False
- def download(self, data, movie):
+ def download(self, data, movie, manual = False):
snatched_status = fireEvent('status.get', 'snatched', single = True)
- successful = fireEvent('download', data = data, movie = movie, single = True)
+ successful = fireEvent('download', data = data, movie = movie, manual = manual, single = True)
if successful:
@@ -171,24 +194,31 @@ class Searcher(Plugin):
log.info('Wrong: Outside retention, age is %s, needs %s or lower: %s' % (nzb['age'], retention, nzb['name']))
return False
- nzb_words = re.split('\W+', simplifyString(nzb['name']))
- required_words = self.conf('required_words').split(',')
+ movie_name = simplifyString(nzb['name'])
+ nzb_words = re.split('\W+', movie_name)
+ required_words = [x.strip() for x in self.conf('required_words').split(',')]
if self.conf('required_words') and not list(set(nzb_words) & set(required_words)):
log.info("NZB doesn't contain any of the required words.")
return False
- ignored_words = self.conf('ignored_words').split(',')
+ ignored_words = [x.strip() for x in self.conf('ignored_words').split(',')]
blacklisted = list(set(nzb_words) & set(ignored_words))
if self.conf('ignored_words') and blacklisted:
log.info("Wrong: '%s' blacklisted words: %s" % (nzb['name'], ", ".join(blacklisted)))
return False
+ pron_tags = ['xxx', 'sex', 'anal', 'tits', 'fuck', 'porn', 'orgy', 'milf', 'boobs']
+ for p_tag in pron_tags:
+ if p_tag in movie_name:
+ log.info('Wrong: %s, probably pr0n' % (nzb['name']))
+ return False
+
#qualities = fireEvent('quality.all', single = True)
preferred_quality = fireEvent('quality.single', identifier = quality['identifier'], single = True)
# Contains lower quality string
- if self.containsOtherQuality(nzb['name'], preferred_quality, single_category):
+ if self.containsOtherQuality(nzb, movie_year = movie['library']['year'], preferred_quality = preferred_quality, single_category = single_category):
log.info('Wrong: %s, looking for %s' % (nzb['name'], quality['label']))
return False
@@ -230,8 +260,10 @@ class Searcher(Plugin):
log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'" % (nzb['name'], movie['library']['titles'][0]['title'], movie['library']['year']))
return False
- def containsOtherQuality(self, name, preferred_quality = {}, single_category = False):
+ def containsOtherQuality(self, nzb, movie_year = None, preferred_quality = {}, single_category = False):
+ name = nzb['name']
+ size = nzb.get('size', 0)
nzb_words = re.split('\W+', simplifyString(name))
qualities = fireEvent('quality.all', single = True)
@@ -246,6 +278,14 @@ class Searcher(Plugin):
if list(set(nzb_words) & set(quality['alternative'])):
found[quality['identifier']] = True
+ # Hack for older movies that don't contain quality tag
+ year_name = fireEvent('scanner.name_year', name, single = True)
+ if movie_year < datetime.datetime.now().year - 3 and not year_name.get('year', None):
+ if size > 3000: # Assume dvdr
+ return 'dvdr' == preferred_quality['identifier']
+ else: # Assume dvdrip
+ return 'dvdrip' == preferred_quality['identifier']
+
# Allow other qualities
for allowed in preferred_quality.get('allow'):
if found.get(allowed):
@@ -284,10 +324,10 @@ class Searcher(Plugin):
check_movie = fireEvent('scanner.name_year', check_name, single = True)
try:
- check_words = re.split('\W+', check_movie.get('name', ''))
- movie_words = re.split('\W+', simplifyString(movie_name))
+ check_words = filter(None, re.split('\W+', check_movie.get('name', '')))
+ movie_words = filter(None, re.split('\W+', simplifyString(movie_name)))
- if len(list(set(check_words) - set(movie_words))) == 0:
+ if len(check_words) > 0 and len(movie_words) > 0 and len(list(set(check_words) - set(movie_words))) == 0:
return True
except:
pass
@@ -306,3 +346,30 @@ class Searcher(Plugin):
pass
return nfo and getImdb(nfo) == imdb_id
+
+ def couldBeReleased(self, wanted_quality, dates, pre_releases):
+
+ now = int(time.time())
+
+ if not dates or (dates.get('theater', 0) == 0 and dates.get('dvd', 0) == 0):
+ return True
+ else:
+ if wanted_quality in pre_releases:
+ # Prerelease 1 week before theaters
+ if dates.get('theater') >= now - 604800 and wanted_quality in pre_releases:
+ return True
+ else:
+ # 6 weeks after theater release
+ if dates.get('theater') < now - 3628800:
+ return True
+
+ # 6 weeks before dvd release
+ if dates.get('dvd') > now - 3628800:
+ return True
+
+ # Dvd should be released
+ if dates.get('dvd') > 0 and dates.get('dvd') < now:
+ return True
+
+
+ return False
diff --git a/couchpotato/core/plugins/subtitle/__init__.py b/couchpotato/core/plugins/subtitle/__init__.py
index 903e934e..858ce9d6 100644
--- a/couchpotato/core/plugins/subtitle/__init__.py
+++ b/couchpotato/core/plugins/subtitle/__init__.py
@@ -20,14 +20,14 @@ config = [{
},
{
'name': 'languages',
- 'description': 'The languages you want to download the sub',
- },
- {
- 'name': 'automatic',
- 'default': True,
- 'type': 'bool',
- 'description': 'Automaticly search & download for movies in library',
+ 'description': 'Comma separated, 2 letter country code. Example: en, nl',
},
+# {
+# 'name': 'automatic',
+# 'default': True,
+# 'type': 'bool',
+# 'description': 'Automaticly search & download for movies in library',
+# },
],
},
],
diff --git a/couchpotato/core/plugins/subtitle/main.py b/couchpotato/core/plugins/subtitle/main.py
index 45f87dbf..da86d493 100644
--- a/couchpotato/core/plugins/subtitle/main.py
+++ b/couchpotato/core/plugins/subtitle/main.py
@@ -3,15 +3,18 @@ from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.core.settings.model import Library, FileType
+from couchpotato.environment import Env
+import subliminal
log = CPLog(__name__)
class Subtitle(Plugin):
+ services = ['opensubtitles', 'thesubdb', 'subswiki']
+
def __init__(self):
- pass
- #addEvent('renamer.before', self.searchSingle)
+ addEvent('renamer.before', self.searchSingle)
def searchLibrary(self):
@@ -33,20 +36,22 @@ class Subtitle(Plugin):
files.append(file.path);
# get subtitles for those files
- subtitles = fireEvent('subtitle.search', files = files, languages = self.getLanguages(), merge = True)
-
- # do something with the returned subtitles
- print subtitles
-
+ subliminal.list_subtitles(files, cache_dir = Env.get('cache_dir'), multi = True, languages = self.getLanguages(), services = self.services)
def searchSingle(self, group):
if self.isDisabled(): return
- subtitles = fireEvent('subtitle.search', files = group['files']['movie'], languages = self.getLanguages(), merge = True)
+ available_languages = sum(group['subtitle_language'].itervalues(), [])
+ downloaded = []
+ for lang in self.getLanguages():
+ if lang not in available_languages:
+ download = subliminal.download_subtitles(group['files']['movie'], multi = True, force = False, languages = [lang], services = self.services, cache_dir = Env.get('cache_dir'))
+ downloaded.extend(download)
- # do something with the returned subtitles
- print subtitles
+ for d_sub in downloaded:
+ group['files']['subtitle'].add(d_sub.path)
+ group['subtitle_language'][d_sub.path] = [d_sub.language]
def getLanguages(self):
- return self.conf('languages').split(',')
+ return [x.strip() for x in self.conf('languages').split(',')]
diff --git a/couchpotato/core/plugins/userscript/template.js b/couchpotato/core/plugins/userscript/template.js
index d2d58adf..090a62ba 100644
--- a/couchpotato/core/plugins/userscript/template.js
+++ b/couchpotato/core/plugins/userscript/template.js
@@ -58,7 +58,7 @@ if (typeof GM_addStyle == 'undefined'){
// Styles
GM_addStyle('\
#cp_popup { font-family: "Helvetica Neue", Helvetica, Arial, Geneva, sans-serif; -moz-border-radius: 6px 0px 0px 6px; -webkit-border-radius: 6px 0px 0px 6px; border-radius: 6px 0px 0px 6px; -moz-box-shadow: 0 0 20px rgba(0,0,0,0.5); -webkit-box-shadow: 0 0 20px rgba(0,0,0,0.5); box-shadow: 0 0 20px rgba(0,0,0,0.5); position:fixed; z-index:9999; bottom:0; right:0; font-size:15px; margin: 20px 0; display: block; background:#4E5969; } \
- #cp_popup:hover { } \
+ #cp_popup.opened { width: 492px; } \
#cp_popup a#add_to { cursor:pointer; text-align:center; text-decoration:none; color: #000; display:block; padding:5px 0 5px 5px; } \
#cp_popup a#close_button { cursor:pointer; float: right; padding:120px 10px 10px; } \
#cp_popup a img { vertical-align: middle; } \
@@ -98,12 +98,14 @@ var osd = function(){
catch(e){}
popup.innerHTML = '';
+ popup.setAttribute('class', 'opened');
popup.appendChild(create('a', {
'innerHTML': '
',
'id': 'close_button',
'onclick': function(){
popup.innerHTML = '';
popup.appendChild(add_button);
+ popup.setAttribute('class', '');
}
}));
popup.appendChild(iframe)
diff --git a/couchpotato/core/providers/automation/imdb/__init__.py b/couchpotato/core/providers/automation/imdb/__init__.py
index 86561e28..338a8b62 100644
--- a/couchpotato/core/providers/automation/imdb/__init__.py
+++ b/couchpotato/core/providers/automation/imdb/__init__.py
@@ -10,7 +10,7 @@ config = [{
'tab': 'automation',
'name': 'imdb_automation',
'label': 'IMDB',
- 'description': 'From any public IMDB watchlists',
+ 'description': 'From any public IMDB watchlists',
'options': [
{
'name': 'automation_enabled',
diff --git a/couchpotato/core/providers/automation/kinepolis/__init__.py b/couchpotato/core/providers/automation/kinepolis/__init__.py
deleted file mode 100644
index 3d4a156c..00000000
--- a/couchpotato/core/providers/automation/kinepolis/__init__.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from .main import Kinepolis
-
-def start():
- return Kinepolis()
-
-config = [{
- 'name': 'kinepolis',
- 'groups': [
- {
- 'tab': 'automation',
- 'name': 'kinepolis_automation',
- 'label': 'Kinepolis',
- 'description': 'from Kinepolis',
- 'options': [
- {
- 'name': 'automation_enabled',
- 'default': False,
- 'type': 'enabler',
- },
- {
- 'name': 'automation_use_requirements',
- 'label': 'Use requirements',
- 'description': 'Use the minimal requirements set above',
- 'default': True,
- 'advanced': True,
- 'type': 'bool',
- },
- ],
- },
- ],
-}]
diff --git a/couchpotato/core/providers/automation/kinepolis/main.py b/couchpotato/core/providers/automation/kinepolis/main.py
deleted file mode 100644
index 08e0c692..00000000
--- a/couchpotato/core/providers/automation/kinepolis/main.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from couchpotato.core.helpers.rss import RSS
-from couchpotato.core.logger import CPLog
-from couchpotato.core.providers.automation.base import Automation
-import xml.etree.ElementTree as XMLTree
-
-log = CPLog(__name__)
-
-
-class Kinepolis(Automation, RSS):
-
- urls = {
- 'top10': 'http://kinepolis.be/nl/top10-box-office/feed',
- }
-
-
- def getIMDBids(self):
-
- if self.isDisabled():
- return
-
- movies = []
- for key in self.urls:
- url = self.urls[key]
- cache_key = 'kinepolis.%s' % key
-
- rss_data = self.getCache(cache_key, url)
- try:
- items = self.getElements(XMLTree.fromstring(rss_data), 'channel/item')
- except Exception, e:
- log.debug('%s, %s' % (self.getName(), e))
- continue
-
- for item in items:
- title = self.getTextElement(item, "title").lower()
- result = self.search(title)
-
- if result:
- if not self.conf('automation_use_requirements') or self.isMinimal(result):
- movies.append(result)
-
- return movies
diff --git a/couchpotato/core/providers/automation/trakt/__init__.py b/couchpotato/core/providers/automation/trakt/__init__.py
index f1c30dfc..88ab454a 100644
--- a/couchpotato/core/providers/automation/trakt/__init__.py
+++ b/couchpotato/core/providers/automation/trakt/__init__.py
@@ -10,7 +10,7 @@ config = [{
'tab': 'automation',
'name': 'trakt_automation',
'label': 'Trakt',
- 'description': 'from watchlist',
+ 'description': 'import movies from your own watchlist',
'options': [
{
'name': 'automation_enabled',
diff --git a/couchpotato/core/providers/base.py b/couchpotato/core/providers/base.py
index 0d00179b..826c7d96 100644
--- a/couchpotato/core/providers/base.py
+++ b/couchpotato/core/providers/base.py
@@ -19,7 +19,7 @@ class Provider(Plugin):
def isAvailable(self, test_url):
- if Env.get('debug'): return True
+ if Env.get('dev'): return True
now = time.time()
host = urlparse(test_url).hostname
@@ -65,9 +65,13 @@ class YarrProvider(Provider):
def belongsTo(self, url, host = None):
try:
hostname = urlparse(url).hostname
- download_url = host if host else self.urls['download']
- if hostname in download_url:
+ if host and hostname in host:
return self
+ else:
+ for url_type in self.urls:
+ download_url = self.urls[url_type]
+ if hostname in download_url:
+ return self
except:
log.debug('Url % s doesn\'t belong to %s' % (url, self.getName()))
diff --git a/couchpotato/core/providers/metadata/mediabrowser/__init__.py b/couchpotato/core/providers/metadata/mediabrowser/__init__.py
deleted file mode 100644
index c061ce3c..00000000
--- a/couchpotato/core/providers/metadata/mediabrowser/__init__.py
+++ /dev/null
@@ -1,39 +0,0 @@
-from .main import MediaBrowser
-
-def start():
- return MediaBrowser()
-
-config = [{
- 'name': 'mediabrowser',
- 'groups': [
- {
- 'tab': 'renamer',
- 'subtab': 'metadata',
- 'name': 'mediabrowser_metadata',
- 'label': 'MediaBrowser',
- 'description': 'Enable metadata MediaBrowser can understand',
- 'options': [
- {
- 'name': 'meta_enabled',
- 'default': False,
- 'type': 'enabler',
- },
- {
- 'name': 'meta_nfo',
- 'default': True,
- 'type': 'bool',
- },
- {
- 'name': 'meta_fanart',
- 'default': True,
- 'type': 'bool',
- },
- {
- 'name': 'meta_thumbnail',
- 'default': True,
- 'type': 'bool',
- },
- ],
- },
- ],
-}]
diff --git a/couchpotato/core/providers/metadata/mediabrowser/main.py b/couchpotato/core/providers/metadata/mediabrowser/main.py
deleted file mode 100644
index 52a83d00..00000000
--- a/couchpotato/core/providers/metadata/mediabrowser/main.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from couchpotato.core.providers.metadata.base import MetaDataBase
-
-class MediaBrowser(MetaDataBase):
-
- pass
diff --git a/couchpotato/core/providers/metadata/sonyps3/__init__.py b/couchpotato/core/providers/metadata/sonyps3/__init__.py
deleted file mode 100644
index 002b8487..00000000
--- a/couchpotato/core/providers/metadata/sonyps3/__init__.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from .main import SonyPS3
-
-def start():
- return SonyPS3()
-
-config = [{
- 'name': 'sonyps3',
- 'groups': [
- {
- 'tab': 'renamer',
- 'subtab': 'metadata',
- 'name': 'sonyps3_metadata',
- 'label': 'Sony PS3',
- 'description': 'Enable metadata your Playstation 3 can understand',
- 'options': [
- {
- 'name': 'meta_enabled',
- 'default': False,
- 'type': 'enabler',
- },
- {
- 'name': 'meta_thumbnail',
- 'default': True,
- 'type': 'bool',
- },
- ],
- },
- ],
-}]
diff --git a/couchpotato/core/providers/metadata/sonyps3/main.py b/couchpotato/core/providers/metadata/sonyps3/main.py
deleted file mode 100644
index 58fdd7fd..00000000
--- a/couchpotato/core/providers/metadata/sonyps3/main.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from couchpotato.core.providers.metadata.base import MetaDataBase
-
-class SonyPS3(MetaDataBase):
-
- pass
diff --git a/couchpotato/core/providers/metadata/wdtv/__init__.py b/couchpotato/core/providers/metadata/wdtv/__init__.py
deleted file mode 100644
index b3dab6e7..00000000
--- a/couchpotato/core/providers/metadata/wdtv/__init__.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from .main import WDTV
-
-def start():
- return WDTV()
-
-config = [{
- 'name': 'wdtv',
- 'groups': [
- {
- 'tab': 'renamer',
- 'subtab': 'metadata',
- 'name': 'wdtv_metadata',
- 'label': 'WDTV',
- 'description': 'Enable metadata WDTV can understand',
- 'options': [
- {
- 'name': 'meta_enabled',
- 'default': False,
- 'type': 'enabler',
- },
- {
- 'name': 'meta_thumbnail',
- 'default': True,
- 'type': 'bool',
- },
- ],
- },
- ],
-}]
diff --git a/couchpotato/core/providers/metadata/wdtv/main.py b/couchpotato/core/providers/metadata/wdtv/main.py
deleted file mode 100644
index 74322a0b..00000000
--- a/couchpotato/core/providers/metadata/wdtv/main.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from couchpotato.core.providers.metadata.base import MetaDataBase
-
-class WDTV(MetaDataBase):
-
- pass
diff --git a/couchpotato/core/providers/metadata/xbmc/__init__.py b/couchpotato/core/providers/metadata/xbmc/__init__.py
index 2a9510e5..71de827a 100644
--- a/couchpotato/core/providers/metadata/xbmc/__init__.py
+++ b/couchpotato/core/providers/metadata/xbmc/__init__.py
@@ -20,19 +20,41 @@ config = [{
},
{
'name': 'meta_nfo',
+ 'label': 'NFO',
'default': True,
'type': 'bool',
},
+ {
+ 'name': 'meta_nfo_name',
+ 'label': 'NFO filename',
+ 'default': '%s.nfo',
+ 'advanced': True,
+ 'description': '%s is the rootname of the movie. For example "/path/to/movie cd1.mkv" will be "/path/to/movie"'
+ },
{
'name': 'meta_fanart',
+ 'label': 'Fanart',
'default': True,
'type': 'bool',
},
+ {
+ 'name': 'meta_fanart_name',
+ 'label': 'Fanart filename',
+ 'default': '%s-fanart.jpg',
+ 'advanced': True,
+ },
{
'name': 'meta_thumbnail',
+ 'label': 'Thumbnail',
'default': True,
'type': 'bool',
},
+ {
+ 'name': 'meta_thumbnail_name',
+ 'label': 'Thumbnail filename',
+ 'default': '%s.tbn',
+ 'advanced': True,
+ },
],
},
],
diff --git a/couchpotato/core/providers/metadata/xbmc/main.py b/couchpotato/core/providers/metadata/xbmc/main.py
index 2908c6f6..7797cce8 100644
--- a/couchpotato/core/providers/metadata/xbmc/main.py
+++ b/couchpotato/core/providers/metadata/xbmc/main.py
@@ -15,13 +15,13 @@ class XBMC(MetaDataBase):
return os.path.join(data['destination_dir'], data['filename'])
def getFanartName(self, root):
- return '%s-fanart.jpg' % root
+ return self.conf('meta_fanart_name') % root
def getThumbnailName(self, root):
- return '%s.tbn' % root
+ return self.conf('meta_thumbnail_name') % root
def getNfoName(self, root):
- return '%s.nfo' % root
+ return self.conf('meta_nfo_name') % root
def getNfo(self, movie_info = {}, data = {}):
nfoxml = Element('movie')
diff --git a/couchpotato/core/providers/movie/couchpotatoapi/main.py b/couchpotato/core/providers/movie/couchpotatoapi/main.py
index aca80ccc..26e42df3 100644
--- a/couchpotato/core/providers/movie/couchpotatoapi/main.py
+++ b/couchpotato/core/providers/movie/couchpotatoapi/main.py
@@ -1,6 +1,5 @@
from couchpotato import get_session
-from couchpotato.api import addApiView
-from couchpotato.core.event import addEvent
+from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.request import jsonified, getParams
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.movie.base import MovieProvider
@@ -12,26 +11,37 @@ log = CPLog(__name__)
class CouchPotatoApi(MovieProvider):
- apiUrl = 'http://couchpota.to/api/%s/'
+ api_url = 'http://couchpota.to/api/%s/'
+ http_time_between_calls = 0
def __init__(self):
- addEvent('movie.release_date', self.releaseDate)
- addApiView('movie.suggest', self.suggestView)
+ #addApiView('movie.suggest', self.suggestView)
- def releaseDate(self, imdb_id):
+ addEvent('movie.info', self.getInfo)
+ addEvent('movie.release_date', self.getReleaseDate)
+ def getInfo(self, identifier = None):
+ return {
+ 'release_date': self.getReleaseDate(identifier)
+ }
+
+ def getReleaseDate(self, identifier = None):
+
+ if identifier is None: return {}
try:
- data = self.urlopen((self.apiUrl % ('eta')) + (id + '/'))
+ headers = {'X-CP-Version': fireEvent('app.version', single = True)}
+ data = self.urlopen((self.api_url % ('eta')) + (identifier + '/'), headers = headers)
dates = json.loads(data)
- log.info('Found ETA for %s: %s' % (imdb_id, dates))
+ log.debug('Found ETA for %s: %s' % (identifier, dates))
+ return dates
except Exception, e:
- log.error('Error getting ETA for %s: %s' % (imdb_id, e))
+ log.error('Error getting ETA for %s: %s' % (identifier, e))
- return dates
+ return {}
def suggest(self, movies = [], ignore = []):
try:
- data = self.urlopen((self.apiUrl % ('suggest')) + ','.join(movies) + '/' + ','.join(ignore) + '/')
+ data = self.urlopen((self.api_url % ('suggest')) + ','.join(movies) + '/' + ','.join(ignore) + '/')
suggestions = json.loads(data)
log.info('Found Suggestions for %s' % (suggestions))
except Exception, e:
diff --git a/couchpotato/core/providers/movie/imdbapi/main.py b/couchpotato/core/providers/movie/imdbapi/main.py
index 604b37c1..568b413e 100644
--- a/couchpotato/core/providers/movie/imdbapi/main.py
+++ b/couchpotato/core/providers/movie/imdbapi/main.py
@@ -1,8 +1,8 @@
from couchpotato.core.event import addEvent, fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import tryInt, tryFloat
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.movie.base import MovieProvider
-from urllib import urlencode
import json
import re
import traceback
@@ -27,8 +27,11 @@ class IMDBAPI(MovieProvider):
name_year = fireEvent('scanner.name_year', q, single = True)
+ if not q or not name_year.get('name'):
+ return []
+
cache_key = 'imdbapi.cache.%s' % q
- cached = self.getCache(cache_key, self.urls['search'] % urlencode({'t': name_year.get('name'), 'y': name_year.get('year')}))
+ cached = self.getCache(cache_key, self.urls['search'] % tryUrlencode({'t': name_year.get('name'), 'y': name_year.get('year', '')}))
if cached:
result = self.parseMovie(cached)
@@ -42,6 +45,9 @@ class IMDBAPI(MovieProvider):
def getInfo(self, identifier = None):
+ if not identifier:
+ return {}
+
cache_key = 'imdbapi.cache.%s' % identifier
cached = self.getCache(cache_key, self.urls['info'] % identifier)
@@ -69,6 +75,8 @@ class IMDBAPI(MovieProvider):
if tmp_movie.get(key).lower() == 'n/a':
del movie[key]
+ year = tryInt(movie.get('Year', ''))
+
movie_data = {
'titles': [movie.get('Title')] if movie.get('Title') else [],
'original_title': movie.get('Title', ''),
@@ -82,7 +90,7 @@ class IMDBAPI(MovieProvider):
'imdb': str(movie.get('ID', '')),
'runtime': self.runtimeToMinutes(movie.get('Runtime', '')),
'released': movie.get('Released', ''),
- 'year': movie.get('Year', ''),
+ 'year': year if isinstance(year, (int)) else None,
'plot': movie.get('Plot', ''),
'genres': movie.get('Genre', '').split(','),
'directors': movie.get('Director', '').split(','),
diff --git a/couchpotato/core/providers/movie/themoviedb/main.py b/couchpotato/core/providers/movie/themoviedb/main.py
index 2838c4ce..a84b5568 100644
--- a/couchpotato/core/providers/movie/themoviedb/main.py
+++ b/couchpotato/core/providers/movie/themoviedb/main.py
@@ -3,6 +3,7 @@ from couchpotato.core.helpers.encoding import simplifyString, toUnicode
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.movie.base import MovieProvider
from libs.themoviedb import tmdb
+import re
log = CPLog(__name__)
@@ -67,8 +68,14 @@ class TheMovieDb(MovieProvider):
if raw:
try:
nr = 0
- for movie in raw:
+ # Sort on returned score first when year is in q
+ if re.search('\s\d{4}', q):
+ movies = sorted(raw, key = lambda k: k['score'], reverse = True)
+ else:
+ movies = raw
+
+ for movie in movies:
results.append(self.parseMovie(movie))
nr += 1
diff --git a/couchpotato/core/providers/nzb/moovee/main.py b/couchpotato/core/providers/nzb/moovee/main.py
index a6353bc9..a6aeb32a 100644
--- a/couchpotato/core/providers/nzb/moovee/main.py
+++ b/couchpotato/core/providers/nzb/moovee/main.py
@@ -1,8 +1,8 @@
from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from dateutil.parser import parse
-from urllib import quote_plus
import re
import time
@@ -27,7 +27,7 @@ class Moovee(NZBProvider):
return results
q = '%s %s' % (movie['library']['titles'][0]['title'], quality.get('identifier'))
- url = self.urls['search'] % quote_plus(q)
+ url = self.urls['search'] % tryUrlencode(q)
cache_key = 'moovee.%s' % q
data = self.getCache(cache_key, url)
diff --git a/couchpotato/core/providers/nzb/mysterbin/main.py b/couchpotato/core/providers/nzb/mysterbin/main.py
index d92595f6..06936f9a 100644
--- a/couchpotato/core/providers/nzb/mysterbin/main.py
+++ b/couchpotato/core/providers/nzb/mysterbin/main.py
@@ -1,11 +1,10 @@
from BeautifulSoup import BeautifulSoup
from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from couchpotato.environment import Env
-import urllib
log = CPLog(__name__)
@@ -13,9 +12,9 @@ log = CPLog(__name__)
class Mysterbin(NZBProvider):
urls = {
- 'search': 'http://www.mysterbin.com/advsearch?%s',
- 'download': 'http://www.mysterbin.com/nzb?c=%s',
- 'nfo': 'http://www.mysterbin.com/nfo?c=%s',
+ 'search': 'https://www.mysterbin.com/advsearch?%s',
+ 'download': 'https://www.mysterbin.com/nzb?c=%s',
+ 'nfo': 'https://www.mysterbin.com/nfo?c=%s',
}
http_time_between_calls = 1 #seconds
@@ -40,8 +39,8 @@ class Mysterbin(NZBProvider):
'nopasswd': 'on',
}
- cache_key = 'mysterbin.%s' % q
- data = self.getCache(cache_key, self.urls['search'] % urllib.urlencode(params))
+ cache_key = 'mysterbin.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
+ data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params))
if data:
try:
diff --git a/couchpotato/core/providers/nzb/newzbin/main.py b/couchpotato/core/providers/nzb/newzbin/main.py
index 96a34a3f..d68bbae3 100644
--- a/couchpotato/core/providers/nzb/newzbin/main.py
+++ b/couchpotato/core/providers/nzb/newzbin/main.py
@@ -1,9 +1,9 @@
from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from dateutil.parser import parse
-from urllib import urlencode
import base64
import time
import xml.etree.ElementTree as XMLTree
@@ -14,7 +14,7 @@ log = CPLog(__name__)
class Newzbin(NZBProvider, RSS):
urls = {
- 'download': 'http://www.newzbin2.es/api/dnzb/',
+ 'download': 'https://www.newzbin2.es/api/dnzb/',
'search': 'https://www.newzbin2.es/search/',
}
@@ -45,7 +45,7 @@ class Newzbin(NZBProvider, RSS):
format_id = self.getFormatId(type)
cat_id = self.getCatId(type)
- arguments = urlencode({
+ arguments = tryUrlencode({
'searchaction': 'Search',
'u_url_posts_only': '0',
'u_show_passworded': '0',
@@ -89,11 +89,14 @@ class Newzbin(NZBProvider, RSS):
title = self.getTextElement(nzb, "title")
if 'error' in title.lower(): continue
- REPORT_NS = 'http://www.newzbin.com/DTD/2007/feeds/report/';
+ REPORT_NS = 'http://www.newzbin2.es/DTD/2007/feeds/report/';
# Add attributes to name
- for attr in nzb.find('{%s}attributes' % REPORT_NS):
- title += ' ' + attr.text
+ try:
+ for attr in nzb.find('{%s}attributes' % REPORT_NS):
+ title += ' ' + attr.text
+ except:
+ pass
id = int(self.getTextElement(nzb, '{%s}id' % REPORT_NS))
size = str(int(self.getTextElement(nzb, '{%s}size' % REPORT_NS)) / 1024 / 1024) + ' mb'
diff --git a/couchpotato/core/providers/nzb/newznab/__init__.py b/couchpotato/core/providers/nzb/newznab/__init__.py
index 212c9847..b637ca8e 100644
--- a/couchpotato/core/providers/nzb/newznab/__init__.py
+++ b/couchpotato/core/providers/nzb/newznab/__init__.py
@@ -19,16 +19,16 @@ config = [{
},
{
'name': 'use',
- 'default': '0'
+ 'default': '0,0,0'
},
{
'name': 'host',
- 'default': 'nzb.su',
+ 'default': 'nzb.su,dognzb.cr,beta.nzbs.org',
'description': 'The hostname of your newznab provider',
},
{
'name': 'api_key',
- 'default': '',
+ 'default': ',,',
'label': 'Api Key',
'description': 'Can be found on your profile page',
'type': 'combined',
diff --git a/couchpotato/core/providers/nzb/newznab/main.py b/couchpotato/core/providers/nzb/newznab/main.py
index c17e6163..99fb771d 100644
--- a/couchpotato/core/providers/nzb/newznab/main.py
+++ b/couchpotato/core/providers/nzb/newznab/main.py
@@ -1,10 +1,10 @@
from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import cleanHost
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from dateutil.parser import parse
-from urllib import urlencode
import time
import xml.etree.ElementTree as XMLTree
@@ -48,7 +48,7 @@ class Newznab(NZBProvider, RSS):
if self.isDisabled(host) or not self.isAvailable(self.getUrl(host['host'], self.urls['search'])):
return results
- arguments = urlencode({
+ arguments = tryUrlencode({
't': self.cat_backup_id,
'r': host['api_key'],
'i': 58,
@@ -81,11 +81,10 @@ class Newznab(NZBProvider, RSS):
return results
cat_id = self.getCatId(quality['identifier'])
- arguments = urlencode({
+ arguments = tryUrlencode({
'imdbid': movie['library']['identifier'].replace('tt', ''),
'cat': cat_id[0],
'apikey': host['api_key'],
- 't': self.urls['search'],
'extended': 1
})
url = "%s&%s" % (self.getUrl(host['host'], self.urls['search']), arguments)
@@ -121,8 +120,8 @@ class Newznab(NZBProvider, RSS):
elif item.attrib.get('name') == 'usenetdate':
date = item.attrib.get('value')
- if date is '': log.info('Date not parsed properly or not available for %s: %s' % (host, self.getTextElement(nzb, "title")))
- if size is 0: log.info('Size not parsed properly or not available for %s: %s' % (host, self.getTextElement(nzb, "title")))
+ if date is '': log.debug('Date not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title")))
+ if size is 0: log.debug('Size not parsed properly or not available for %s: %s' % (host['host'], self.getTextElement(nzb, "title")))
id = self.getTextElement(nzb, "guid").split('/')[-1:].pop()
new = {
@@ -158,9 +157,9 @@ class Newznab(NZBProvider, RSS):
def getHosts(self):
- uses = str(self.conf('use')).split(',')
- hosts = self.conf('host').split(',')
- api_keys = self.conf('api_key').split(',')
+ uses = [x.strip() for x in str(self.conf('use')).split(',')]
+ hosts = [x.strip() for x in self.conf('host').split(',')]
+ api_keys = [x.strip() for x in self.conf('api_key').split(',')]
list = []
for nr in range(len(hosts)):
diff --git a/couchpotato/core/providers/nzb/nzbclub/main.py b/couchpotato/core/providers/nzb/nzbclub/main.py
index 07ae2c2d..eb33fe18 100644
--- a/couchpotato/core/providers/nzb/nzbclub/main.py
+++ b/couchpotato/core/providers/nzb/nzbclub/main.py
@@ -1,6 +1,5 @@
-from BeautifulSoup import BeautifulSoup, SoupStrainer
from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
@@ -8,7 +7,6 @@ from couchpotato.core.providers.nzb.base import NZBProvider
from couchpotato.environment import Env
from dateutil.parser import parse
import time
-import urllib
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__)
@@ -17,7 +15,7 @@ log = CPLog(__name__)
class NZBClub(NZBProvider, RSS):
urls = {
- 'search': 'http://www.nzbclub.com/nzbfeed.aspx?%s',
+ 'search': 'https://www.nzbclub.com/nzbfeed.aspx?%s',
}
http_time_between_calls = 3 #seconds
@@ -41,8 +39,8 @@ class NZBClub(NZBProvider, RSS):
'ns': 1,
}
- cache_key = 'nzbclub.%s' % q
- data = self.getCache(cache_key, self.urls['search'] % urllib.urlencode(params))
+ cache_key = 'nzbclub.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
+ data = self.getCache(cache_key, self.urls['search'] % tryUrlencode(params))
if data:
try:
try:
@@ -68,8 +66,8 @@ class NZBClub(NZBProvider, RSS):
'name': toUnicode(self.getTextElement(nzb, "title")),
'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
'size': tryInt(size) / 1024 / 1024,
- 'url': enclosure['url'],
- 'download': enclosure['url'].replace(' ', '_'),
+ 'url': enclosure['url'].replace(' ', '_'),
+ 'download': self.download,
'detail_url': self.getTextElement(nzb, "link"),
'description': description,
}
diff --git a/couchpotato/core/providers/nzb/nzbindex/main.py b/couchpotato/core/providers/nzb/nzbindex/main.py
index 1f56a299..22e202b6 100644
--- a/couchpotato/core/providers/nzb/nzbindex/main.py
+++ b/couchpotato/core/providers/nzb/nzbindex/main.py
@@ -1,12 +1,11 @@
from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.encoding import toUnicode
+from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from couchpotato.environment import Env
from dateutil.parser import parse
-from urllib import urlencode
import re
import time
import xml.etree.ElementTree as XMLTree
@@ -30,7 +29,7 @@ class NzbIndex(NZBProvider, RSS):
return results
q = '%s %s %s' % (movie['library']['titles'][0]['title'], movie['library']['year'], quality.get('identifier'))
- arguments = urlencode({
+ arguments = tryUrlencode({
'q': q,
'age': Env.setting('retention', 'nzb'),
'sort': 'agedesc',
@@ -43,7 +42,7 @@ class NzbIndex(NZBProvider, RSS):
})
url = "%s?%s" % (self.urls['api'], arguments)
- cache_key = 'nzbindex.%s' % q
+ cache_key = 'nzbindex.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, url)
if data:
diff --git a/couchpotato/core/providers/nzb/nzbmatrix/main.py b/couchpotato/core/providers/nzb/nzbmatrix/main.py
index 43b95873..8e82133b 100644
--- a/couchpotato/core/providers/nzb/nzbmatrix/main.py
+++ b/couchpotato/core/providers/nzb/nzbmatrix/main.py
@@ -1,10 +1,10 @@
from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from couchpotato.environment import Env
from dateutil.parser import parse
-from urllib import urlencode
import time
import xml.etree.ElementTree as XMLTree
@@ -16,7 +16,7 @@ class NZBMatrix(NZBProvider, RSS):
urls = {
'download': 'https://api.nzbmatrix.com/v1.1/download.php?id=%s',
'detail': 'https://nzbmatrix.com/nzb-details.php?id=%s&hit=1',
- 'search': 'http://rss.nzbmatrix.com/rss.php',
+ 'search': 'https://rss.nzbmatrix.com/rss.php',
}
cat_ids = [
@@ -37,7 +37,7 @@ class NZBMatrix(NZBProvider, RSS):
cat_ids = ','.join(['%s' % x for x in self.getCatId(quality.get('identifier'))])
- arguments = urlencode({
+ arguments = tryUrlencode({
'term': movie['library']['identifier'],
'subcat': cat_ids,
'username': self.conf('username'),
diff --git a/couchpotato/core/providers/nzb/nzbs/main.py b/couchpotato/core/providers/nzb/nzbs/main.py
index a7e8fc66..4281f819 100644
--- a/couchpotato/core/providers/nzb/nzbs/main.py
+++ b/couchpotato/core/providers/nzb/nzbs/main.py
@@ -1,10 +1,9 @@
from couchpotato.core.event import fireEvent
-from couchpotato.core.helpers.encoding import simplifyString
+from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
from dateutil.parser import parse
-from urllib import urlencode
import time
import xml.etree.ElementTree as XMLTree
@@ -14,10 +13,10 @@ log = CPLog(__name__)
class Nzbs(NZBProvider, RSS):
urls = {
- 'download': 'http://nzbs.org/index.php?action=getnzb&nzbid=%s%s',
- 'nfo': 'http://nzbs.org/index.php?action=view&nzbid=%s&nfo=1',
- 'detail': 'http://nzbs.org/index.php?action=view&nzbid=%s',
- 'api': 'http://nzbs.org/rss.php',
+ 'download': 'https://nzbs.org/index.php?action=getnzb&nzbid=%s%s',
+ 'nfo': 'https://nzbs.org/index.php?action=view&nzbid=%s&nfo=1',
+ 'detail': 'https://nzbs.org/index.php?action=view&nzbid=%s',
+ 'api': 'https://nzbs.org/rss.php',
}
cat_ids = [
@@ -36,7 +35,7 @@ class Nzbs(NZBProvider, RSS):
return results
cat_id = self.getCatId(quality.get('identifier'))
- arguments = urlencode({
+ arguments = tryUrlencode({
'action':'search',
'q': simplifyString(movie['library']['titles'][0]['title']),
'catid': cat_id[0],
diff --git a/couchpotato/core/providers/nzb/x264/main.py b/couchpotato/core/providers/nzb/x264/main.py
index 0db9eabc..d1b9f727 100644
--- a/couchpotato/core/providers/nzb/x264/main.py
+++ b/couchpotato/core/providers/nzb/x264/main.py
@@ -1,8 +1,8 @@
from couchpotato.core.event import fireEvent
+from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.nzb.base import NZBProvider
-from urllib import quote_plus
import re
log = CPLog(__name__)
@@ -26,9 +26,9 @@ class X264(NZBProvider):
return results
q = '%s %s %s' % (movie['library']['titles'][0]['title'], movie['library']['year'], quality.get('identifier'))
- url = self.urls['search'] % quote_plus(q)
+ url = self.urls['search'] % tryUrlencode(q)
- cache_key = 'x264.%s' % q
+ cache_key = 'x264.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, url)
if data:
match = re.compile(self.regex, re.DOTALL).finditer(data)
diff --git a/couchpotato/core/providers/subtitle/__init__.py b/couchpotato/core/providers/subtitle/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/couchpotato/core/providers/subtitle/base.py b/couchpotato/core/providers/subtitle/base.py
deleted file mode 100644
index d26a6d1a..00000000
--- a/couchpotato/core/providers/subtitle/base.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from couchpotato.core.event import addEvent
-from couchpotato.core.providers.base import Provider
-
-
-class SubtitleProvider(Provider):
-
- type = 'subtitle'
-
- def __init__(self):
- addEvent('subtitle.search', self.search)
-
- def search(self, group):
- pass
diff --git a/couchpotato/core/providers/subtitle/subliminal/__init__.py b/couchpotato/core/providers/subtitle/subliminal/__init__.py
deleted file mode 100644
index bc3cb856..00000000
--- a/couchpotato/core/providers/subtitle/subliminal/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from .main import SubliminalProvider
-
-def start():
- return SubliminalProvider()
-
-config = []
diff --git a/couchpotato/core/providers/subtitle/subliminal/main.py b/couchpotato/core/providers/subtitle/subliminal/main.py
deleted file mode 100644
index 9616aeac..00000000
--- a/couchpotato/core/providers/subtitle/subliminal/main.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from couchpotato.core.logger import CPLog
-from couchpotato.core.providers.subtitle.base import SubtitleProvider
-from couchpotato.environment import Env
-from libs import subliminal
-
-log = CPLog(__name__)
-
-
-class SubliminalProvider(SubtitleProvider):
-
- plugins = ['OpenSubtitles', 'TheSubDB', 'SubsWiki']
-
- def search(self, files = [], languages = []):
-
- # download subtitles
- with subliminal.Subliminal(cache_dir = Env.get('cache_dir'), multi = True,
- languages = self.getLanguages(), plugins = self.plugins) as subli:
- subtitles = subli.downloadSubtitles(files)
-
- print subtitles
-
- return subtitles
diff --git a/couchpotato/core/providers/torrent/kickasstorrents/main.py b/couchpotato/core/providers/torrent/kickasstorrents/main.py
index aff0508f..6a039c71 100644
--- a/couchpotato/core/providers/torrent/kickasstorrents/main.py
+++ b/couchpotato/core/providers/torrent/kickasstorrents/main.py
@@ -17,6 +17,7 @@ class KickAssTorrents(TorrentProvider):
'test': 'http://www.kat.ph/',
'detail': 'http://www.kat.ph/%s-t%s.html',
'search': 'http://www.kat.ph/%s-i%s/',
+ 'download': 'http://torcache.net/',
}
cat_ids = [
@@ -36,7 +37,7 @@ class KickAssTorrents(TorrentProvider):
if self.isDisabled() or not self.isAvailable(self.urls['test']):
return results
- cache_key = 'kickasstorrents.%s' % movie['library']['identifier']
+ cache_key = 'kickasstorrents.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, self.urls['search'] % (movie['library']['titles'][0]['title'], movie['library']['identifier'].replace('tt', '')))
if data:
@@ -127,7 +128,7 @@ class KickAssTorrents(TorrentProvider):
return tryInt(age)
def download(self, url = '', nzb_id = ''):
- compressed_data = super(KickAssTorrents, self).download(url = url, nzb_id = nzb_id)
+ compressed_data = self.urlopen(url = url, headers = {'Referer': 'http://kat.ph/'})
compressedstream = StringIO.StringIO(compressed_data)
gzipper = gzip.GzipFile(fileobj = compressedstream)
diff --git a/couchpotato/core/providers/trailer/hdtrailers/main.py b/couchpotato/core/providers/trailer/hdtrailers/main.py
index e6628760..26980702 100644
--- a/couchpotato/core/providers/trailer/hdtrailers/main.py
+++ b/couchpotato/core/providers/trailer/hdtrailers/main.py
@@ -1,9 +1,9 @@
from BeautifulSoup import SoupStrainer, BeautifulSoup
+from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import mergeDicts
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.trailer.base import TrailerProvider
from string import letters, digits
-from urllib import urlencode
import re
log = CPLog(__name__)
@@ -44,7 +44,7 @@ class HDTrailers(TrailerProvider):
movie_name = group['library']['titles'][0]['title']
- url = "%s?%s" % (self.url['backup'], urlencode({'s':movie_name}))
+ url = "%s?%s" % (self.url['backup'], tryUrlencode({'s':movie_name}))
data = self.getCache('hdtrailers.alt.%s' % group['library']['identifier'], url)
try:
diff --git a/couchpotato/core/settings/__init__.py b/couchpotato/core/settings/__init__.py
index 60c77cb2..48c7c329 100644
--- a/couchpotato/core/settings/__init__.py
+++ b/couchpotato/core/settings/__init__.py
@@ -4,9 +4,11 @@ from couchpotato.core.event import addEvent, fireEvent
from couchpotato.core.helpers.encoding import isInt, toUnicode
from couchpotato.core.helpers.request import getParams, jsonified
from couchpotato.core.helpers.variable import mergeDicts, tryInt
+from couchpotato.core.settings.model import Properties
import ConfigParser
import os.path
import time
+import traceback
class Settings(object):
@@ -190,3 +192,28 @@ class Settings(object):
return jsonified({
'success': True,
})
+
+ def getProperty(self, identifier):
+ from couchpotato import get_session
+
+ db = get_session()
+ try:
+ prop = db.query(Properties).filter_by(identifier = identifier).first()
+ return prop.value if prop else None
+ except:
+ return None
+
+ def setProperty(self, identifier, value = ''):
+ from couchpotato import get_session
+
+ db = get_session()
+
+ p = db.query(Properties).filter_by(identifier = identifier).first()
+ if not p:
+ p = Properties()
+ db.add(p)
+
+ p.identifier = identifier
+ p.value = toUnicode(value)
+
+ db.commit()
diff --git a/couchpotato/core/settings/model.py b/couchpotato/core/settings/model.py
index 4dd54d32..bcf5adb8 100644
--- a/couchpotato/core/settings/model.py
+++ b/couchpotato/core/settings/model.py
@@ -44,8 +44,8 @@ class Movie(Entity):
library = ManyToOne('Library')
status = ManyToOne('Status')
profile = ManyToOne('Profile')
- releases = OneToMany('Release')
- files = ManyToMany('File')
+ releases = OneToMany('Release', cascade = 'all, delete-orphan')
+ files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True)
class Library(Entity):
@@ -59,9 +59,9 @@ class Library(Entity):
info = Field(JsonType)
status = ManyToOne('Status')
- movies = OneToMany('Movie')
- titles = OneToMany('LibraryTitle')
- files = ManyToMany('File')
+ movies = OneToMany('Movie', cascade = 'all, delete-orphan')
+ titles = OneToMany('LibraryTitle', cascade = 'all, delete-orphan')
+ files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True)
class LibraryTitle(Entity):
@@ -94,9 +94,9 @@ class Release(Entity):
movie = ManyToOne('Movie')
status = ManyToOne('Status')
quality = ManyToOne('Quality')
- files = ManyToMany('File')
- history = OneToMany('History')
- info = OneToMany('ReleaseInfo')
+ files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True)
+ history = OneToMany('History', cascade = 'all, delete-orphan')
+ info = OneToMany('ReleaseInfo', cascade = 'all, delete-orphan')
class ReleaseInfo(Entity):
@@ -229,6 +229,12 @@ class Folder(Entity):
label = Field(Unicode(255))
+class Properties(Entity):
+
+ identifier = Field(String(50), index = True)
+ value = Field(Unicode(255), nullable = False)
+
+
def setup():
"""Setup the database and create the tables that don't exists yet"""
from elixir import setup_all, create_all
diff --git a/couchpotato/environment.py b/couchpotato/environment.py
index ac256c18..1ac184f0 100644
--- a/couchpotato/environment.py
+++ b/couchpotato/environment.py
@@ -61,6 +61,15 @@ class Env(object):
return s
+ @staticmethod
+ def prop(identifier, value = None, default = None):
+ s = Env.get('settings')
+ if value == None:
+ v = s.getProperty(identifier)
+ return v if v else default
+
+ s.setProperty(identifier, value)
+
@staticmethod
def getPermission(setting_type):
perm = Env.get('settings').get('permission_%s' % setting_type, default = '0777')
diff --git a/couchpotato/runner.py b/couchpotato/runner.py
index 217f52ba..66d9d7d0 100644
--- a/couchpotato/runner.py
+++ b/couchpotato/runner.py
@@ -52,7 +52,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
locale.setlocale(locale.LC_ALL, "")
encoding = locale.getpreferredencoding()
except (locale.Error, IOError):
- pass
+ encoding = None
# for OSes that are poorly configured I'll just force UTF-8
if not encoding or encoding in ('ANSI_X3.4-1968', 'US-ASCII', 'ASCII'):
diff --git a/couchpotato/static/images/icon.info.png b/couchpotato/static/images/icon.info.png
new file mode 100644
index 00000000..12cd1aef
Binary files /dev/null and b/couchpotato/static/images/icon.info.png differ
diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js
index edafdf4c..8f412eea 100644
--- a/couchpotato/static/scripts/couchpotato.js
+++ b/couchpotato/static/scripts/couchpotato.js
@@ -70,13 +70,13 @@ var CouchPotato = new Class({
[new Element('a.orange', {
'text': 'Restart',
'events': {
- 'click': self.restart.bind(self)
+ 'click': self.restartQA.bind(self)
}
}),
new Element('a.red', {
'text': 'Shutdown',
'events': {
- 'click': self.shutdown.bind(self)
+ 'click': self.shutdownQA.bind(self)
}
}),
new Element('a', {
@@ -161,6 +161,25 @@ var CouchPotato = new Class({
self.checkAvailable(1000);
},
+ shutdownQA: function(e){
+ var self = this;
+
+ var q = new Question('Are you sure you want to shutdown CouchPotato?', '', [{
+ 'text': 'Shutdown',
+ 'class': 'shutdown red',
+ 'events': {
+ 'click': function(e){
+ (e).preventDefault();
+ self.shutdown();
+ q.close.delay(100, q);
+ }
+ }
+ }, {
+ 'text': 'No, nevah!',
+ 'cancel': true
+ }]);
+ },
+
restart: function(message, title){
var self = this;
@@ -169,6 +188,25 @@ var CouchPotato = new Class({
self.checkAvailable(1000);
},
+ restartQA: function(e, message, title){
+ var self = this;
+
+ var q = new Question('Are you sure you want to restart CouchPotato?', '', [{
+ 'text': 'Restart',
+ 'class': 'restart orange',
+ 'events': {
+ 'click': function(e){
+ (e).preventDefault();
+ self.restart(message, title);
+ q.close.delay(100, q);
+ }
+ }
+ }, {
+ 'text': 'No, nevah!',
+ 'cancel': true
+ }]);
+ },
+
checkForUpdate: function(func){
var self = this;
@@ -190,7 +228,6 @@ var CouchPotato = new Class({
},
'onSuccess': function(){
self.unBlockPage();
- self.fireEvent('load');
}
});
diff --git a/couchpotato/static/scripts/library/prefix_free.js b/couchpotato/static/scripts/library/prefix_free.js
index 1ca634ee..4876187b 100644
--- a/couchpotato/static/scripts/library/prefix_free.js
+++ b/couchpotato/static/scripts/library/prefix_free.js
@@ -25,20 +25,24 @@ var self = window.StyleFix = {
var url = link.href || link.getAttribute('data-href'),
base = url.replace(/[^\/]+$/, ''),
parent = link.parentNode,
- xhr = new XMLHttpRequest();
+ xhr = new XMLHttpRequest(),
+ process;
- xhr.open('GET', url);
-
xhr.onreadystatechange = function() {
if(xhr.readyState === 4) {
+ process();
+ }
+ };
+
+ process = function() {
var css = xhr.responseText;
- if(css && link.parentNode) {
+ if(css && link.parentNode && (!xhr.status || xhr.status < 400 || xhr.status > 600)) {
css = self.fix(css, true, link);
// Convert relative URLs to absolute, if needed
if(base) {
- css = css.replace(/url\(((?:"|')?)(.+?)\1\)/gi, function($0, quote, url) {
+ css = css.replace(/url\(\s*?((?:"|')?)(.+?)\1\s*?\)/gi, function($0, quote, url) {
if(!/^([a-z]{3,10}:|\/|#)/i.test(url)) { // If url not absolute & not a hash
// May contain sequences like /../ and /./ but those DO work
return 'url("' + base + url + '")';
@@ -60,10 +64,21 @@ var self = window.StyleFix = {
parent.insertBefore(style, link);
parent.removeChild(link);
}
- }
};
-
- xhr.send(null);
+
+ try {
+ xhr.open('GET', url);
+ xhr.send(null);
+ } catch (e) {
+ // Fallback to XDomainRequest if available
+ if (typeof XDomainRequest != "undefined") {
+ xhr = new XDomainRequest();
+ xhr.onerror = xhr.onprogress = function() {};
+ xhr.onload = process;
+ xhr.open("GET", url);
+ xhr.send(null);
+ }
+ }
link.setAttribute('data-inprogress', '');
},
@@ -135,7 +150,7 @@ function $(expr, con) {
})();
/**
- * PrefixFree 1.0.4
+ * PrefixFree 1.0.5
* @author Lea Verou
* MIT license
*/
@@ -300,6 +315,10 @@ var functions = {
'element': {
property: 'backgroundImage',
params: '#foo'
+ },
+ 'cross-fade': {
+ property: 'backgroundImage',
+ params: 'url(a.png), url(b.png), 50%'
}
};
diff --git a/couchpotato/static/scripts/library/question.js b/couchpotato/static/scripts/library/question.js
index b80005e3..cab63465 100644
--- a/couchpotato/static/scripts/library/question.js
+++ b/couchpotato/static/scripts/library/question.js
@@ -18,13 +18,7 @@ var Question = new Class( {
createMask : function() {
var self = this
- $(document.body).mask( {
- 'hideOnClick' : true,
- 'destroyOnHide' : true,
- 'onHide' : function() {
- self.container.destroy();
- }
- }).show();
+ self.mask = new Element('div.mask').fade('hide').inject(document.body).fade('in');
},
createQuestion : function() {
@@ -71,7 +65,11 @@ var Question = new Class( {
},
close : function() {
- $(document.body).get('mask').destroy();
+ var self = this;
+ self.mask.fade('out');
+ (function(){self.mask.destroy()}).delay(1000);
+
+ this.container.destroy();
},
toElement : function() {
diff --git a/couchpotato/static/scripts/page/manage.js b/couchpotato/static/scripts/page/manage.js
index 74dba986..0385ee0f 100644
--- a/couchpotato/static/scripts/page/manage.js
+++ b/couchpotato/static/scripts/page/manage.js
@@ -38,7 +38,6 @@ Page.Manage = new Class({
refresh: function(full){
var self = this;
- p(full)
Api.request('manage.update', {
'data': {
diff --git a/couchpotato/static/scripts/page/settings.js b/couchpotato/static/scripts/page/settings.js
index 94643cc7..92cdf36b 100644
--- a/couchpotato/static/scripts/page/settings.js
+++ b/couchpotato/static/scripts/page/settings.js
@@ -574,7 +574,7 @@ Option.Directory = new Class({
self.el.adopt(
self.createLabel(),
- new Element('span.directory.inlay', {
+ self.directory_inlay = new Element('span.directory.inlay', {
'events': {
'click': self.showBrowser.bind(self)
}
@@ -664,7 +664,7 @@ Option.Directory = new Class({
}
})
)
- ).inject(self.input, 'before');
+ ).inject(self.directory_inlay, 'before');
new Form.Check(self.show_hidden);
}
diff --git a/couchpotato/static/scripts/page/wanted.js b/couchpotato/static/scripts/page/wanted.js
index 8d1dd931..af6d774c 100644
--- a/couchpotato/static/scripts/page/wanted.js
+++ b/couchpotato/static/scripts/page/wanted.js
@@ -61,7 +61,7 @@ window.addEvent('domready', function(){
'name': 'profile'
}),
new Element('a.button.edit', {
- 'text': 'Save',
+ 'text': 'Save & Search',
'events': {
'click': self.save.bind(self)
}
@@ -150,7 +150,7 @@ window.addEvent('domready', function(){
var self = this;
self.el = new Element('a.delete', {
- 'title': 'Remove the movie from your wanted list',
+ 'title': 'Remove the movie from this CP list',
'events': {
'click': self.showConfirm.bind(self)
}
@@ -286,6 +286,7 @@ window.addEvent('domready', function(){
},
})
+ ,'Delete': MovieActions.Wanted.Delete
};
})
\ No newline at end of file
diff --git a/couchpotato/static/style/main.css b/couchpotato/static/style/main.css
index a128ee32..d580368c 100644
--- a/couchpotato/static/style/main.css
+++ b/couchpotato/static/style/main.css
@@ -13,7 +13,6 @@ body {
background: #4e5969;
overflow-y: scroll;
height: 100%;
- text-align: justify;
}
body.noscroll { overflow: hidden; }
@@ -154,6 +153,7 @@ body > .spinner, .mask{
.icon.refresh { background-image: url('../images/icon.refresh.png'); }
.icon.rating { background-image: url('../images/icon.rating.png'); }
.icon.files { background-image: url('../images/icon.files.png'); }
+.icon.info { background-image: url('../images/icon.info.png'); }
/*** Navigation ***/
.header {
@@ -305,9 +305,9 @@ body > .spinner, .mask{
text-align: center;
position: relative;
top: -70px;
- padding: 15px 0 20px;
+ padding: 2px 0;
background: #ff6134;
- font-size: 26px;
+ font-size: 12px;
border-radius: 0 0 5px 5px;
box-shadow: 0 2px 1px rgba(0,0,0, 0.3);
}
@@ -491,7 +491,7 @@ body > .spinner, .mask{
width: auto;
}
.question .answer:hover {
- background: #f1f1f1;
+ background: #000;
}
.question .answer.delete {
diff --git a/couchpotato/static/style/page/settings.css b/couchpotato/static/style/page/settings.css
index 464be778..4d157f58 100644
--- a/couchpotato/static/style/page/settings.css
+++ b/couchpotato/static/style/page/settings.css
@@ -208,7 +208,7 @@
z-index: 2;
position: absolute;
width: 450px;
- margin: 28px 0 20px -4px;
+ margin: 28px 0 20px 18%;
background: #5c697b;
border-radius: 3px;
box-shadow: 0 0 50px rgba(0,0,0,0.55);
@@ -221,7 +221,7 @@
display: block;
position: absolute;
width: 0px;
- margin: -6px 0 0 38%;
+ margin: -6px 0 0 22%;
}
.page .directory_list ul {
diff --git a/couchpotato/templates/_desktop.html b/couchpotato/templates/_desktop.html
index 63aa4dd8..e6d1ccb2 100644
--- a/couchpotato/templates/_desktop.html
+++ b/couchpotato/templates/_desktop.html
@@ -66,8 +66,8 @@
'data': {
'type': 'error',
'message': Browser.name + ' ' + Browser.version + ': \n' + message,
- 'page': window.location.href,
- 'file': file,
+ 'page': window.location.href.replace(window.location.host, 'HOST'),
+ 'file': file.replace(window.location.host, 'HOST'),
'line': line
}
});
diff --git a/couchpotato/templates/api.html b/couchpotato/templates/api.html
index ec067f95..f0e76e36 100644
--- a/couchpotato/templates/api.html
+++ b/couchpotato/templates/api.html
@@ -18,6 +18,12 @@
You can also use the API over another domain using JSONP, the callback function should be in 'callback_func'
{{ fireEvent('app.api_url', single = True)|safe }}/updater.info/?callback_func=myfunction
+
+
+ Get the API key:
+ /getkey/?p=md5(password)&u=md5(username)
+ Will return {"api_key": "XXXXXXXXXX", "success": true}. When username or password is empty you don't need to md5 it.
+
{% for route in routes %}
diff --git a/init/synology b/init/synology
new file mode 100644
index 00000000..859582bd
--- /dev/null
+++ b/init/synology
@@ -0,0 +1,81 @@
+#!/bin/sh
+
+# Package
+PACKAGE="couchpotato"
+DNAME="CouchPotato"
+
+# Others
+INSTALL_DIR="/opt/${PACKAGE}"
+PYTHON_DIR="/opt/bin"
+PATH="${INSTALL_DIR}/bin:${INSTALL_DIR}/env/bin:${PYTHON_DIR}/bin:/usr/local/bin:/bin:/usr/bin:/usr/syno/bin"
+RUNAS="couchpotato"
+PYTHON="${PYTHON_DIR}/python2.7"
+COUCHPOTATO="${INSTALL_DIR}/CouchPotato.py"
+CFG_FILE="${INSTALL_DIR}/var/settings.conf"
+PID_FILE="${INSTALL_DIR}/var/couchpotato.pid"
+LOG_FILE="${INSTALL_DIR}/var/logs/CouchPotato.log"
+
+
+start_daemon()
+{
+su ${RUNAS} -c "PATH=${PATH} ${PYTHON} ${COUCHPOTATO} --daemon --pid_file ${PID_FILE} --config ${CFG_FILE}"
+}
+
+stop_daemon()
+{
+ kill `cat ${PID_FILE}`
+ wait_for_status 1 20
+ rm -f ${PID_FILE}
+}
+
+daemon_status()
+{
+ if [ -f ${PID_FILE} ] && [ -d /proc/`cat ${PID_FILE}` ]; then
+return 0
+ fi
+return 1
+}
+
+wait_for_status()
+{
+ counter=$2
+ while [ ${counter} -gt 0 ]; do
+daemon_status
+ [ $? -eq $1 ] && break
+let counter=counter-1
+ sleep 1
+ done
+}
+case $1 in
+ start)
+ if daemon_status; then
+echo ${DNAME} is already running
+ else
+echo Starting ${DNAME} ...
+ start_daemon
+ fi
+ ;;
+ stop)
+ if daemon_status; then
+echo Stopping ${DNAME} ...
+ stop_daemon
+ else
+echo ${DNAME} is not running
+ fi
+ ;;
+ status)
+ if daemon_status; then
+echo ${DNAME} is running
+ exit 0
+ else
+echo ${DNAME} is not running
+ exit 1
+ fi
+ ;;
+ log)
+ echo ${LOG_FILE}
+ ;;
+ *)
+ exit 1
+ ;;
+esac
\ No newline at end of file
diff --git a/init/ubuntu b/init/ubuntu
index 8abc9e79..d6af148a 100644
--- a/init/ubuntu
+++ b/init/ubuntu
@@ -45,7 +45,7 @@ case "$1" in
start)
echo "Starting $DESC"
rm -rf $PID_PATH || return 1
- install -d --mode=0755 -o $RUN_AS -g $RUN_AS $PID_PATH || return 1
+ install -d --mode=0755 -o $RUN_AS $PID_PATH || return 1
start-stop-daemon -d $APP_PATH -c $RUN_AS --start --background --pidfile $PID_FILE --exec $DAEMON -- $DAEMON_OPTS
;;
stop)
diff --git a/libs/axl/axel.py b/libs/axl/axel.py
index 5607450d..86921a97 100644
--- a/libs/axl/axel.py
+++ b/libs/axl/axel.py
@@ -56,7 +56,7 @@ class Event(object):
>> None
"""
- def __init__(self, sender = None, asynch = False, exc_info = False,
+ def __init__(self, name = None, sender = None, asynch = False, exc_info = False,
lock = None, threads = 3, traceback = False):
""" Creates an event
@@ -96,6 +96,7 @@ class Event(object):
)
"""
self.in_order = False
+ self.name = name
self.asynchronous = asynch
self.exc_info = exc_info
self.lock = lock
@@ -178,7 +179,7 @@ class Event(object):
""" Executes all handlers stored in the queue """
while True:
try:
- h_ = self.queue.get()
+ h_ = self.queue.get(timeout = 2)
handler, memoize, timeout = self.handlers[h_]
if self.lock and self.in_order:
diff --git a/libs/enzyme/__init__.py b/libs/enzyme/__init__.py
index 7bf572e9..c283dd59 100644
--- a/libs/enzyme/__init__.py
+++ b/libs/enzyme/__init__.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2003-2006 Thomas Schueppel
-# Copyright (C) 2003-2006 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Thomas Schueppel
+# Copyright 2003-2006 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -17,14 +17,13 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
+# along with enzyme. If not, see .
import mimetypes
import os
import sys
from exceptions import *
+
PARSERS = [('asf', ['video/asf'], ['asf', 'wmv', 'wma']),
('flv', ['video/flv'], ['flv']),
('mkv', ['video/x-matroska', 'application/mkv'], ['mkv', 'mka', 'webm']),
@@ -32,10 +31,18 @@ PARSERS = [('asf', ['video/asf'], ['asf', 'wmv', 'wma']),
('mpeg', ['video/mpeg'], ['mpeg', 'mpg', 'mp4', 'ts']),
('ogm', ['application/ogg'], ['ogm', 'ogg', 'ogv']),
('real', ['video/real'], ['rm', 'ra', 'ram']),
- ('riff', ['video/avi'], ['wav', 'avi'])]
+ ('riff', ['video/avi'], ['wav', 'avi'])
+]
def parse(path):
+ """Parse metadata of the given video
+
+ :param string path: path to the video file to parse
+ :return: a parser corresponding to the video's mimetype or extension
+ :rtype: :class:`~enzyme.core.AVContainer`
+
+ """
if not os.path.isfile(path):
raise ValueError('Invalid path')
extension = os.path.splitext(path)[1][1:]
@@ -47,7 +54,7 @@ def parse(path):
parser_mime = parser_name
if extension in parser_extensions:
parser_ext = parser_name
- parser = parser_mime or parser_ext
+ parser = parser_mime or parser_ext
if not parser:
raise NoParserError()
mod = __import__(parser, globals=globals(), locals=locals(), fromlist=[], level=-1)
diff --git a/libs/enzyme/asf.py b/libs/enzyme/asf.py
index e7cca1b2..a623b591 100644
--- a/libs/enzyme/asf.py
+++ b/libs/enzyme/asf.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2003-2006 Thomas Schueppel
-# Copyright (C) 2003-2006 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Thomas Schueppel
+# Copyright 2003-2006 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -17,30 +17,29 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
+# along with enzyme. If not, see .
+from exceptions import ParseError
+import core
+import logging
+import string
+import struct
+
__all__ = ['Parser']
-import struct
-import string
-import logging
-from exceptions import *
-import core
# get logging object
log = logging.getLogger(__name__)
def _guid(input):
# Remove any '-'
- s = string.join(string.split(input,'-'), '')
+ s = string.join(string.split(input, '-'), '')
r = ''
if len(s) != 32:
return ''
- x = ''
- for i in range(0,16):
- r+=chr(int(s[2*i:2*i+2],16))
- guid = struct.unpack('>IHHBB6s',r)
+ for i in range(0, 16):
+ r += chr(int(s[2 * i:2 * i + 2], 16))
+ guid = struct.unpack('>IHHBB6s', r)
return guid
GUIDS = {
@@ -93,8 +92,7 @@ GUIDS = {
'ASF_Web_Stream_Format' : _guid('DA1E6B13-8359-4050-B398-388E965BF00C'),
'ASF_No_Error_Correction' : _guid('20FB5700-5B55-11CF-A8FD-00805F5C442B'),
- 'ASF_Audio_Spread' : _guid('BFC3CD50-618F-11CF-8BB2-00AA00B4E220'),
- }
+ 'ASF_Audio_Spread' : _guid('BFC3CD50-618F-11CF-8BB2-00AA00B4E220')}
class Asf(core.AVContainer):
@@ -114,7 +112,7 @@ class Asf(core.AVContainer):
raise ParseError()
(guidstr, objsize, objnum, reserved1, \
- reserved2) = struct.unpack('<16sQIBB',h)
+ reserved2) = struct.unpack('<16sQIBB', h)
guid = self._parseguid(guidstr)
if (guid != GUIDS['ASF_Header_Object']):
@@ -122,9 +120,9 @@ class Asf(core.AVContainer):
if reserved1 != 0x01 or reserved2 != 0x02:
raise ParseError()
- log.debug("asf header size: %d / %d objects" % (objsize,objnum))
- header = file.read(objsize-30)
- for i in range(0,objnum):
+ log.debug(u'Header size: %d / %d objects' % (objsize, objnum))
+ header = file.read(objsize - 30)
+ for _ in range(0, objnum):
h = self._getnextheader(header)
header = header[h[1]:]
@@ -148,19 +146,19 @@ class Asf(core.AVContainer):
stream._appendtable('ASFMETADATA', metadata)
- def _parseguid(self,string):
+ def _parseguid(self, string):
return struct.unpack('
-# Copyright (C) 2003-2006 Thomas Schueppel
-# Copyright (C) 2003-2006 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Thomas Schueppel
+# Copyright 2003-2006 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -17,21 +17,14 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
-
+# along with enzyme. If not, see .
import re
import logging
import fourcc
import language
-from exceptions import *
from strutils import str_to_unicode, unicode_to_str
UNPRINTABLE_KEYS = ['thumbnail', 'url', 'codec_private']
-EXTENSION_DEVICE = 'device'
-EXTENSION_DIRECTORY = 'directory'
-EXTENSION_STREAM = 'stream'
MEDIACORE = ['title', 'caption', 'comment', 'size', 'type', 'subtype', 'timestamp',
'keywords', 'country', 'language', 'langcode', 'url', 'artist',
'mime', 'datetime', 'tags', 'hash']
@@ -52,8 +45,6 @@ log = logging.getLogger(__name__)
class Media(object):
- media = None
-
"""
Media is the base class to all Media Metadata Containers. It defines
the basic structures that handle metadata. Media and its derivates
@@ -61,10 +52,11 @@ class Media(object):
Specific derivates contain additional keys to the dublin core set that is
defined in Media.
"""
+ media = None
_keys = MEDIACORE
table_mapping = {}
- def __init__(self, hash = None):
+ def __init__(self, hash=None):
if hash is not None:
# create Media based on dict
for key, value in hash.items():
@@ -139,20 +131,21 @@ class Media(object):
result += '| ' + re.sub(r'\n(.)', r'\n| \1', unicode(item))
# print tables
- if log.level >= 10:
- for name, table in self.tables.items():
- result += '+-- Table %s\n' % str(name)
- for key, value in table.items():
- try:
- value = unicode(value)
- if len(value) > 50:
- value = u'' % len(value)
- except (UnicodeDecodeError, TypeError), e:
- try:
- value = u'' % len(value)
- except AttributeError:
- value = u''
- result += u'| | %s: %s\n' % (unicode(key), value)
+ #FIXME: WTH?
+# if log.level >= 10:
+# for name, table in self.tables.items():
+# result += '+-- Table %s\n' % str(name)
+# for key, value in table.items():
+# try:
+# value = unicode(value)
+# if len(value) > 50:
+# value = u'' % len(value)
+# except (UnicodeDecodeError, TypeError):
+# try:
+# value = u'' % len(value)
+# except AttributeError:
+# value = u''
+# result += u'| | %s: %s\n' % (unicode(key), value)
return result
def __str__(self):
@@ -253,7 +246,7 @@ class Media(object):
"""
return hasattr(self, key)
- def get(self, attr, default = None):
+ def get(self, attr, default=None):
"""
Returns the given attribute. If the attribute is not set by
the parser return 'default'.
@@ -315,7 +308,7 @@ class Tag(object):
Tag values are strings (for binary data), unicode objects, or datetime
objects for tags that represent dates or times.
"""
- def __init__(self, value = None, langcode = 'und', binary = False):
+ def __init__(self, value=None, langcode='und', binary=False):
super(Tag, self).__init__()
self.value = value
self.langcode = langcode
@@ -363,7 +356,7 @@ class Tags(dict, Tag):
The attribute RATING has a value (PG), but it also has a child tag
COUNTRY that specifies the country code the rating belongs to.
"""
- def __init__(self, value = None, langcode = 'und', binary = False):
+ def __init__(self, value=None, langcode='und', binary=False):
super(Tags, self).__init__()
self.value = value
self.langcode = langcode
@@ -410,7 +403,7 @@ class Chapter(Media):
"""
_keys = ['enabled', 'name', 'pos', 'id']
- def __init__(self, name = None, pos = 0):
+ def __init__(self, name=None, pos=0):
Media.__init__(self)
self.name = name
self.pos = pos
@@ -424,7 +417,7 @@ class Subtitle(Media):
_keys = ['enabled', 'default', 'langcode', 'language', 'trackno', 'title',
'id', 'codec']
- def __init__(self, language = None):
+ def __init__(self, language=None):
Media.__init__(self)
self.language = language
diff --git a/libs/enzyme/exceptions.py b/libs/enzyme/exceptions.py
index 403aa947..5bc4f7ac 100644
--- a/libs/enzyme/exceptions.py
+++ b/libs/enzyme/exceptions.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
+# Copyright 2011-2012 Antoine Bertin
#
# This file is part of enzyme.
#
@@ -15,10 +15,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
-
+# along with enzyme. If not, see .
class Error(Exception):
pass
diff --git a/libs/enzyme/flv.py b/libs/enzyme/flv.py
index c2dd4067..61f34a78 100644
--- a/libs/enzyme/flv.py
+++ b/libs/enzyme/flv.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2003-2006 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -16,16 +16,15 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
-__all__ = ['Parser']
-
-from exceptions import *
+# along with enzyme. If not, see .
+from exceptions import ParseError
import core
import logging
import struct
+__all__ = ['Parser']
+
+
# get logging object
log = logging.getLogger(__name__)
@@ -44,7 +43,7 @@ FLV_AUDIO_CODECID = (0x0001, 0x0002, 0x0055, 0x0001)
# video flags
FLV_VIDEO_CODECID_MASK = 0x0f
-FLV_VIDEO_CODECID = ( 'FLV1', 'MSS1', 'VP60') # wild guess
+FLV_VIDEO_CODECID = ('FLV1', 'MSS1', 'VP60') # wild guess
FLV_DATA_TYPE_NUMBER = 0x00
FLV_DATA_TYPE_BOOL = 0x01
@@ -70,7 +69,7 @@ class FlashVideo(core.AVContainer):
"""
table_mapping = { 'FLVINFO' : FLVINFO }
- def __init__(self,file):
+ def __init__(self, file):
core.AVContainer.__init__(self)
self.mime = 'video/flv'
self.type = 'Flash Video'
@@ -78,7 +77,7 @@ class FlashVideo(core.AVContainer):
if len(data) < 13 or struct.unpack('>3sBBII', data)[0] != 'FLV':
raise ParseError()
- for i in range(10):
+ for _ in range(10):
if self.audio and self.video:
break
data = file.read(11)
@@ -116,13 +115,13 @@ class FlashVideo(core.AVContainer):
file.seek(size - 1, 1)
elif chunk[0] == FLV_TAG_TYPE_META:
- log.info('metadata %s', str(chunk))
+ log.info(u'metadata %r', str(chunk))
metadata = file.read(size)
try:
while metadata:
length, value = self._parse_value(metadata)
if isinstance(value, dict):
- log.info('metadata: %s', value)
+ log.info(u'metadata: %r', value)
if value.get('creator'):
self.copyright = value.get('creator')
if value.get('width'):
@@ -139,7 +138,7 @@ class FlashVideo(core.AVContainer):
except (IndexError, struct.error, TypeError):
pass
else:
- log.info('unkown %s', str(chunk))
+ log.info(u'unkown %r', str(chunk))
file.seek(size, 1)
file.seek(4, 1)
@@ -157,16 +156,16 @@ class FlashVideo(core.AVContainer):
if ord(data[0]) == FLV_DATA_TYPE_STRING:
length = (ord(data[1]) << 8) + ord(data[2])
- return length + 3, data[3:length+3]
+ return length + 3, data[3:length + 3]
if ord(data[0]) == FLV_DATA_TYPE_ECMARRAY:
init_length = len(data)
num = struct.unpack('>I', data[1:5])[0]
data = data[5:]
result = {}
- for i in range(num):
+ for _ in range(num):
length = (ord(data[0]) << 8) + ord(data[1])
- key = data[2:length+2]
+ key = data[2:length + 2]
data = data[length + 2:]
length, value = self._parse_value(data)
if not length:
@@ -175,7 +174,7 @@ class FlashVideo(core.AVContainer):
data = data[length:]
return init_length - len(data), result
- log.info('unknown code: %x. Stop metadata parser', ord(data[0]))
+ log.info(u'unknown code: %x. Stop metadata parser', ord(data[0]))
return 0, None
diff --git a/libs/enzyme/fourcc.py b/libs/enzyme/fourcc.py
index 60cbb3c8..ac15b0b2 100644
--- a/libs/enzyme/fourcc.py
+++ b/libs/enzyme/fourcc.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2003-2006 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -16,10 +16,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
-
+# along with enzyme. If not, see .
import string
import re
import struct
diff --git a/libs/enzyme/infos.py b/libs/enzyme/infos.py
index d6196682..a6f0bcc5 100644
--- a/libs/enzyme/infos.py
+++ b/libs/enzyme/infos.py
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
+# Copyright 2011-2012 Antoine Bertin
#
# This file is part of enzyme.
#
@@ -15,12 +15,5 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
-__title__ = 'enzyme'
-__description__ = 'Video metadata parser'
-__version__ = '0.1'
-__author__ = 'Antoine Bertin'
-__email__ = 'diaoulael@gmail.com'
-__license__ = 'GPLv3'
+# along with enzyme. If not, see .
+__version__ = '0.2'
diff --git a/libs/enzyme/language.py b/libs/enzyme/language.py
index 9a52692a..3957f9d9 100644
--- a/libs/enzyme/language.py
+++ b/libs/enzyme/language.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2003-2006 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -16,10 +16,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
-
+# along with enzyme. If not, see .
import re
__all__ = ['resolve']
@@ -44,7 +41,7 @@ def resolve(code):
if code in spec[:-1]:
return code, spec[-1]
- return code, u'Unknown (%s)' % code
+ return code, u'Unknown (%r)' % code
# Parsed from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
diff --git a/libs/enzyme/mkv.py b/libs/enzyme/mkv.py
index eac0a380..aba5325e 100644
--- a/libs/enzyme/mkv.py
+++ b/libs/enzyme/mkv.py
@@ -1,9 +1,9 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2003-2006 Thomas Schueppel
-# Copyright (C) 2003-2006 Dirk Meyer
-# Copyright (C) 2003-2006 Jason Tackaberry
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Thomas Schueppel
+# Copyright 2003-2006 Dirk Meyer
+# Copyright 2003-2006 Jason Tackaberry
#
# This file is part of enzyme.
#
@@ -18,18 +18,17 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
-__all__ = ['Parser']
-
+# along with enzyme. If not, see .
from datetime import datetime
-from exceptions import *
+from exceptions import ParseError
from struct import unpack
import core
import logging
import re
+__all__ = ['Parser']
+
+
# get logging object
log = logging.getLogger(__name__)
@@ -236,7 +235,7 @@ class EbmlEntity:
self.compute_id(inbuf)
if self.id_len == 0:
- log.error("EBML entity not found, bad file format")
+ log.error(u'EBML entity not found, bad file format')
raise ParseError()
self.entity_len, self.len_size = self.compute_len(inbuf[self.id_len:])
@@ -247,7 +246,7 @@ class EbmlEntity:
# if the data size is 8 or less, it could be a numeric value
self.value = 0
if self.entity_len <= 8:
- for pos, shift in zip(range(self.entity_len), range((self.entity_len-1)*8, -1, -8)):
+ for pos, shift in zip(range(self.entity_len), range((self.entity_len - 1) * 8, -1, -8)):
self.value |= ord(self.entity_data[pos]) << shift
@@ -271,19 +270,19 @@ class EbmlEntity:
if len(inbuf) < 2:
return 0
self.id_len = 2
- self.entity_id = ord(inbuf[0])<<8 | ord(inbuf[1])
+ self.entity_id = ord(inbuf[0]) << 8 | ord(inbuf[1])
elif first & 0x20:
if len(inbuf) < 3:
return 0
self.id_len = 3
- self.entity_id = (ord(inbuf[0])<<16) | (ord(inbuf[1])<<8) | \
+ self.entity_id = (ord(inbuf[0]) << 16) | (ord(inbuf[1]) << 8) | \
(ord(inbuf[2]))
elif first & 0x10:
if len(inbuf) < 4:
return 0
self.id_len = 4
- self.entity_id = (ord(inbuf[0])<<24) | (ord(inbuf[1])<<16) | \
- (ord(inbuf[2])<<8) | (ord(inbuf[3]))
+ self.entity_id = (ord(inbuf[0]) << 24) | (ord(inbuf[1]) << 16) | \
+ (ord(inbuf[2]) << 8) | (ord(inbuf[3]))
self.entity_str = inbuf[0:self.id_len]
@@ -381,7 +380,7 @@ class Matroska(core.AVContainer):
if header.get_id() != MATROSKA_HEADER_ID:
raise ParseError()
- log.debug("HEADER ID found %08X" % header.get_id() )
+ log.debug(u'HEADER ID found %08X' % header.get_id())
self.mime = 'video/x-matroska'
self.type = 'Matroska'
self.has_idx = False
@@ -392,10 +391,10 @@ class Matroska(core.AVContainer):
# Record file offset of segment data for seekheads
self.segment.offset = header.get_total_len() + segment.get_header_len()
if segment.get_id() != MATROSKA_SEGMENT_ID:
- log.debug("SEGMENT ID not found %08X" % segment.get_id())
+ log.debug(u'SEGMENT ID not found %08X' % segment.get_id())
return
- log.debug("SEGMENT ID found %08X" % segment.get_id())
+ log.debug(u'SEGMENT ID found %08X' % segment.get_id())
try:
for elem in self.process_one_level(segment):
if elem.get_id() == MATROSKA_SEEKHEAD_ID:
@@ -404,12 +403,12 @@ class Matroska(core.AVContainer):
pass
if not self.has_idx:
- log.warning('File has no index')
+ log.warning(u'File has no index')
self._set('corrupt', True)
def process_elem(self, elem):
elem_id = elem.get_id()
- log.debug('BEGIN: process element %s' % hex(elem_id))
+ log.debug(u'BEGIN: process element %r' % hex(elem_id))
if elem_id == MATROSKA_SEGMENT_INFO_ID:
duration = 0
scalecode = 1000000.0
@@ -423,7 +422,7 @@ class Matroska(core.AVContainer):
elif ielem_id == MATROSKA_TITLE_ID:
self.title = ielem.get_utf8()
elif ielem_id == MATROSKA_DATE_UTC_ID:
- timestamp = unpack('!q', ielem.get_data())[0] / 10.0**9
+ timestamp = unpack('!q', ielem.get_data())[0] / 10.0 ** 9
# Date is offset 2001-01-01 00:00:00 (timestamp 978307200.0)
self.timestamp = int(timestamp + 978307200)
@@ -447,7 +446,7 @@ class Matroska(core.AVContainer):
elif elem_id == MATROSKA_CUES_ID:
self.has_idx = True
- log.debug('END: process element %s' % hex(elem_id))
+ log.debug(u'END: process element %r' % hex(elem_id))
return True
@@ -479,7 +478,7 @@ class Matroska(core.AVContainer):
index = 0
while index < tracks.get_len():
trackelem = EbmlEntity(tracksbuf[index:])
- log.debug ("ELEMENT %X found" % trackelem.get_id())
+ log.debug (u'ELEMENT %X found' % trackelem.get_id())
self.process_track(trackelem)
index += trackelem.get_total_len() + trackelem.get_crc_len()
@@ -503,20 +502,20 @@ class Matroska(core.AVContainer):
elements = [x for x in self.process_one_level(track)]
track_type = [x.get_value() for x in elements if x.get_id() == MATROSKA_TRACK_TYPE_ID]
if not track_type:
- log.debug('Bad track: no type id found')
+ log.debug(u'Bad track: no type id found')
return
track_type = track_type[0]
track = None
if track_type == MATROSKA_VIDEO_TRACK:
- log.debug("Video track found")
+ log.debug(u'Video track found')
track = self.process_video_track(elements)
elif track_type == MATROSKA_AUDIO_TRACK:
- log.debug("Audio track found")
+ log.debug(u'Audio track found')
track = self.process_audio_track(elements)
elif track_type == MATROSKA_SUBTITLES_TRACK:
- log.debug("Subtitle track found")
+ log.debug(u'Subtitle track found')
track = core.Subtitle()
self.set_track_defaults(track)
track.id = len(self.subtitles)
@@ -529,7 +528,7 @@ class Matroska(core.AVContainer):
elem_id = elem.get_id()
if elem_id == MATROSKA_TRACK_LANGUAGE_ID:
track.language = elem.get_str()
- log.debug("Track language found: %s" % track.language)
+ log.debug(u'Track language found: %r' % track.language)
elif elem_id == MATROSKA_NAME_ID:
track.title = elem.get_utf8()
elif elem_id == MATROSKA_TRACK_NUMBER_ID:
@@ -671,7 +670,7 @@ class Matroska(core.AVContainer):
elif elem_id == MATROSKA_CHAPTER_UID_ID:
self.objects_by_uid[elem.get_value()] = chap
- log.debug('Chapter "%s" found', chap.name)
+ log.debug(u'Chapter %r found', chap.name)
chap.id = len(self.chapters)
self.chapters.append(chap)
@@ -704,10 +703,10 @@ class Matroska(core.AVContainer):
# Right now we only support attachments that could be cover images.
# Make a guess to see if this attachment is a cover image.
- if mimetype.startswith("image/") and u"cover" in (name+desc).lower() and data:
+ if mimetype.startswith("image/") and u"cover" in (name + desc).lower() and data:
self.thumbnail = data
- log.debug('Attachment "%s" found' % name)
+ log.debug(u'Attachment %r found' % name)
def process_tags(self, tags):
@@ -744,7 +743,7 @@ class Matroska(core.AVContainer):
self.objects_by_uid[target].tags.update(tags_dict)
self.tags_to_attributes(self.objects_by_uid[target], tags_dict)
except KeyError:
- log.warning('Tags assigned to unknown/unsupported target uid %d', target)
+ log.warning(u'Tags assigned to unknown/unsupported target uid %d', target)
else:
self.tags.update(tags_dict)
self.tags_to_attributes(self, tags_dict)
@@ -816,7 +815,7 @@ class Matroska(core.AVContainer):
try:
value = [filter(item) for item in value] if isinstance(value, list) else filter(value)
except Exception, e:
- log.warning('Failed to convert tag to core attribute: %s', e)
+ log.warning(u'Failed to convert tag to core attribute: %r', e)
# Special handling for tv series recordings. The 'title' tag
# can be used for both the series and the episode name. The
# same is true for trackno which may refer to the season
diff --git a/libs/enzyme/mp4.py b/libs/enzyme/mp4.py
index 41bf43da..c53f30d3 100644
--- a/libs/enzyme/mp4.py
+++ b/libs/enzyme/mp4.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2003-2007 Thomas Schueppel
-# Copyright (C) 2003-2007 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2007 Thomas Schueppel
+# Copyright 2003-2007 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -17,16 +17,14 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
+# along with enzyme. If not, see .
__all__ = ['Parser']
import zlib
import logging
import StringIO
import struct
-from exceptions import *
+from exceptions import ParseError
import core
# get logging object
@@ -169,7 +167,7 @@ class MPEG4(core.AVContainer):
self.type = 'Quicktime Video'
h = file.read(8)
try:
- (size, type) = struct.unpack('>I4s',h)
+ (size, type) = struct.unpack('>I4s', h)
except struct.error:
# EOF.
raise ParseError()
@@ -183,18 +181,18 @@ class MPEG4(core.AVContainer):
self.mime = 'video/mp4'
self.type = 'MPEG-4 Video'
size -= 4
- file.seek(size-8, 1)
+ file.seek(size - 8, 1)
h = file.read(8)
- (size, type) = struct.unpack('>I4s',h)
+ (size, type) = struct.unpack('>I4s', h)
while type in ['mdat', 'skip']:
# movie data at the beginning, skip
- file.seek(size-8, 1)
+ file.seek(size - 8, 1)
h = file.read(8)
- (size, type) = struct.unpack('>I4s',h)
+ (size, type) = struct.unpack('>I4s', h)
if not type in ['moov', 'wide', 'free']:
- log.debug('invalid header: %r' % type)
+ log.debug(u'invalid header: %r' % type)
raise ParseError()
# Extended size
@@ -216,37 +214,37 @@ class MPEG4(core.AVContainer):
if len(s) < 8:
return 0
- atomsize,atomtype = struct.unpack('>I4s', s)
+ atomsize, atomtype = struct.unpack('>I4s', s)
if not str(atomtype).decode('latin1').isalnum():
# stop at nonsense data
return 0
- log.debug('%s [%X]' % (atomtype,atomsize))
+ log.debug(u'%r [%X]' % (atomtype, atomsize))
if atomtype == 'udta':
# Userdata (Metadata)
pos = 0
tabl = {}
i18ntabl = {}
- atomdata = file.read(atomsize-8)
- while pos < atomsize-12:
- (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8])
+ atomdata = file.read(atomsize - 8)
+ while pos < atomsize - 12:
+ (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
if ord(datatype[0]) == 169:
# i18n Metadata...
- mypos = 8+pos
- while mypos + 4 < datasize+pos:
+ mypos = 8 + pos
+ while mypos + 4 < datasize + pos:
# first 4 Bytes are i18n header
- (tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos+4])
+ (tlen, lang) = struct.unpack('>HH', atomdata[mypos:mypos + 4])
i18ntabl[lang] = i18ntabl.get(lang, {})
- l = atomdata[mypos+4:mypos+tlen+4]
+ l = atomdata[mypos + 4:mypos + tlen + 4]
i18ntabl[lang][datatype[1:]] = l
- mypos += tlen+4
+ mypos += tlen + 4
elif datatype == 'WLOC':
# Drop Window Location
pass
else:
- if ord(atomdata[pos+8:pos+datasize][0]) > 1:
- tabl[datatype] = atomdata[pos+8:pos+datasize]
+ if ord(atomdata[pos + 8:pos + datasize][0]) > 1:
+ tabl[datatype] = atomdata[pos + 8:pos + datasize]
pos += datasize
if len(i18ntabl.keys()) > 0:
for k in i18ntabl.keys():
@@ -254,19 +252,19 @@ class MPEG4(core.AVContainer):
self._appendtable('QTUDTA', i18ntabl[k])
self._appendtable('QTUDTA', tabl)
else:
- log.debug('NO i18')
+ log.debug(u'NO i18')
self._appendtable('QTUDTA', tabl)
elif atomtype == 'trak':
- atomdata = file.read(atomsize-8)
+ atomdata = file.read(atomsize - 8)
pos = 0
trackinfo = {}
tracktype = None
- while pos < atomsize-8:
- (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8])
+ while pos < atomsize - 8:
+ (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
if datatype == 'tkhd':
- tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos+8:pos+datasize])
+ tkhd = struct.unpack('>6I8x4H36xII', atomdata[pos + 8:pos + datasize])
trackinfo['width'] = tkhd[10] >> 16
trackinfo['height'] = tkhd[11] >> 16
trackinfo['id'] = tkhd[3]
@@ -277,23 +275,23 @@ class MPEG4(core.AVContainer):
# XXX Apple time. FIXME to work on Apple, too
self.timestamp = int(tkhd[1]) - 2082844800
except Exception, e:
- log.exception('There was trouble extracting timestamp')
+ log.exception(u'There was trouble extracting timestamp')
elif datatype == 'mdia':
pos += 8
datasize -= 8
- log.debug('--> mdia information')
+ log.debug(u'--> mdia information')
while datasize:
- mdia = struct.unpack('>I4s', atomdata[pos:pos+8])
+ mdia = struct.unpack('>I4s', atomdata[pos:pos + 8])
if mdia[1] == 'mdhd':
# Parse based on version of mdhd header. See
# http://wiki.multimedia.cx/index.php?title=QuickTime_container#mdhd
ver = ord(atomdata[pos + 8])
if ver == 0:
- mdhd = struct.unpack('>IIIIIhh', atomdata[pos+8:pos+8+24])
+ mdhd = struct.unpack('>IIIIIhh', atomdata[pos + 8:pos + 8 + 24])
elif ver == 1:
- mdhd = struct.unpack('>IQQIQhh', atomdata[pos+8:pos+8+36])
+ mdhd = struct.unpack('>IQQIQhh', atomdata[pos + 8:pos + 8 + 36])
else:
mdhd = None
@@ -313,34 +311,34 @@ class MPEG4(core.AVContainer):
pos -= (mdia[0] - 8)
datasize += (mdia[0] - 8)
elif mdia[1] == 'hdlr':
- hdlr = struct.unpack('>I4s4s', atomdata[pos+8:pos+8+12])
+ hdlr = struct.unpack('>I4s4s', atomdata[pos + 8:pos + 8 + 12])
if hdlr[1] == 'mhlr':
if hdlr[2] == 'vide':
tracktype = 'video'
if hdlr[2] == 'soun':
tracktype = 'audio'
elif mdia[1] == 'stsd':
- stsd = struct.unpack('>2I', atomdata[pos+8:pos+8+8])
+ stsd = struct.unpack('>2I', atomdata[pos + 8:pos + 8 + 8])
if stsd[1] > 0:
- codec = atomdata[pos+16:pos+16+8]
+ codec = atomdata[pos + 16:pos + 16 + 8]
codec = struct.unpack('>I4s', codec)
trackinfo['codec'] = codec[1]
if codec[1] == 'jpeg':
tracktype = 'image'
elif mdia[1] == 'dinf':
- dref = struct.unpack('>I4s', atomdata[pos+8:pos+8+8])
- log.debug(' --> %s, %s (useless)' % mdia)
+ dref = struct.unpack('>I4s', atomdata[pos + 8:pos + 8 + 8])
+ log.debug(u' --> %r, %r (useless)' % mdia)
if dref[1] == 'dref':
- num = struct.unpack('>I', atomdata[pos+20:pos+20+4])[0]
- rpos = pos+20+4
+ num = struct.unpack('>I', atomdata[pos + 20:pos + 20 + 4])[0]
+ rpos = pos + 20 + 4
for ref in range(num):
# FIXME: do somthing if this references
- ref = struct.unpack('>I3s', atomdata[rpos:rpos+7])
- data = atomdata[rpos+7:rpos+ref[0]]
+ ref = struct.unpack('>I3s', atomdata[rpos:rpos + 7])
+ data = atomdata[rpos + 7:rpos + ref[0]]
rpos += ref[0]
else:
if mdia[1].startswith('st'):
- log.debug(' --> %s, %s (sample)' % mdia)
+ log.debug(u' --> %r, %r (sample)' % mdia)
elif mdia[1] == 'vmhd' and not tracktype:
# indicates that this track is video
tracktype = 'video'
@@ -348,19 +346,19 @@ class MPEG4(core.AVContainer):
# indicates that this track is audio
tracktype = 'audio'
else:
- log.debug(' --> %s, %s (unknown)' % mdia)
+ log.debug(u' --> %r, %r (unknown)' % mdia)
pos += mdia[0]
datasize -= mdia[0]
elif datatype == 'udta':
- log.debug(struct.unpack('>I4s', atomdata[:8]))
+ log.debug(u'udta: %r' % struct.unpack('>I4s', atomdata[:8]))
else:
if datatype == 'edts':
- log.debug('--> %s [%d] (edit list)' % \
+ log.debug(u'--> %r [%d] (edit list)' % \
(datatype, datasize))
else:
- log.debug('--> %s [%d] (unknown)' % \
+ log.debug(u'--> %r [%d] (unknown)' % \
(datatype, datasize))
pos += datasize
@@ -380,7 +378,7 @@ class MPEG4(core.AVContainer):
mvhd = struct.unpack('>6I2h', file.read(28))
self.length = max(self.length, mvhd[4] / mvhd[3])
self.volume = mvhd[6]
- file.seek(atomsize-8-28,1)
+ file.seek(atomsize - 8 - 28, 1)
elif atomtype == 'cmov':
@@ -389,21 +387,21 @@ class MPEG4(core.AVContainer):
if not atomtype == 'dcom':
return atomsize
- method = struct.unpack('>4s', file.read(datasize-8))[0]
+ method = struct.unpack('>4s', file.read(datasize - 8))[0]
datasize, atomtype = struct.unpack('>I4s', file.read(8))
if not atomtype == 'cmvd':
return atomsize
if method == 'zlib':
- data = file.read(datasize-8)
+ data = file.read(datasize - 8)
try:
decompressed = zlib.decompress(data)
except Exception, e:
try:
decompressed = zlib.decompress(data[4:])
except Exception, e:
- log.exception('There was a proble decompressiong atom')
+ log.exception(u'There was a proble decompressiong atom')
return atomsize
decompressedIO = StringIO.StringIO(decompressed)
@@ -411,9 +409,9 @@ class MPEG4(core.AVContainer):
pass
else:
- log.info('unknown compression %s' % method)
+ log.info(u'unknown compression %r' % method)
# unknown compression method
- file.seek(datasize-8,1)
+ file.seek(datasize - 8, 1)
elif atomtype == 'moov':
# decompressed movie info
@@ -423,10 +421,10 @@ class MPEG4(core.AVContainer):
elif atomtype == 'mdat':
pos = file.tell() + atomsize - 8
# maybe there is data inside the mdat
- log.info('parsing mdat')
+ log.info(u'parsing mdat')
while self._readatom(file):
pass
- log.info('end of mdat')
+ log.info(u'end of mdat')
file.seek(pos, 0)
@@ -437,24 +435,24 @@ class MPEG4(core.AVContainer):
elif atomtype == 'rmda':
# reference
- atomdata = file.read(atomsize-8)
+ atomdata = file.read(atomsize - 8)
pos = 0
url = ''
quality = 0
datarate = 0
- while pos < atomsize-8:
- (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos+8])
+ while pos < atomsize - 8:
+ (datasize, datatype) = struct.unpack('>I4s', atomdata[pos:pos + 8])
if datatype == 'rdrf':
- rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos+8:pos+20])
+ rflags, rtype, rlen = struct.unpack('>I4sI', atomdata[pos + 8:pos + 20])
if rtype == 'url ':
- url = atomdata[pos+20:pos+20+rlen]
+ url = atomdata[pos + 20:pos + 20 + rlen]
if url.find('\0') > 0:
url = url[:url.find('\0')]
elif datatype == 'rmqu':
- quality = struct.unpack('>I', atomdata[pos+8:pos+12])[0]
+ quality = struct.unpack('>I', atomdata[pos + 8:pos + 12])[0]
elif datatype == 'rmdr':
- datarate = struct.unpack('>I', atomdata[pos+12:pos+16])[0]
+ datarate = struct.unpack('>I', atomdata[pos + 12:pos + 16])[0]
pos += datasize
if url:
@@ -462,11 +460,11 @@ class MPEG4(core.AVContainer):
else:
if not atomtype in ['wide', 'free']:
- log.info('unhandled base atom %s' % atomtype)
+ log.info(u'unhandled base atom %r' % atomtype)
# Skip unknown atoms
try:
- file.seek(atomsize-8,1)
+ file.seek(atomsize - 8, 1)
except IOError:
return 0
diff --git a/libs/enzyme/mpeg.py b/libs/enzyme/mpeg.py
index d7c44db2..3d43ba4f 100644
--- a/libs/enzyme/mpeg.py
+++ b/libs/enzyme/mpeg.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2003-2006 Thomas Schueppel
-# Copyright (C) 2003-2006 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Thomas Schueppel
+# Copyright 2003-2006 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -17,16 +17,14 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
+# along with enzyme. If not, see .
__all__ = ['Parser']
import os
import struct
import logging
import stat
-from exceptions import *
+from exceptions import ParseError
import core
# get logging object
@@ -49,7 +47,7 @@ START_CODE = {
0xB7 : 'sequence end',
0xB8 : 'group of pictures',
}
-for i in range(0x01,0xAF):
+for i in range(0x01, 0xAF):
START_CODE[i] = 'slice_start_code'
##------------------------------------------------------------------------
@@ -96,14 +94,14 @@ TS_SYNC = 0x47
##------------------------------------------------------------------------
FRAME_RATE = [
0,
- 24000.0/1001, ## 3-2 pulldown NTSC (CPB/Main Level)
- 24, ## Film (CPB/Main Level)
- 25, ## PAL/SECAM or 625/60 video
- 30000.0/1001, ## NTSC (CPB/Main Level)
- 30, ## drop-frame NTSC or component 525/60 (CPB/Main Level)
- 50, ## double-rate PAL
- 60000.0/1001, ## double-rate NTSC
- 60, ## double-rate, drop-frame NTSC/component 525/60 video
+ 24000.0 / 1001, ## 3-2 pulldown NTSC (CPB/Main Level)
+ 24, ## Film (CPB/Main Level)
+ 25, ## PAL/SECAM or 625/60 video
+ 30000.0 / 1001, ## NTSC (CPB/Main Level)
+ 30, ## drop-frame NTSC or component 525/60 (CPB/Main Level)
+ 50, ## double-rate PAL
+ 60000.0 / 1001, ## double-rate NTSC
+ 60, ## double-rate, drop-frame NTSC/component 525/60 video
]
##------------------------------------------------------------------------
@@ -114,9 +112,9 @@ FRAME_RATE = [
## it, a stream that doesn't adhere to one of these aspect ratios is
## technically considered non-compliant.
##------------------------------------------------------------------------
-ASPECT_RATIO = ( None, # Forbidden
- 1.0, # 1/1 (VGA)
- 4.0 / 3, # 4/3 (TV)
+ASPECT_RATIO = (None, # Forbidden
+ 1.0, # 1/1 (VGA)
+ 4.0 / 3, # 4/3 (TV)
16.0 / 9, # 16/9 (Widescreen)
2.21 # (Cinema)
)
@@ -131,7 +129,7 @@ class MPEG(core.AVContainer):
no additional metadata like title, etc; only codecs, length and
resolution is reported back.
"""
- def __init__(self,file):
+ def __init__(self, file):
core.AVContainer.__init__(self)
self.sequence_header_offset = 0
self.mpeg_version = 2
@@ -190,29 +188,29 @@ class MPEG(core.AVContainer):
a.codec = ac
- def dxy(self,file):
+ def dxy(self, file):
"""
get width and height of the video
"""
- file.seek(self.sequence_header_offset+4,0)
+ file.seek(self.sequence_header_offset + 4, 0)
v = file.read(4)
- x = struct.unpack('>H',v[:2])[0] >> 4
- y = struct.unpack('>H',v[1:3])[0] & 0x0FFF
- return (x,y)
+ x = struct.unpack('>H', v[:2])[0] >> 4
+ y = struct.unpack('>H', v[1:3])[0] & 0x0FFF
+ return (x, y)
- def framerate_aspect(self,file):
+ def framerate_aspect(self, file):
"""
read framerate and aspect ratio
"""
- file.seek(self.sequence_header_offset+7,0)
- v = struct.unpack( '>B', file.read(1) )[0]
+ file.seek(self.sequence_header_offset + 7, 0)
+ v = struct.unpack('>B', file.read(1))[0]
try:
- fps = FRAME_RATE[v&0xf]
+ fps = FRAME_RATE[v & 0xf]
except IndexError:
fps = None
- if v>>4 < len(ASPECT_RATIO):
- aspect = ASPECT_RATIO[v>>4]
+ if v >> 4 < len(ASPECT_RATIO):
+ aspect = ASPECT_RATIO[v >> 4]
else:
aspect = None
return (fps, aspect)
@@ -238,18 +236,18 @@ class MPEG(core.AVContainer):
if pos == -1 or len(buffer) - pos < 5:
buffer = buffer[-10:]
continue
- ext = (ord(buffer[pos+4]) >> 4)
+ ext = (ord(buffer[pos + 4]) >> 4)
if ext == 8:
pass
elif ext == 1:
- if (ord(buffer[pos+5]) >> 3) & 1:
+ if (ord(buffer[pos + 5]) >> 3) & 1:
self._set('progressive', True)
else:
self._set('interlaced', True)
return True
else:
- log.debug('ext', ext)
- buffer = buffer[pos+4:]
+ log.debug(u'ext: %r' % ext)
+ buffer = buffer[pos + 4:]
return False
@@ -292,12 +290,12 @@ class MPEG(core.AVContainer):
## Some parts in the code are based on mpgtx (mpgtx.sf.net)
- def bitrate(self,file):
+ def bitrate(self, file):
"""
read the bitrate (most of the time broken)
"""
- file.seek(self.sequence_header_offset+8,0)
- t,b = struct.unpack( '>HB', file.read(3) )
+ file.seek(self.sequence_header_offset + 8, 0)
+ t, b = struct.unpack('>HB', file.read(3))
vrate = t << 2 | b >> 6
return vrate * 400
@@ -309,9 +307,9 @@ class MPEG(core.AVContainer):
if len(buffer) < 6:
return None
- highbit = (ord(buffer[0])&0x20)>>5
+ highbit = (ord(buffer[0]) & 0x20) >> 5
- low4Bytes= ((long(ord(buffer[0])) & 0x18) >> 3) << 30
+ low4Bytes = ((long(ord(buffer[0])) & 0x18) >> 3) << 30
low4Bytes |= (ord(buffer[0]) & 0x03) << 28
low4Bytes |= ord(buffer[1]) << 20
low4Bytes |= (ord(buffer[2]) & 0xF8) << 12
@@ -319,10 +317,10 @@ class MPEG(core.AVContainer):
low4Bytes |= ord(buffer[3]) << 5
low4Bytes |= (ord(buffer[4])) >> 3
- sys_clock_ref=(ord(buffer[4]) & 0x3) << 7
- sys_clock_ref|=(ord(buffer[5]) >> 1)
+ sys_clock_ref = (ord(buffer[4]) & 0x3) << 7
+ sys_clock_ref |= (ord(buffer[5]) >> 1)
- return (long(highbit * (1<<16) * (1<<16)) + low4Bytes) / 90000
+ return (long(highbit * (1 << 16) * (1 << 16)) + low4Bytes) / 90000
def ReadSCRMpeg1(self, buffer):
@@ -340,7 +338,7 @@ class MPEG(core.AVContainer):
low4Bytes |= ord(buffer[3]) << 7;
low4Bytes |= ord(buffer[4]) >> 1;
- return (long(highbit) * (1<<16) * (1<<16) + low4Bytes) / 90000;
+ return (long(highbit) * (1 << 16) * (1 << 16) + low4Bytes) / 90000;
def ReadPTS(self, buffer):
@@ -350,7 +348,7 @@ class MPEG(core.AVContainer):
high = ((ord(buffer[0]) & 0xF) >> 1)
med = (ord(buffer[1]) << 7) + (ord(buffer[2]) >> 1)
low = (ord(buffer[3]) << 7) + (ord(buffer[4]) >> 1)
- return ((long(high) << 30 ) + (med << 15) + low) / 90000
+ return ((long(high) << 30) + (med << 15) + low) / 90000
def ReadHeader(self, buffer, offset):
@@ -358,25 +356,25 @@ class MPEG(core.AVContainer):
Handle MPEG header in buffer on position offset
Return None on error, new offset or 0 if the new offset can't be scanned
"""
- if buffer[offset:offset+3] != '\x00\x00\x01':
+ if buffer[offset:offset + 3] != '\x00\x00\x01':
return None
- id = ord(buffer[offset+3])
+ id = ord(buffer[offset + 3])
if id == PADDING_PKT:
- return offset + (ord(buffer[offset+4]) << 8) + \
- ord(buffer[offset+5]) + 6
+ return offset + (ord(buffer[offset + 4]) << 8) + \
+ ord(buffer[offset + 5]) + 6
if id == PACK_PKT:
- if ord(buffer[offset+4]) & 0xF0 == 0x20:
+ if ord(buffer[offset + 4]) & 0xF0 == 0x20:
self.type = 'MPEG-1 Video'
self.get_time = self.ReadSCRMpeg1
self.mpeg_version = 1
return offset + 12
- elif (ord(buffer[offset+4]) & 0xC0) == 0x40:
+ elif (ord(buffer[offset + 4]) & 0xC0) == 0x40:
self.type = 'MPEG-2 Video'
self.get_time = self.ReadSCRMpeg2
- return offset + (ord(buffer[offset+13]) & 0x07) + 14
+ return offset + (ord(buffer[offset + 13]) & 0x07) + 14
else:
# I have no idea what just happened, but for some DVB
# recordings done with mencoder this points to a
@@ -412,10 +410,10 @@ class MPEG(core.AVContainer):
if id in [PRIVATE_STREAM1, PRIVATE_STREAM2]:
# private stream. we don't know, but maybe we can guess later
- add = ord(buffer[offset+8])
+ add = ord(buffer[offset + 8])
# if (ord(buffer[offset+6]) & 4) or 1:
# id = ord(buffer[offset+10+add])
- if buffer[offset+11+add:offset+15+add].find('\x0b\x77') != -1:
+ if buffer[offset + 11 + add:offset + 15 + add].find('\x0b\x77') != -1:
# AC3 stream
for a in self.audio:
if a.id == id:
@@ -442,18 +440,18 @@ class MPEG(core.AVContainer):
This MPEG starts with a sequence of 0x00 followed by a PACK Header
http://dvd.sourceforge.net/dvdinfo/packhdr.html
"""
- file.seek(0,0)
+ file.seek(0, 0)
buffer = file.read(10000)
offset = 0
# seek until the 0 byte stop
- while offset < len(buffer)-100 and buffer[offset] == '\0':
+ while offset < len(buffer) - 100 and buffer[offset] == '\0':
offset += 1
offset -= 2
# test for mpeg header 0x00 0x00 0x01
header = '\x00\x00\x01%s' % chr(PACK_PKT)
- if offset < 0 or not buffer[offset:offset+4] == header:
+ if offset < 0 or not buffer[offset:offset + 4] == header:
if not force:
return 0
# brute force and try to find the pack header in the first
@@ -470,9 +468,9 @@ class MPEG(core.AVContainer):
self.ReadHeader(buffer, offset)
# store first timestamp
- self.start = self.get_time(buffer[offset+4:])
+ self.start = self.get_time(buffer[offset + 4:])
while len(buffer) > offset + 1000 and \
- buffer[offset:offset+3] == '\x00\x00\x01':
+ buffer[offset:offset + 3] == '\x00\x00\x01':
# read the mpeg header
new_offset = self.ReadHeader(buffer, offset)
@@ -486,12 +484,12 @@ class MPEG(core.AVContainer):
# skip padding 0 before a new header
while len(buffer) > offset + 10 and \
- not ord(buffer[offset+2]):
+ not ord(buffer[offset + 2]):
offset += 1
else:
# seek to new header by brute force
- offset += buffer[offset+4:].find('\x00\x00\x01') + 4
+ offset += buffer[offset + 4:].find('\x00\x00\x01') + 4
# fill in values for support functions:
self.__seek_size__ = 1000000
@@ -555,15 +553,15 @@ class MPEG(core.AVContainer):
self.video[-1]._set('id', id)
# new mpeg starting
- if buffer[header_length+9:header_length+13] == \
+ if buffer[header_length + 9:header_length + 13] == \
'\x00\x00\x01\xB3' and not self.sequence_header_offset:
# yes, remember offset for later use
- self.sequence_header_offset = offset + header_length+9
+ self.sequence_header_offset = offset + header_length + 9
elif ord(buffer[3]) == 189 or ord(buffer[3]) == 191:
# private stream. we don't know, but maybe we can guess later
id = id or ord(buffer[3]) & 0xF
if align and \
- buffer[header_length+9:header_length+11] == '\x0b\x77':
+ buffer[header_length + 9:header_length + 11] == '\x0b\x77':
# AC3 stream
for a in self.audio:
if a.id == id:
@@ -581,7 +579,7 @@ class MPEG(core.AVContainer):
if ptsdts and ptsdts == ord(buffer[9]) >> 4:
if ord(buffer[9]) >> 4 != ptsdts:
- log.warning('WARNING: bad PTS/DTS, please contact us')
+ log.warning(u'WARNING: bad PTS/DTS, please contact us')
return packet_length, None
# timestamp = self.ReadPTS(buffer[9:14])
@@ -595,8 +593,8 @@ class MPEG(core.AVContainer):
def isPES(self, file):
- log.info('trying mpeg-pes scan')
- file.seek(0,0)
+ log.info(u'trying mpeg-pes scan')
+ file.seek(0, 0)
buffer = file.read(3)
# header (also valid for all mpegs)
@@ -613,7 +611,7 @@ class MPEG(core.AVContainer):
return 0
if timestamp != None and not hasattr(self, 'start'):
self.get_time = self.ReadPTS
- bpos = buffer[offset+timestamp:offset+timestamp+5]
+ bpos = buffer[offset + timestamp:offset + timestamp + 5]
self.start = self.get_time(bpos)
if self.sequence_header_offset and hasattr(self, 'start'):
# we have all informations we need
@@ -702,13 +700,13 @@ class MPEG(core.AVContainer):
# Transport Stream ===============================================
def isTS(self, file):
- file.seek(0,0)
+ file.seek(0, 0)
buffer = file.read(TS_PACKET_LENGTH * 2)
c = 0
while c + TS_PACKET_LENGTH < len(buffer):
- if ord(buffer[c]) == ord(buffer[c+TS_PACKET_LENGTH]) == TS_SYNC:
+ if ord(buffer[c]) == ord(buffer[c + TS_PACKET_LENGTH]) == TS_SYNC:
break
c += 1
else:
@@ -718,7 +716,7 @@ class MPEG(core.AVContainer):
self.type = 'MPEG-TS'
while c + TS_PACKET_LENGTH < len(buffer):
- start = ord(buffer[c+1]) & 0x40
+ start = ord(buffer[c + 1]) & 0x40
# maybe load more into the buffer
if c + 2 * TS_PACKET_LENGTH > len(buffer) and c < 500000:
buffer += file.read(10000)
@@ -728,30 +726,30 @@ class MPEG(core.AVContainer):
c += TS_PACKET_LENGTH
continue
- tsid = ((ord(buffer[c+1]) & 0x3F) << 8) + ord(buffer[c+2])
- adapt = (ord(buffer[c+3]) & 0x30) >> 4
+ tsid = ((ord(buffer[c + 1]) & 0x3F) << 8) + ord(buffer[c + 2])
+ adapt = (ord(buffer[c + 3]) & 0x30) >> 4
offset = 4
if adapt & 0x02:
# meta info present, skip it for now
- adapt_len = ord(buffer[c+offset])
+ adapt_len = ord(buffer[c + offset])
offset += adapt_len + 1
- if not ord(buffer[c+1]) & 0x40:
+ if not ord(buffer[c + 1]) & 0x40:
# no new pes or psi in stream payload starting
pass
elif adapt & 0x01:
# PES
- timestamp = self.ReadPESHeader(c+offset, buffer[c+offset:],
+ timestamp = self.ReadPESHeader(c + offset, buffer[c + offset:],
tsid)[1]
if timestamp != None:
if not hasattr(self, 'start'):
self.get_time = self.ReadPTS
timestamp = c + offset + timestamp
- self.start = self.get_time(buffer[timestamp:timestamp+5])
+ self.start = self.get_time(buffer[timestamp:timestamp + 5])
elif not hasattr(self, 'audio_ok'):
timestamp = c + offset + timestamp
- start = self.get_time(buffer[timestamp:timestamp+5])
+ start = self.get_time(buffer[timestamp:timestamp + 5])
if start is not None and self.start is not None and \
abs(start - self.start) < 10:
# looks ok
@@ -759,7 +757,7 @@ class MPEG(core.AVContainer):
else:
# timestamp broken
del self.start
- log.warning('Timestamp error, correcting')
+ log.warning(u'Timestamp error, correcting')
if hasattr(self, 'start') and self.start and \
self.sequence_header_offset and self.video and self.audio:
@@ -786,31 +784,31 @@ class MPEG(core.AVContainer):
c = 0
while c + TS_PACKET_LENGTH < len(buffer):
- if ord(buffer[c]) == ord(buffer[c+TS_PACKET_LENGTH]) == TS_SYNC:
+ if ord(buffer[c]) == ord(buffer[c + TS_PACKET_LENGTH]) == TS_SYNC:
break
c += 1
else:
return None
while c + TS_PACKET_LENGTH < len(buffer):
- start = ord(buffer[c+1]) & 0x40
+ start = ord(buffer[c + 1]) & 0x40
if not start:
c += TS_PACKET_LENGTH
continue
- tsid = ((ord(buffer[c+1]) & 0x3F) << 8) + ord(buffer[c+2])
- adapt = (ord(buffer[c+3]) & 0x30) >> 4
+ tsid = ((ord(buffer[c + 1]) & 0x3F) << 8) + ord(buffer[c + 2])
+ adapt = (ord(buffer[c + 3]) & 0x30) >> 4
offset = 4
if adapt & 0x02:
# meta info present, skip it for now
- offset += ord(buffer[c+offset]) + 1
+ offset += ord(buffer[c + offset]) + 1
if adapt & 0x01:
- timestamp = self.ReadPESHeader(c+offset, buffer[c+offset:], tsid)[1]
+ timestamp = self.ReadPESHeader(c + offset, buffer[c + offset:], tsid)[1]
if timestamp is None:
# this should not happen
- log.error('bad TS')
+ log.error(u'bad TS')
return None
return c + offset + timestamp
c += TS_PACKET_LENGTH
@@ -841,7 +839,7 @@ class MPEG(core.AVContainer):
if pos == None:
break
end = self.get_time(buffer[pos:]) or end
- buffer = buffer[pos+100:]
+ buffer = buffer[pos + 100:]
file.close()
return end
@@ -855,7 +853,7 @@ class MPEG(core.AVContainer):
if end == None or self.start == None:
return None
if self.start > end:
- return int(((long(1) << 33) - 1 ) / 90000) - self.start + end
+ return int(((long(1) << 33) - 1) / 90000) - self.start + end
return end - self.start
@@ -897,7 +895,7 @@ class MPEG(core.AVContainer):
return 0
file = open(self.filename)
- log.debug('scanning file...')
+ log.debug(u'scanning file...')
while 1:
file.seek(self.__seek_size__ * 10, 1)
buffer = file.read(self.__sample_size__)
@@ -906,10 +904,10 @@ class MPEG(core.AVContainer):
pos = self.__search__(buffer)
if pos == None:
continue
- log.debug('buffer position: %s' % self.get_time(buffer[pos:]))
+ log.debug(u'buffer position: %r' % self.get_time(buffer[pos:]))
file.close()
- log.debug('done scanning file')
+ log.debug(u'done scanning file')
Parser = MPEG
diff --git a/libs/enzyme/ogm.py b/libs/enzyme/ogm.py
index 6857cb66..4198be24 100644
--- a/libs/enzyme/ogm.py
+++ b/libs/enzyme/ogm.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2003-2006 Thomas Schueppel
-# Copyright (C) 2003-2006 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Thomas Schueppel
+# Copyright 2003-2006 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -17,9 +17,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
+# along with enzyme. If not, see .
__all__ = ['Parser']
import struct
@@ -27,7 +25,7 @@ import re
import stat
import os
import logging
-from exceptions import *
+from exceptions import ParseError
import core
# get logging object
@@ -82,7 +80,7 @@ class Ogm(core.AVContainer):
# seek to the end of the stream, to avoid scanning the whole file
if (os.stat(file.name)[stat.ST_SIZE] > 50000):
- file.seek(os.stat(file.name)[stat.ST_SIZE]-49000)
+ file.seek(os.stat(file.name)[stat.ST_SIZE] - 49000)
# read the rest of the file into a buffer
h = file.read()
@@ -147,21 +145,21 @@ class Ogm(core.AVContainer):
self._appendtable('VORBISCOMMENT', header)
- def _parseOGGS(self,file):
+ def _parseOGGS(self, file):
h = file.read(27)
if len(h) == 0:
# Regular File end
return None, None
elif len(h) < 27:
- log.debug("%d Bytes of Garbage found after End." % len(h))
+ log.debug(u'%d Bytes of Garbage found after End.' % len(h))
return None, None
if h[:4] != "OggS":
- log.debug("Invalid Ogg")
+ log.debug(u'Invalid Ogg')
raise ParseError()
version = ord(h[4])
if version != 0:
- log.debug("Unsupported OGG/OGM Version %d." % version)
+ log.debug(u'Unsupported OGG/OGM Version %d' % version)
return None, None
head = struct.unpack(' serial:
stream = self.all_streams[serial]
if hasattr(stream, 'samplerate') and \
@@ -197,27 +195,27 @@ class Ogm(core.AVContainer):
return granulepos, nextlen + 27 + pageSegCount
- def _parseMeta(self,h):
+ def _parseMeta(self, h):
flags = ord(h[0])
headerlen = len(h)
if headerlen >= 7 and h[1:7] == 'vorbis':
header = {}
nextlen, self.encoder = self._extractHeaderString(h[7:])
- numItems = struct.unpack('= struct.calcsize(STREAM_HEADER_VIDEO)+1:
+ headerlen >= struct.calcsize(STREAM_HEADER_VIDEO) + 1:
# New Directshow Format
htype = header[1:9]
if htype[:5] == 'video':
- sh = header[9:struct.calcsize(STREAM_HEADER_VIDEO)+9]
+ sh = header[9:struct.calcsize(STREAM_HEADER_VIDEO) + 9]
streamheader = struct.unpack(STREAM_HEADER_VIDEO, sh)
vi = core.VideoStream()
(type, ssize, timeunit, samplerate, vi.length, buffersize, \
@@ -270,13 +268,13 @@ class Ogm(core.AVContainer):
self.all_streams.append(vi)
elif htype[:5] == 'audio':
- sha = header[9:struct.calcsize(STREAM_HEADER_AUDIO)+9]
+ sha = header[9:struct.calcsize(STREAM_HEADER_AUDIO) + 9]
streamheader = struct.unpack(STREAM_HEADER_AUDIO, sha)
ai = core.AudioStream()
(type, ssize, timeunit, ai.samplerate, ai.length, buffersize, \
ai.bitrate, ai.channels, bloc, ai.bitrate) = streamheader
self.samplerate = ai.samplerate
- log.debug("Samplerate %d" % self.samplerate)
+ log.debug(u'Samplerate %d' % self.samplerate)
self.audio.append(ai)
self.all_streams.append(ai)
@@ -287,15 +285,15 @@ class Ogm(core.AVContainer):
self.all_streams.append(subtitle)
else:
- log.debug("Unknown Header")
+ log.debug(u'Unknown Header')
- def _extractHeaderString(self,header):
+ def _extractHeaderString(self, header):
len = struct.unpack('
-# Copyright (C) 2003-2006 Thomas Schueppel
-# Copyright (C) 2003-2006 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Thomas Schueppel
+# Copyright 2003-2006 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -17,14 +17,12 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
+# along with enzyme. If not, see .
__all__ = ['Parser']
import struct
import logging
-from exceptions import *
+from exceptions import ParseError
import core
# http://www.pcisys.net/~melanson/codecs/rmff.htm
@@ -34,13 +32,13 @@ import core
log = logging.getLogger(__name__)
class RealVideo(core.AVContainer):
- def __init__(self,file):
+ def __init__(self, file):
core.AVContainer.__init__(self)
self.mime = 'video/real'
self.type = 'Real Video'
h = file.read(10)
try:
- (object_id,object_size,object_version) = struct.unpack('>4sIH',h)
+ (object_id, object_size, object_version) = struct.unpack('>4sIH', h)
except struct.error:
# EOF.
raise ParseError()
@@ -49,47 +47,47 @@ class RealVideo(core.AVContainer):
raise ParseError()
file_version, num_headers = struct.unpack('>II', file.read(8))
- log.debug("size: %d, ver: %d, headers: %d" % \
- (object_size, file_version,num_headers))
- for i in range(0,num_headers):
+ log.debug(u'size: %d, ver: %d, headers: %d' % \
+ (object_size, file_version, num_headers))
+ for _ in range(0, num_headers):
try:
- oi = struct.unpack('>4sIH',file.read(10))
+ oi = struct.unpack('>4sIH', file.read(10))
except (struct.error, IOError):
# Header data we expected wasn't there. File may be
# only partially complete.
break
if object_id == 'DATA' and oi[0] != 'INDX':
- log.debug('INDX chunk expected after DATA but not found -- file corrupt')
+ log.debug(u'INDX chunk expected after DATA but not found -- file corrupt')
break
- (object_id,object_size,object_version) = oi
+ (object_id, object_size, object_version) = oi
if object_id == 'DATA':
# Seek over the data chunk rather than reading it in.
file.seek(object_size - 10, 1)
else:
- self._read_header(object_id, file.read(object_size-10))
- log.debug("%s [%d]" % (object_id,object_size-10))
+ self._read_header(object_id, file.read(object_size - 10))
+ log.debug(u'%r [%d]' % (object_id, object_size - 10))
# Read all the following headers
- def _read_header(self,object_id,s):
+ def _read_header(self, object_id, s):
if object_id == 'PROP':
prop = struct.unpack('>9IHH', s)
- log.debug(prop)
+ log.debug(u'PROP: %r' % prop)
if object_id == 'MDPR':
mdpr = struct.unpack('>H7I', s[:30])
- log.debug(mdpr)
- self.length = mdpr[7]/1000.0
+ log.debug(u'MDPR: %r' % mdpr)
+ self.length = mdpr[7] / 1000.0
(stream_name_size,) = struct.unpack('>B', s[30:31])
- stream_name = s[31:31+stream_name_size]
- pos = 31+stream_name_size
- (mime_type_size,) = struct.unpack('>B', s[pos:pos+1])
- mime = s[pos+1:pos+1+mime_type_size]
- pos += mime_type_size+1
- (type_specific_len,) = struct.unpack('>I', s[pos:pos+4])
- type_specific = s[pos+4:pos+4+type_specific_len]
- pos += 4+type_specific_len
+ stream_name = s[31:31 + stream_name_size]
+ pos = 31 + stream_name_size
+ (mime_type_size,) = struct.unpack('>B', s[pos:pos + 1])
+ mime = s[pos + 1:pos + 1 + mime_type_size]
+ pos += mime_type_size + 1
+ (type_specific_len,) = struct.unpack('>I', s[pos:pos + 4])
+ type_specific = s[pos + 4:pos + 4 + type_specific_len]
+ pos += 4 + type_specific_len
if mime[:5] == 'audio':
ai = core.AudioStream()
ai.id = mdpr[0]
@@ -101,20 +99,20 @@ class RealVideo(core.AVContainer):
vi.bitrate = mdpr[2]
self.video.append(vi)
else:
- log.debug("Unknown: %s" % mime)
+ log.debug(u'Unknown: %r' % mime)
if object_id == 'CONT':
pos = 0
- (title_len,) = struct.unpack('>H', s[pos:pos+2])
- self.title = s[2:title_len+2]
- pos += title_len+2
- (author_len,) = struct.unpack('>H', s[pos:pos+2])
- self.artist = s[pos+2:pos+author_len+2]
- pos += author_len+2
- (copyright_len,) = struct.unpack('>H', s[pos:pos+2])
- self.copyright = s[pos+2:pos+copyright_len+2]
- pos += copyright_len+2
- (comment_len,) = struct.unpack('>H', s[pos:pos+2])
- self.comment = s[pos+2:pos+comment_len+2]
+ (title_len,) = struct.unpack('>H', s[pos:pos + 2])
+ self.title = s[2:title_len + 2]
+ pos += title_len + 2
+ (author_len,) = struct.unpack('>H', s[pos:pos + 2])
+ self.artist = s[pos + 2:pos + author_len + 2]
+ pos += author_len + 2
+ (copyright_len,) = struct.unpack('>H', s[pos:pos + 2])
+ self.copyright = s[pos + 2:pos + copyright_len + 2]
+ pos += copyright_len + 2
+ (comment_len,) = struct.unpack('>H', s[pos:pos + 2])
+ self.comment = s[pos + 2:pos + comment_len + 2]
Parser = RealVideo
diff --git a/libs/enzyme/riff.py b/libs/enzyme/riff.py
index 221a5fc7..516c727b 100644
--- a/libs/enzyme/riff.py
+++ b/libs/enzyme/riff.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2003-2006 Thomas Schueppel
-# Copyright (C) 2003-2006 Dirk Meyer
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2003-2006 Thomas Schueppel
+# Copyright 2003-2006 Dirk Meyer
#
# This file is part of enzyme.
#
@@ -17,9 +17,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
+# along with enzyme. If not, see .
__all__ = ['Parser']
import os
@@ -27,7 +25,7 @@ import struct
import string
import logging
import time
-from exceptions import *
+from exceptions import ParseError
import core
# get logging object
@@ -70,7 +68,7 @@ class Riff(core.AVContainer):
"""
table_mapping = { 'AVIINFO' : AVIINFO }
- def __init__(self,file):
+ def __init__(self, file):
core.AVContainer.__init__(self)
# read the header
h = file.read(12)
@@ -90,12 +88,12 @@ class Riff(core.AVContainer):
while self._parseRIFFChunk(file):
pass
except IOError:
- log.exception('error in file, stop parsing')
+ log.exception(u'error in file, stop parsing')
self._find_subtitles(file.name)
if not self.has_idx and isinstance(self, core.AVContainer):
- log.debug('WARNING: avi has no index')
+ log.debug(u'WARNING: avi has no index')
self._set('corrupt', True)
@@ -104,9 +102,9 @@ class Riff(core.AVContainer):
Search for subtitle files. Right now only VobSub is supported
"""
base = os.path.splitext(filename)[0]
- if os.path.isfile(base+'.idx') and \
- (os.path.isfile(base+'.sub') or os.path.isfile(base+'.rar')):
- file = open(base+'.idx')
+ if os.path.isfile(base + '.idx') and \
+ (os.path.isfile(base + '.sub') or os.path.isfile(base + '.rar')):
+ file = open(base + '.idx')
if file.readline().find('VobSub index file') > 0:
for line in file.readlines():
if line.find('id') == 0:
@@ -117,10 +115,10 @@ class Riff(core.AVContainer):
file.close()
- def _parseAVIH(self,t):
+ def _parseAVIH(self, t):
retval = {}
- v = struct.unpack(' 1024*500:
+ if key[2:] != 'dc' or sz > 1024 * 500:
# This chunk is not video or is unusually big (> 500KB);
# skip it.
file.seek(sz, 1)
@@ -338,7 +336,7 @@ class Riff(core.AVContainer):
startcode = 0xff
def bits(v, o, n):
# Returns n bits in v, offset o bits.
- return (v & 2**n-1 << (64-n-o)) >> 64-n-o
+ return (v & 2 ** n - 1 << (64 - n - o)) >> 64 - n - o
while pos < sz:
startcode = ((startcode << 8) | ord(data[pos])) & 0xffffffff
@@ -350,7 +348,7 @@ class Riff(core.AVContainer):
if startcode >= 0x120 and startcode <= 0x12F:
# We have the VOL startcode. Pull 64 bits of it and treat
# as a bitstream
- v = struct.unpack(">Q", data[pos : pos+8])[0]
+ v = struct.unpack(">Q", data[pos : pos + 8])[0]
offset = 10
if bits(v, 9, 1):
# is_ol_id, skip over vo_ver_id and vo_priority
@@ -385,36 +383,36 @@ class Riff(core.AVContainer):
if i < size:
# Seek past whatever might be remaining of the movi list.
- file.seek(size-i,1)
+ file.seek(size - i, 1)
- def _parseLIST(self,t):
+ def _parseLIST(self, t):
retval = {}
i = 0
size = len(t)
- while i < size-8:
+ while i < size - 8:
# skip zero
if ord(t[i]) == 0: i += 1
- key = t[i:i+4]
+ key = t[i:i + 4]
sz = 0
if key == 'LIST':
- sz = struct.unpack(' 0:
@@ -466,17 +464,17 @@ class Riff(core.AVContainer):
except ValueError:
pass
else:
- log.debug('no support for time format %s', value)
- i+=sz
+ log.debug(u'no support for time format %r', value)
+ i += sz
return retval
- def _parseRIFFChunk(self,file):
+ def _parseRIFFChunk(self, file):
h = file.read(8)
if len(h) < 8:
return False
name = h[:4]
- size = struct.unpack(' 80000:
- log.debug('RIFF LIST "%s" too long to parse: %s bytes' % (key, size))
- t = file.seek(size-4,1)
+ log.debug(u'RIFF LIST %r too long to parse: %r bytes' % (key, size))
+ t = file.seek(size - 4, 1)
return True
elif size < 5:
- log.debug('RIFF LIST "%s" too short: %s bytes' % (key, size))
+ log.debug(u'RIFF LIST %r too short: %r bytes' % (key, size))
return True
- t = file.read(size-4)
- log.debug('parse RIFF LIST "%s": %d bytes' % (key, size))
+ t = file.read(size - 4)
+ log.debug(u'parse RIFF LIST %r: %d bytes' % (key, size))
value = self._parseLIST(t)
self.header[key] = value
if key == 'INFO':
@@ -511,7 +509,7 @@ class Riff(core.AVContainer):
# no need to add this info to a table
pass
else:
- log.debug('Skipping table info %s' % key)
+ log.debug(u'Skipping table info %r' % key)
elif name == 'JUNK':
self.junkStart = file.tell() - 8
@@ -519,15 +517,15 @@ class Riff(core.AVContainer):
file.seek(size, 1)
elif name == 'idx1':
self.has_idx = True
- log.debug('idx1: %s bytes' % size)
+ log.debug(u'idx1: %r bytes' % size)
# no need to parse this
- t = file.seek(size,1)
+ t = file.seek(size, 1)
elif name == 'RIFF':
- log.debug("New RIFF chunk, extended avi [%i]" % size)
+ log.debug(u'New RIFF chunk, extended avi [%i]' % size)
type = file.read(4)
if type != 'AVIX':
- log.debug("Second RIFF chunk is %s, not AVIX, skipping", type)
- file.seek(size-4, 1)
+ log.debug(u'Second RIFF chunk is %r, not AVIX, skipping', type)
+ file.seek(size - 4, 1)
# that's it, no new informations should be in AVIX
return False
elif name == 'fmt ' and size <= 50:
@@ -556,11 +554,11 @@ class Riff(core.AVContainer):
elif not name.strip(string.printable + string.whitespace):
# check if name is something usefull at all, maybe it is no
# avi or broken
- t = file.seek(size,1)
- log.debug("Skipping %s [%i]" % (name,size))
+ t = file.seek(size, 1)
+ log.debug(u'Skipping %r [%i]' % (name, size))
else:
# bad avi
- log.debug("Bad or broken avi")
+ log.debug(u'Bad or broken avi')
return False
return True
diff --git a/libs/enzyme/strutils.py b/libs/enzyme/strutils.py
index b1df1d91..8578aefa 100644
--- a/libs/enzyme/strutils.py
+++ b/libs/enzyme/strutils.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# enzyme - Video metadata parser
-# Copyright (C) 2011 Antoine Bertin
-# Copyright (C) 2006-2009 Dirk Meyer
-# Copyright (C) 2006-2009 Jason Tackaberry
+# Copyright 2011-2012 Antoine Bertin
+# Copyright 2006-2009 Dirk Meyer
+# Copyright 2006-2009 Jason Tackaberry
#
# This file is part of enzyme.
#
@@ -17,9 +17,7 @@
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
-#
-
+# along with enzyme. If not, see .
__all__ = ['ENCODING', 'str_to_unicode', 'unicode_to_str']
import locale
diff --git a/libs/guessit/__init__.py b/libs/guessit/__init__.py
index cb62db02..a86f71bb 100644
--- a/libs/guessit/__init__.py
+++ b/libs/guessit/__init__.py
@@ -18,10 +18,10 @@
# along with this program. If not, see .
#
-__version__ = '0.3-dev'
-__all__ = [ 'Guess', 'Language',
- 'guess_file_info', 'guess_video_info',
- 'guess_movie_info', 'guess_episode_info' ]
+__version__ = '0.3.1'
+__all__ = ['Guess', 'Language',
+ 'guess_file_info', 'guess_video_info',
+ 'guess_movie_info', 'guess_episode_info']
from guessit.guess import Guess, merge_all
@@ -31,6 +31,7 @@ import logging
log = logging.getLogger("guessit")
+
class NullHandler(logging.Handler):
def emit(self, record):
pass
@@ -40,9 +41,7 @@ h = NullHandler()
log.addHandler(h)
-
-
-def guess_file_info(filename, filetype, info = [ 'filename' ]):
+def guess_file_info(filename, filetype, info=None):
"""info can contain the names of the various plugins, such as 'filename' to
detect filename info, or 'hash_md5' to get the md5 hash of the file.
@@ -52,27 +51,30 @@ def guess_file_info(filename, filetype, info = [ 'filename' ]):
result = []
hashers = []
+ if info is None:
+ info = ['filename']
+
if isinstance(info, basestring):
- info = [ info ]
+ info = [info]
for infotype in info:
if infotype == 'filename':
- m = IterativeMatcher(filename, filetype = filetype)
+ m = IterativeMatcher(filename, filetype=filetype)
result.append(m.matched())
elif infotype == 'hash_mpc':
- import hash_mpc
+ from guessit.hash_mpc import hash_file
try:
- result.append(Guess({ 'hash_mpc': hash_mpc.hash_file(filename) },
- confidence = 1.0))
+ result.append(Guess({'hash_mpc': hash_file(filename)},
+ confidence=1.0))
except Exception, e:
log.warning('Could not compute MPC-style hash because: %s' % e)
elif infotype == 'hash_ed2k':
- import hash_ed2k
+ from guessit.hash_ed2k import hash_file
try:
- result.append(Guess({ 'hash_ed2k': hash_ed2k.hash_file(filename) },
- confidence = 1.0))
+ result.append(Guess({'hash_ed2k': hash_file(filename)},
+ confidence=1.0))
except Exception, e:
log.warning('Could not compute ed2k hash because: %s' % e)
@@ -88,18 +90,6 @@ def guess_file_info(filename, filetype, info = [ 'filename' ]):
else:
log.warning('Invalid infotype: %s' % infotype)
-
- """For plugins which depend on some optional library, import them like that:
-
- if infotype == 'plugin_name':
- try:
- import optional_lib
- except ImportError:
- raise Exception, 'The plugin module cannot be loaded because the optional_lib lib is missing'
-
- # do some stuff
- """
-
# do all the hashes now, but on a single pass
if hashers:
try:
@@ -112,22 +102,21 @@ def guess_file_info(filename, filetype, info = [ 'filename' ]):
hasher.update(chunk)
for infotype, hasher in hashers:
- result.append(Guess({ infotype: hasher.hexdigest() },
- confidence = 1.0))
+ result.append(Guess({infotype: hasher.hexdigest()},
+ confidence=1.0))
except Exception, e:
log.warning('Could not compute hash because: %s' % e)
-
return merge_all(result)
-def guess_video_info(filename, info = [ 'filename' ]):
+def guess_video_info(filename, info=None):
return guess_file_info(filename, 'autodetect', info)
-def guess_movie_info(filename, info = [ 'filename' ]):
+
+def guess_movie_info(filename, info=None):
return guess_file_info(filename, 'movie', info)
-def guess_episode_info(filename, info = [ 'filename' ]):
+
+def guess_episode_info(filename, info=None):
return guess_file_info(filename, 'episode', info)
-
-
diff --git a/libs/guessit/autodetect.py b/libs/guessit/autodetect.py
deleted file mode 100644
index fe05b68b..00000000
--- a/libs/guessit/autodetect.py
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack
-#
-# GuessIt is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# GuessIt is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
-#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
-#
-
-#from guessit import movie, episode
-import os, os.path
-import logging
-
-log = logging.getLogger('guessit.autodetect')
-
-def within(x, nrange):
- """Return whether a number is inside a given range, specified as a list or tuple
- of the lower and upper bounds."""
- low, high = nrange
- return low <= x <= high
-
-def guess_filename_info(filename):
- log.debug('Trying to guess info for file: ' + filename)
-
- # try to guess info as if it were an episode
- episode_info = episode.guess_episode_filename(filename)
-
- # 1- if we found either season/episodeNumber, then we're pretty sure it must
- # be an episode
- if 'season' in episode_info or 'episodeNumber' in episode_info:
- log.debug('Likely an episode as it contains season and/or episodeNumber: ' + filename)
- episode_info.update({ 'type': 'episode' }, confidence = 0.9)
- return episode_info
-
- # try to guess info as if it were a movie
- movie_info = movie.guess_movie_filename(filename)
-
- # 2- if the file exists, try to guess its type using its size
- if os.path.exists(filename):
- size = os.stat(filename).st_size / (1024 * 1024)
-
- # if size <= 1/2 of 1CD -> episode (very unlikely a movie so small)
- if size < 400:
- log.debug('Likely an episode due to its small size (%dMB): %s' % (size, filename))
- episode_info.update({ 'type': 'episode' }, confidence = 0.8)
- return episode_info
-
- # if size > 2G -> movie (even fullHD eps aren't that big yet)
- if size > 2048:
- log.debug('Likely a movie due to its big size (%dMB): %s' % (size, filename))
- movie_info.update({ 'type': 'movie' }, confidence = 0.8)
- return movie_info
-
- # if size == 1CD or 2CDs -> movie
- if within(size, [690, 710]) or within(size, [1380, 1420]):
- log.debug('Likely a movie due to its size close to a CD size (%dMB): %s' % (size, filename))
- movie_info.update({ 'type': 'movie' }, confidence = 0.8)
- return movie_info
-
-
- # 3- if all else fails, assume it's a movie
- log.debug('Couldn\'t make an informed guess... Assuming file is a movie: %s' % filename)
- movie_info.update({ 'type': 'movie' }, confidence = 0.5)
- return movie_info
diff --git a/libs/guessit/date.py b/libs/guessit/date.py
index 9f79daec..4d0b510e 100644
--- a/libs/guessit/date.py
+++ b/libs/guessit/date.py
@@ -21,6 +21,7 @@
import datetime
import re
+
def search_year(string):
"""Looks for year patterns, and if found return the year and group span.
Assumes there are sentinels at the beginning and end of the string that
@@ -62,34 +63,35 @@ def search_date(string):
dsep = r'[-/ \.]'
- date_rexps = [ # 20010823
- r'[^0-9]' +
- r'(?P[0-9]{4})' +
- r'(?P[0-9]{2})' +
- r'(?P[0-9]{2})' +
- r'[^0-9]',
+ date_rexps = [
+ # 20010823
+ r'[^0-9]' +
+ r'(?P[0-9]{4})' +
+ r'(?P[0-9]{2})' +
+ r'(?P[0-9]{2})' +
+ r'[^0-9]',
- # 2001-08-23
- r'[^0-9]' +
- r'(?P[0-9]{4})' + dsep +
- r'(?P[0-9]{2})' + dsep +
- r'(?P[0-9]{2})' +
- r'[^0-9]',
+ # 2001-08-23
+ r'[^0-9]' +
+ r'(?P[0-9]{4})' + dsep +
+ r'(?P[0-9]{2})' + dsep +
+ r'(?P[0-9]{2})' +
+ r'[^0-9]',
- # 23-08-2001
- r'[^0-9]' +
- r'(?P[0-9]{2})' + dsep +
- r'(?P[0-9]{2})' + dsep +
- r'(?P[0-9]{4})' +
- r'[^0-9]',
+ # 23-08-2001
+ r'[^0-9]' +
+ r'(?P[0-9]{2})' + dsep +
+ r'(?P[0-9]{2})' + dsep +
+ r'(?P[0-9]{4})' +
+ r'[^0-9]',
- # 23-08-01
- r'[^0-9]' +
- r'(?P[0-9]{2})' + dsep +
- r'(?P[0-9]{2})' + dsep +
- r'(?P[0-9]{2})' +
- r'[^0-9]',
- ]
+ # 23-08-01
+ r'[^0-9]' +
+ r'(?P[0-9]{2})' + dsep +
+ r'(?P[0-9]{2})' + dsep +
+ r'(?P[0-9]{2})' +
+ r'[^0-9]',
+ ]
for drexp in date_rexps:
match = re.search(drexp, string)
@@ -98,7 +100,7 @@ def search_date(string):
year, month, day = int(d['year']), int(d['month']), int(d['day'])
# years specified as 2 digits should be adjusted here
if year < 100:
- if year > (datetime.date.today().year % 100)+ 5:
+ if year > (datetime.date.today().year % 100) + 5:
year = 1900 + year
else:
year = 2000 + year
@@ -120,8 +122,9 @@ def search_date(string):
continue
# looks like we have a valid date
- # note: span is [+1,-1] because we don't want to include the non-digit char
+ # note: span is [+1,-1] because we don't want to include the
+ # non-digit char
start, end = match.span()
- return (date, (start+1, end-1))
+ return (date, (start + 1, end - 1))
return None, None
diff --git a/libs/guessit/filetype.py b/libs/guessit/filetype.py
deleted file mode 100644
index fdc9d695..00000000
--- a/libs/guessit/filetype.py
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack
-#
-# GuessIt is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# GuessIt is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
-#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
-#
-
-from guessit.patterns import subtitle_exts, video_exts, episode_rexps, find_properties, canonical_form
-import os.path
-import re
-import logging
-
-log = logging.getLogger("guessit.filetype")
-
-
-def guess_filetype(filename, filetype = 'autodetect'):
- other = {}
-
- # look at the extension first
- fileext = os.path.splitext(filename)[1][1:].lower()
- if fileext in subtitle_exts:
- if 'movie' in filetype:
- filetype = 'moviesubtitle'
- elif 'episode' in filetype:
- filetype = 'episodesubtitle'
- else:
- filetype = 'subtitle'
- other = { 'container': fileext }
- elif fileext in video_exts:
- if filetype == 'autodetect':
- filetype = 'video'
- other = { 'container': fileext }
- else:
- if filetype == 'autodetect':
- filetype = 'unknown'
- other = { 'extension': fileext }
-
- # now look whether there are some specific hints for episode vs movie
- if filetype in ('video', 'subtitle'):
- for rexp, confidence, span_adjust in episode_rexps:
- match = re.search(rexp, filename, re.IGNORECASE)
- if match:
- if filetype == 'video':
- filetype = 'episode'
- elif filetype == 'subtitle':
- filetype = 'episodesubtitle'
- break
-
- for prop, value, start, end in find_properties(filename):
- if canonical_form(value) == 'DVB':
- if filetype == 'video':
- filetype = 'episode'
- elif filetype == 'subtitle':
- filetype = 'episodesubtitle'
- break
-
- # if no episode info found, assume it's a movie
- if filetype == 'video':
- filetype = 'movie'
- elif filetype == 'subtitle':
- filetype = 'moviesubtitle'
-
- return filetype, other
diff --git a/libs/guessit/fileutils.py b/libs/guessit/fileutils.py
index ceefee1e..13d12c51 100644
--- a/libs/guessit/fileutils.py
+++ b/libs/guessit/fileutils.py
@@ -50,11 +50,11 @@ def split_path(path):
# on Unix systems, the root folder is '/'
if head == '/' and tail == '':
- return [ '/' ] + result
+ return ['/'] + result
# on Windows, the root folder is a drive letter (eg: 'C:\')
if len(head) == 3 and head[1:] == ':\\' and tail == '':
- return [ head ] + result
+ return [head] + result
if head == '' and tail == '':
return result
@@ -64,17 +64,10 @@ def split_path(path):
path = head
continue
- result = [ tail ] + result
+ result = [tail] + result
path = head
-def split_path_components(filename):
- """Returns the filename split into [ dir*, basename, ext ]."""
- result = split_path(filename)
- basename = result.pop(-1)
- return result + list(os.path.splitext(basename))
-
-
def file_in_same_dir(ref_file, desired_file):
"""Return the path for a file in the same dir as a given reference file.
@@ -82,17 +75,17 @@ def file_in_same_dir(ref_file, desired_file):
'~/smewt/smewt.settings'
"""
- return os.path.join(*(split_path(ref_file)[:-1] + [ desired_file ]))
+ return os.path.join(*(split_path(ref_file)[:-1] + [desired_file]))
def load_file_in_same_dir(ref_file, filename):
"""Load a given file. Works even when the file is contained inside a zip."""
- path = split_path(ref_file)[:-1] + [ filename ]
+ path = split_path(ref_file)[:-1] + [filename]
for i, p in enumerate(path):
if p.endswith('.zip'):
- zfilename = os.path.join(*path[:i+1])
+ zfilename = os.path.join(*path[:i + 1])
zfile = zipfile.ZipFile(zfilename)
- return zfile.read('/'.join(path[i+1:]))
+ return zfile.read('/'.join(path[i + 1:]))
return open(os.path.join(*path)).read()
diff --git a/libs/guessit/guess.py b/libs/guessit/guess.py
index 92ec4bea..9950a12e 100644
--- a/libs/guessit/guess.py
+++ b/libs/guessit/guess.py
@@ -26,9 +26,12 @@ log = logging.getLogger("guessit.guess")
class Guess(dict):
- """A Guess is a dictionary which has an associated confidence for each of its values.
+ """A Guess is a dictionary which has an associated confidence for each of
+ its values.
+
+ As it is a subclass of dict, you can use it everywhere you expect a
+ simple dict."""
- As it is a subclass of dict, you can use it everywhere you expect a simple dict"""
def __init__(self, *args, **kwargs):
try:
confidence = kwargs.pop('confidence')
@@ -52,20 +55,20 @@ class Guess(dict):
elif isinstance(value, unicode):
data[prop] = value.encode('utf-8')
elif isinstance(value, list):
- data[prop] = [ str(x) for x in value ]
+ data[prop] = [str(x) for x in value]
return data
def nice_string(self):
data = self.to_utf8_dict()
- parts = json.dumps(data, indent = 4).split('\n')
+ parts = json.dumps(data, indent=4).split('\n')
for i, p in enumerate(parts):
if p[:5] != ' "':
continue
prop = p.split('"')[1]
- parts[i] = (' [%.2f] "' % (self._confidence.get(prop) or -1)) + p[5:]
+ parts[i] = (' [%.2f] "' % self.confidence(prop)) + p[5:]
return '\n'.join(parts)
@@ -73,9 +76,9 @@ class Guess(dict):
return str(self.to_utf8_dict())
def confidence(self, prop):
- return self._confidence[prop]
+ return self._confidence.get(prop, -1)
- def set(self, prop, value, confidence = None):
+ def set(self, prop, value, confidence=None):
self[prop] = value
if confidence is not None:
self._confidence[prop] = confidence
@@ -83,7 +86,7 @@ class Guess(dict):
def set_confidence(self, prop, value):
self._confidence[prop] = value
- def update(self, other, confidence = None):
+ def update(self, other, confidence=None):
dict.update(self, other)
if isinstance(other, Guess):
for prop in other:
@@ -94,36 +97,36 @@ class Guess(dict):
self._confidence[prop] = confidence
def update_highest_confidence(self, other):
- """Update this guess with the values from the given one. In case there is
- property present in both, only the one with the highest one is kept."""
+ """Update this guess with the values from the given one. In case
+ there is property present in both, only the one with the highest one
+ is kept."""
if not isinstance(other, Guess):
- raise ValueError, 'Can only call this function on Guess instances'
+ raise ValueError('Can only call this function on Guess instances')
for prop in other:
- if prop in self and self._confidence[prop] >= other._confidence[prop]:
+ if prop in self and self.confidence(prop) >= other.confidence(prop):
continue
self[prop] = other[prop]
- self._confidence[prop] = other._confidence[prop]
-
-
+ self._confidence[prop] = other.confidence(prop)
def choose_int(g1, g2):
- """Function used by merge_similar_guesses to choose between 2 possible properties
- when they are integers."""
+ """Function used by merge_similar_guesses to choose between 2 possible
+ properties when they are integers."""
v1, c1 = g1 # value, confidence
v2, c2 = g2
if (v1 == v2):
- return (v1, 1 - (1-c1)*(1-c2))
+ return (v1, 1 - (1 - c1) * (1 - c2))
else:
if c1 > c2:
return (v1, c1 - c2)
else:
return (v2, c2 - c1)
+
def choose_string(g1, g2):
- """Function used by merge_similar_guesses to choose between 2 possible properties
- when they are strings.
+ """Function used by merge_similar_guesses to choose between 2 possible
+ properties when they are strings.
If the 2 strings are similar, or one is contained in the other, the latter is returned
with an increased confidence.
@@ -142,7 +145,7 @@ def choose_string(g1, g2):
('Hello', 0.75)
>>> choose_string(('Hello', 0.4), ('Hello World', 0.4))
- ('Hello', 0.64000000000000001)
+ ('Hello', 0.64)
>>> choose_string(('simpsons', 0.5), ('The Simpsons', 0.5))
('The Simpsons', 0.75)
@@ -159,7 +162,7 @@ def choose_string(g1, g2):
v1, v2 = v1.strip(), v2.strip()
v1l, v2l = v1.lower(), v2.lower()
- combined_prob = 1 - (1-c1)*(1-c2)
+ combined_prob = 1 - (1 - c1) * (1 - c2)
if v1l == v2l:
return (v1, combined_prob)
@@ -191,26 +194,31 @@ def _merge_similar_guesses_nocheck(guesses, prop, choose):
This function assumes there are at least 2 valid guesses."""
- similar = [ guess for guess in guesses if prop in guess ]
+ similar = [guess for guess in guesses if prop in guess]
g1, g2 = similar[0], similar[1]
other_props = set(g1) & set(g2) - set([prop])
if other_props:
+ log.debug('guess 1: %s' % g1)
+ log.debug('guess 2: %s' % g2)
for prop in other_props:
if g1[prop] != g2[prop]:
- log.warning('both guesses to be merged have more than one different property in common, bailing out...')
+ log.warning('both guesses to be merged have more than one '
+ 'different property in common, bailing out...')
return
- # merge all props of s2 into s1, updating the confidence for the considered property
+ # merge all props of s2 into s1, updating the confidence for the
+ # considered property
v1, v2 = g1[prop], g2[prop]
c1, c2 = g1.confidence(prop), g2.confidence(prop)
new_value, new_confidence = choose((v1, c1), (v2, c2))
if new_confidence >= c1:
- log.debug("Updating matching property '%s' with confidence %.2f" % (prop, new_confidence))
+ msg = "Updating matching property '%s' with confidence %.2f"
else:
- log.debug("Updating non-matching property '%s' with confidence %.2f" % (prop, new_confidence))
+ msg = "Updating non-matching property '%s' with confidence %.2f"
+ log.debug(msg % (prop, new_confidence))
g2[prop] = new_value
g2.set_confidence(prop, new_confidence)
@@ -218,12 +226,13 @@ def _merge_similar_guesses_nocheck(guesses, prop, choose):
g1.update(g2)
guesses.remove(g2)
+
def merge_similar_guesses(guesses, prop, choose):
"""Take a list of guesses and merge those which have the same properties,
increasing or decreasing the confidence depending on whether their values
are similar."""
- similar = [ guess for guess in guesses if prop in guess ]
+ similar = [guess for guess in guesses if prop in guess]
if len(similar) < 2:
# nothing to merge
return
@@ -233,9 +242,13 @@ def merge_similar_guesses(guesses, prop, choose):
if len(similar) > 2:
log.debug('complex merge, trying our best...')
+ before = len(guesses)
_merge_similar_guesses_nocheck(guesses, prop, choose)
- merge_similar_guesses(guesses, prop, choose)
- return
+ after = len(guesses)
+ if after < before:
+ # recurse only when the previous call actually did something,
+ # otherwise we end up in an infinite loop
+ merge_similar_guesses(guesses, prop, choose)
def merge_append_guesses(guesses, prop):
@@ -245,14 +258,12 @@ def merge_append_guesses(guesses, prop):
DEPRECATED, remove with old guessers
"""
-
-
- similar = [ guess for guess in guesses if prop in guess ]
+ similar = [guess for guess in guesses if prop in guess]
if not similar:
return
merged = similar[0]
- merged[prop] = [ merged[prop] ]
+ merged[prop] = [merged[prop]]
# TODO: what to do with global confidence? mean of them all?
for m in similar[1:]:
@@ -261,17 +272,18 @@ def merge_append_guesses(guesses, prop):
merged[prop].append(m[prop])
else:
if prop2 in m:
- log.warning('overwriting property "%s" with value ' % (prop2, m[prop2]))
+ log.warning('overwriting property "%s" with value %s' % (prop2, m[prop2]))
merged[prop2] = m[prop2]
# TODO: confidence also
guesses.remove(m)
-def merge_all(guesses, append = []):
- """Merges all the guesses in a single result, removes very unlikely values, and returns it.
- You can specify a list of properties that should be appended into a list instead of being
- merged.
+def merge_all(guesses, append=None):
+ """Merge all the guesses in a single result, remove very unlikely values,
+ and return it.
+ You can specify a list of properties that should be appended into a list
+ instead of being merged.
>>> merge_all([ Guess({ 'season': 2 }, confidence = 0.6),
... Guess({ 'episodeNumber': 13 }, confidence = 0.8) ])
@@ -286,20 +298,24 @@ def merge_all(guesses, append = []):
return Guess()
result = guesses[0]
+ if append is None:
+ append = []
for g in guesses[1:]:
# first append our appendable properties
for prop in append:
if prop in g:
- result.set(prop, result.get(prop, []) + [ g[prop] ],
- # TODO: what to do with confidence here? maybe an arithmetic mean...
- confidence = g.confidence(prop))
+ result.set(prop, result.get(prop, []) + [g[prop]],
+ # TODO: what to do with confidence here? maybe an
+ # arithmetic mean...
+ confidence=g.confidence(prop))
del g[prop]
# then merge the remaining ones
- if set(result) & set(g):
- log.warning('duplicate properties %s in merged result...' % (set(result) & set(g)))
+ dups = set(result) & set(g)
+ if dups:
+ log.warning('duplicate properties %s in merged result...' % dups)
result.update_highest_confidence(g)
@@ -314,4 +330,3 @@ def merge_all(guesses, append = []):
result[prop] = list(set(result[prop]))
return result
-
diff --git a/libs/guessit/hash_ed2k.py b/libs/guessit/hash_ed2k.py
index eb1a4ea6..43303c58 100644
--- a/libs/guessit/hash_ed2k.py
+++ b/libs/guessit/hash_ed2k.py
@@ -18,8 +18,9 @@
# along with this program. If not, see .
#
-from guessit import Guess
-import hashlib, os.path
+import hashlib
+import os.path
+
def hash_file(filename):
"""Returns the ed2k hash of a given file.
@@ -31,6 +32,7 @@ def hash_file(filename):
os.path.getsize(filename),
hash_filehash(filename).upper())
+
def hash_filehash(filename):
"""Returns the ed2k hash of a given file.
@@ -42,8 +44,10 @@ def hash_filehash(filename):
def gen(f):
while True:
x = f.read(9728000)
- if x: yield x
- else: return
+ if x:
+ yield x
+ else:
+ return
def md4_hash(data):
m = md4()
@@ -55,4 +59,5 @@ def hash_filehash(filename):
hashes = [md4_hash(data).digest() for data in a]
if len(hashes) == 1:
return hashes[0].encode("hex")
- else: return md4_hash(reduce(lambda a,d: a + d, hashes, "")).hexd
+ else:
+ return md4_hash(reduce(lambda a, d: a + d, hashes, "")).hexd
diff --git a/libs/guessit/hash_mpc.py b/libs/guessit/hash_mpc.py
index 2023f3d5..7475940a 100644
--- a/libs/guessit/hash_mpc.py
+++ b/libs/guessit/hash_mpc.py
@@ -18,8 +18,9 @@
# along with this program. If not, see .
#
-from guessit import Guess
-import struct, os
+import struct
+import os
+
def hash_file(filename):
"""This function is taken from:
@@ -32,25 +33,24 @@ def hash_file(filename):
f = open(filename, "rb")
filesize = os.path.getsize(filename)
- hash = filesize
+ hash_value = filesize
if filesize < 65536 * 2:
- raise Exception, "SizeError: size is %d, should be > 132K..." % filesize
+ raise Exception("SizeError: size is %d, should be > 132K..." % filesize)
- for x in range(65536/bytesize):
- buffer = f.read(bytesize)
- (l_value,)= struct.unpack(longlongformat, buffer)
- hash += l_value
- hash = hash & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number
+ for x in range(65536 / bytesize):
+ buf = f.read(bytesize)
+ (l_value,) = struct.unpack(longlongformat, buf)
+ hash_value += l_value
+ hash_value = hash_value & 0xFFFFFFFFFFFFFFFF #to remain as 64bit number
-
- f.seek(max(0,filesize-65536),0)
- for x in range(65536/bytesize):
- buffer = f.read(bytesize)
- (l_value,)= struct.unpack(longlongformat, buffer)
- hash += l_value
- hash = hash & 0xFFFFFFFFFFFFFFFF
+ f.seek(max(0, filesize - 65536), 0)
+ for x in range(65536 / bytesize):
+ buf = f.read(bytesize)
+ (l_value,) = struct.unpack(longlongformat, buf)
+ hash_value += l_value
+ hash_value = hash_value & 0xFFFFFFFFFFFFFFFF
f.close()
- returnedhash = "%016x" % hash
- return returnedhash
+
+ return "%016x" % hash_value
diff --git a/libs/guessit/language.py b/libs/guessit/language.py
index 51cd5461..777d0e25 100644
--- a/libs/guessit/language.py
+++ b/libs/guessit/language.py
@@ -19,28 +19,28 @@
#
from guessit import fileutils
-import os.path
-import re
import logging
log = logging.getLogger('guessit.language')
-
-
# downloaded from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
#
# Description of the fields:
# "An alpha-3 (bibliographic) code, an alpha-3 (terminologic) code (when given),
# an alpha-2 code (when given), an English name, and a French name of a language
# are all separated by pipe (|) characters."
+_iso639_contents = fileutils.load_file_in_same_dir(__file__,
+ 'ISO-639-2_utf-8.txt')
language_matrix = [ l.strip().decode('utf-8').split('|')
- for l in fileutils.load_file_in_same_dir(__file__, 'ISO-639-2_utf-8.txt').split('\n') ]
+ for l in _iso639_contents.split('\n') ]
-lng3 = frozenset(filter(bool, (l[0] for l in language_matrix)))
-lng3term = frozenset(filter(bool, (l[1] for l in language_matrix)))
-lng2 = frozenset(filter(bool, (l[2] for l in language_matrix)))
-lng_en_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[3].lower().split('; '))))
-lng_fr_name = frozenset(filter(bool, (lng for l in language_matrix for lng in l[4].lower().split('; '))))
+lng3 = frozenset(l[0] for l in language_matrix if l[0])
+lng3term = frozenset(l[1] for l in language_matrix if l[1])
+lng2 = frozenset(l[2] for l in language_matrix if l[2])
+lng_en_name = frozenset(lng for l in language_matrix
+ for lng in l[3].lower().split('; ') if lng)
+lng_fr_name = frozenset(lng for l in language_matrix
+ for lng in l[4].lower().split('; ') if lng)
lng_all_names = lng3 | lng3term | lng2 | lng_en_name | lng_fr_name
lng3_to_lng3term = dict((l[0], l[1]) for l in language_matrix if l[1])
@@ -50,22 +50,29 @@ lng3_to_lng2 = dict((l[0], l[2]) for l in language_matrix if l[2])
lng2_to_lng3 = dict((l[2], l[0]) for l in language_matrix if l[2])
# we only return the first given english name, hoping it is the most used one
-lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0]) for l in language_matrix if l[3])
-lng_en_name_to_lng3 = dict((en_name.lower(), l[0]) for l in language_matrix if l[3] for en_name in l[3].split('; '))
+lng3_to_lng_en_name = dict((l[0], l[3].split('; ')[0])
+ for l in language_matrix if l[3])
+lng_en_name_to_lng3 = dict((en_name.lower(), l[0])
+ for l in language_matrix if l[3]
+ for en_name in l[3].split('; '))
# we only return the first given french name, hoping it is the most used one
-lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0]) for l in language_matrix if l[4])
-lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0]) for l in language_matrix if l[4] for fr_name in l[4].split('; '))
+lng3_to_lng_fr_name = dict((l[0], l[4].split('; ')[0])
+ for l in language_matrix if l[4])
+lng_fr_name_to_lng3 = dict((fr_name.lower(), l[0])
+ for l in language_matrix if l[4]
+ for fr_name in l[4].split('; '))
def is_language(language):
return language.lower() in lng_all_names
+
class Language(object):
"""This class represents a human language.
- You can initialize it with pretty much everything, as it knows conversion from
- ISO-639 2-letter and 3-letter codes, English and French names.
+ You can initialize it with pretty much everything, as it knows conversion
+ from ISO-639 2-letter and 3-letter codes, English and French names.
>>> Language('fr')
Language(French)
@@ -79,12 +86,16 @@ class Language(object):
if len(language) == 2:
lang = lng2_to_lng3.get(language)
elif len(language) == 3:
- lang = language if language in lng3 else lng3term_to_lng3.get(language)
+ lang = (language
+ if language in lng3
+ else lng3term_to_lng3.get(language))
else:
- lang = lng_en_name_to_lng3.get(language) or lng_fr_name_to_lng3.get(language)
+ lang = (lng_en_name_to_lng3.get(language) or
+ lng_fr_name_to_lng3.get(language))
if lang is None:
- raise ValueError, 'The given string "%s" could not be identified as a language' % language
+ msg = 'The given string "%s" could not be identified as a language'
+ raise ValueError(msg % language)
self.lang = lang
@@ -103,7 +114,6 @@ class Language(object):
def french_name(self):
return lng3_to_lng_fr_name[self.lang]
-
def __hash__(self):
return hash(self.lang)
@@ -132,19 +142,15 @@ class Language(object):
return 'Language(%s)' % self
-
-def search_language(string, lang_filter = None):
+def search_language(string, lang_filter=None):
"""Looks for language patterns, and if found return the language object,
its group span and an associated confidence.
you can specify a list of allowed languages using the lang_filter argument,
as in lang_filter = [ 'fr', 'eng', 'spanish' ]
- Assumes there are sentinels at the beginning and end of the string that
- always allow matching a non-letter delimiting the language.
-
>>> search_language('movie [en].avi')
- (Language(English), (7, 9), 0.80000000000000004)
+ (Language(English), (7, 9), 0.8)
>>> search_language('the zen fat cat and the gay mad men got a new fan', lang_filter = ['en', 'fr', 'es'])
(None, None, None)
@@ -153,25 +159,27 @@ def search_language(string, lang_filter = None):
# list of common words which could be interpreted as languages, but which
# are far too common to be able to say they represent a language in the
# middle of a string (where they most likely carry their commmon meaning)
- lng_common_words = frozenset([ # english words
- 'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to',
- 'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan',
- 'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as',
- 'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr',
- # french words
- 'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que',
- 'mal', 'est', 'vol', 'or', 'mon', 'se',
- # spanish words
- 'la', 'el', 'del', 'por', 'mar',
- # other
- 'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san'
- ])
+ lng_common_words = frozenset([
+ # english words
+ 'is', 'it', 'am', 'mad', 'men', 'man', 'run', 'sin', 'st', 'to',
+ 'no', 'non', 'war', 'min', 'new', 'car', 'day', 'bad', 'bat', 'fan',
+ 'fry', 'cop', 'zen', 'gay', 'fat', 'cherokee', 'got', 'an', 'as',
+ 'cat', 'her', 'be', 'hat', 'sun', 'may', 'my', 'mr',
+ # french words
+ 'bas', 'de', 'le', 'son', 'vo', 'vf', 'ne', 'ca', 'ce', 'et', 'que',
+ 'mal', 'est', 'vol', 'or', 'mon', 'se',
+ # spanish words
+ 'la', 'el', 'del', 'por', 'mar',
+ # other
+ 'ind', 'arw', 'ts', 'ii', 'bin', 'chan', 'ss', 'san', 'oss', 'iii',
+ 'vi'
+ ])
sep = r'[](){} \._-+'
if lang_filter:
lang_filter = set(Language(l) for l in lang_filter)
- slow = string.lower()
+ slow = ' %s ' % string.lower()
confidence = 1.0 # for all of them
for lang in lng_all_names:
@@ -183,7 +191,7 @@ def search_language(string, lang_filter = None):
if pos != -1:
end = pos + len(lang)
# make sure our word is always surrounded by separators
- if slow[pos-1] not in sep or slow[end] not in sep:
+ if slow[pos - 1] not in sep or slow[end] not in sep:
continue
language = Language(slow[pos:end])
@@ -201,10 +209,11 @@ def search_language(string, lang_filter = None):
elif len(lang) == 3:
confidence = 0.9
else:
- # Note: we could either be really confident that we found a language
- # or assume that full language names are too common words
+ # Note: we could either be really confident that we found a
+ # language or assume that full language names are too
+ # common words
confidence = 0.3 # going with the low-confidence route here
- return language, (pos, end), confidence
+ return language, (pos - 1, end - 1), confidence
return None, None, None
diff --git a/libs/guessit/matcher.py b/libs/guessit/matcher.py
index 63e25987..cac172de 100644
--- a/libs/guessit/matcher.py
+++ b/libs/guessit/matcher.py
@@ -2,8 +2,7 @@
# -*- coding: utf-8 -*-
#
# GuessIt - A library for guessing information from filenames
-# Copyright (c) 2011 Nicolas Wack
-# Copyright (c) 2011 Ricard Marxer
+# Copyright (c) 2012 Nicolas Wack
#
# GuessIt is free software; you can redistribute it and/or modify it under
# the terms of the Lesser GNU General Public License as published by
@@ -19,282 +18,18 @@
# along with this program. If not, see .
#
-from guessit import fileutils, textutils
-from guessit.guess import Guess, merge_similar_guesses, merge_all, choose_int, choose_string
-from guessit.date import search_date, search_year
-from guessit.language import search_language
-from guessit.filetype import guess_filetype
-from guessit.patterns import video_exts, subtitle_exts, sep, deleted, video_rexps, websites, episode_rexps, weak_episode_rexps, non_episode_title, find_properties, canonical_form, unlikely_series
-from guessit.matchtree import get_group, find_group, leftover_valid_groups, tree_to_string
-from guessit.textutils import find_first_level_groups, split_on_groups, blank_region, clean_string, to_utf8
-from guessit.fileutils import split_path_components
-import datetime
-import os.path
-import re
+from guessit.matchtree import MatchTree
+from guessit.textutils import to_utf8
+from guessit.guess import (merge_similar_guesses, merge_all,
+ choose_int, choose_string)
import copy
import logging
-import mimetypes
log = logging.getLogger("guessit.matcher")
-
-def split_explicit_groups(string):
- """return the string split into explicit groups, that is, those either
- between parenthese, square brackets or curly braces, and those separated
- by a dash."""
- result = find_first_level_groups(string, '()')
- result = reduce(lambda l, x: l + find_first_level_groups(x, '[]'), result, [])
- result = reduce(lambda l, x: l + find_first_level_groups(x, '{}'), result, [])
- # do not do this at this moment, it is not strong enough and can break other
- # patterns, such as dates, etc...
- #result = reduce(lambda l, x: l + x.split('-'), result, [])
-
- return result
-
-
-def format_guess(guess):
- """Format all the found values to their natural type.
- For instance, a year would be stored as an int value, etc...
-
- Note that this modifies the dictionary given as input.
- """
- for prop, value in guess.items():
- if prop in ('season', 'episodeNumber', 'year', 'cdNumber', 'cdNumberTotal'):
- guess[prop] = int(guess[prop])
- elif isinstance(value, basestring):
- if prop in ('edition',):
- value = clean_string(value)
- guess[prop] = canonical_form(value)
-
- return guess
-
-
-def guess_groups(string, result, filetype):
- # add sentinels so we can match a separator char at either end of
- # our groups, even when they are at the beginning or end of the string
- # we will adjust the span accordingly later
- #
- # filetype can either be movie, moviesubtitle, episode, episodesubtitle
- current = ' ' + string + ' '
-
- regions = [] # list of (start, end) of matched regions
-
- def guessed(match_dict, confidence):
- guess = format_guess(Guess(match_dict, confidence = confidence))
- result.append(guess)
- log.debug('Found with confidence %.2f: %s' % (confidence, guess))
- return guess
-
- def update_found(string, guess, span, span_adjust = (0,0)):
- span = (span[0] + span_adjust[0],
- span[1] + span_adjust[1])
- regions.append((span, guess))
- return blank_region(string, span)
-
- # try to find dates first, as they are very specific
- date, span = search_date(current)
- if date:
- guess = guessed({ 'date': date }, confidence = 1.0)
- current = update_found(current, guess, span)
-
- # for non episodes only, look for year information
- if filetype not in ('episode', 'episodesubtitle'):
- year, span = search_year(current)
- if year:
- guess = guessed({ 'year': year }, confidence = 1.0)
- current = update_found(current, guess, span)
-
- # specific regexps (ie: cd number, season X episode, ...)
- for rexp, confidence, span_adjust in video_rexps:
- match = re.search(rexp, current, re.IGNORECASE)
- if match:
- metadata = match.groupdict()
- # is this the better place to put it? (maybe, as it is at least the soonest that we can catch it)
- if 'cdNumberTotal' in metadata and metadata['cdNumberTotal'] is None:
- del metadata['cdNumberTotal']
-
- guess = guessed(metadata, confidence = confidence)
- current = update_found(current, guess, match.span(), span_adjust)
-
- if filetype in ('episode', 'episodesubtitle'):
- for rexp, confidence, span_adjust in episode_rexps:
- match = re.search(rexp, current, re.IGNORECASE)
- if match:
- metadata = match.groupdict()
- guess = guessed(metadata, confidence = confidence)
- current = update_found(current, guess, match.span(), span_adjust)
-
-
- # Now websites, but as exact string instead of regexps
- clow = current.lower()
- for site in websites:
- pos = clow.find(site.lower())
- if pos != -1:
- guess = guessed({ 'website': site }, confidence = confidence)
- current = update_found(current, guess, (pos, pos+len(site)))
- clow = current.lower()
-
-
- # release groups have certain constraints, cannot be included in the previous general regexps
- group_names = [ r'\.(Xvid)-(?P.*?)[ \.]',
- r'\.(DivX)-(?P.*?)[\. ]',
- r'\.(DVDivX)-(?P.*?)[\. ]',
- ]
- for rexp in group_names:
- match = re.search(rexp, current, re.IGNORECASE)
- if match:
- metadata = match.groupdict()
- metadata.update({ 'videoCodec': match.group(1) })
- guess = guessed(metadata, confidence = 0.8)
- current = update_found(current, guess, match.span(), span_adjust = (1, -1))
-
-
- # common well-defined words and regexps
- confidence = 1.0 # for all of them
- for prop, value, pos, end in find_properties(current):
- guess = guessed({ prop: value }, confidence = confidence)
- current = update_found(current, guess, (pos, end))
-
-
- # weak guesses for episode number, only run it if we don't have an estimate already
- if filetype in ('episode', 'episodesubtitle'):
- if not any('episodeNumber' in match for match in result):
- for rexp, _, span_adjust in weak_episode_rexps:
- match = re.search(rexp, current, re.IGNORECASE)
- if match:
- metadata = match.groupdict()
- epnum = int(metadata['episodeNumber'])
- if epnum > 100:
- guess = guessed({ 'season': epnum // 100,
- 'episodeNumber': epnum % 100 }, confidence = 0.6)
- else:
- guess = guessed(metadata, confidence = 0.3)
- current = update_found(current, guess, match.span(), span_adjust)
-
- # try to find languages now
- language, span, confidence = search_language(current)
- while language:
- # is it a subtitle language?
- if 'sub' in clean_string(current[:span[0]]).lower().split(' '):
- guess = guessed({ 'subtitleLanguage': language }, confidence = confidence)
- else:
- guess = guessed({ 'language': language }, confidence = confidence)
- current = update_found(current, guess, span)
-
- language, span, confidence = search_language(current)
-
-
- # remove our sentinels now and ajust spans accordingly
- assert(current[0] == ' ' and current[-1] == ' ')
- current = current[1:-1]
- regions = [ ((start-1, end-1), guess) for (start, end), guess in regions ]
-
- # split into '-' separated subgroups (with required separator chars
- # around the dash)
- didx = current.find('-')
- while didx > 0:
- regions.append(((didx, didx), None))
- didx = current.find('-', didx+1)
-
- # cut our final groups, and rematch the guesses to the group that created
- # id, None if it is a leftover group
- region_spans = [ span for span, guess in regions ]
- string_groups = split_on_groups(string, region_spans)
- remaining_groups = split_on_groups(current, region_spans)
- guesses = []
-
- pos = 0
- for group in string_groups:
- found = False
- for span, guess in regions:
- if span[0] == pos:
- guesses.append(guess)
- found = True
- if not found:
- guesses.append(None)
-
- pos += len(group)
-
- return zip(string_groups,
- remaining_groups,
- guesses)
-
-
-def match_from_epnum_position(match_tree, epnum_pos, guessed, update_found):
- """guessed is a callback function to call with the guessed group
- update_found is a callback to update the match group and returns leftover groups."""
- pidx, eidx, gidx = epnum_pos
-
- # a few helper functions to be able to filter using high-level semantics
- def same_pgroup_before(group):
- _, (ppidx, eeidx, ggidx) = group
- return ppidx == pidx and (eeidx, ggidx) < (eidx, gidx)
-
- def same_pgroup_after(group):
- _, (ppidx, eeidx, ggidx) = group
- return ppidx == pidx and (eeidx, ggidx) > (eidx, gidx)
-
- def same_egroup_before(group):
- _, (ppidx, eeidx, ggidx) = group
- return ppidx == pidx and eeidx == eidx and ggidx < gidx
-
- def same_egroup_after(group):
- _, (ppidx, eeidx, ggidx) = group
- return ppidx == pidx and eeidx == eidx and ggidx > gidx
-
- leftover = leftover_valid_groups(match_tree)
-
- # if we have at least 1 valid group before the episodeNumber, then it's probably
- # the series name
- series_candidates = filter(same_pgroup_before, leftover)
- if len(series_candidates) >= 1:
- guess = guessed({ 'series': series_candidates[0][0] }, confidence = 0.7)
- leftover = update_found(leftover, series_candidates[0][1], guess)
-
- # only 1 group after (in the same path group) and it's probably the episode title
- title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
- filter(same_pgroup_after, leftover))
- if len(title_candidates) == 1:
- guess = guessed({ 'title': title_candidates[0][0] }, confidence = 0.5)
- leftover = update_found(leftover, title_candidates[0][1], guess)
- else:
- # try in the same explicit group, with lower confidence
- title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
- filter(same_egroup_after, leftover))
- if len(title_candidates) == 1:
- guess = guessed({ 'title': title_candidates[0][0] }, confidence = 0.4)
- leftover = update_found(leftover, title_candidates[0][1], guess)
-
- # epnumber is the first group and there are only 2 after it in same path group
- # -> season title - episode title
- already_has_title = (find_group(match_tree, 'title') != [])
-
- title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
- filter(same_pgroup_after, leftover))
- if (not already_has_title and # no title
- not filter(same_pgroup_before, leftover) and # no groups before
- len(title_candidates) == 2): # only 2 groups after
-
- guess = guessed({ 'series': title_candidates[0][0] }, confidence = 0.4)
- leftover = update_found(leftover, title_candidates[0][1], guess)
- guess = guessed({ 'title': title_candidates[1][0] }, confidence = 0.4)
- leftover = update_found(leftover, title_candidates[1][1], guess)
-
-
- # if we only have 1 remaining valid group in the pathpart before the filename,
- # then it's likely that it is the series name
- series_candidates = [ group for group in leftover if group[1][0] == pidx-1 ]
- if len(series_candidates) == 1:
- guess = guessed({ 'series': series_candidates[0][0] }, confidence = 0.5)
- leftover = update_found(leftover, series_candidates[0][1], guess)
-
- return match_tree
-
-
-
class IterativeMatcher(object):
- def __init__(self, filename, filetype = 'autodetect'):
+ def __init__(self, filename, filetype='autodetect'):
"""An iterative matcher tries to match different patterns that appear
in the filename.
@@ -325,7 +60,7 @@ class IterativeMatcher(object):
The first 3 lines indicates the group index in which a char in the
filename is located. So for instance, x264 is the group (0, 4, 1), and
it corresponds to a video codec, denoted by the letter'v' in the 4th line.
- (for more info, see guess.matchtree.tree_to_string)
+ (for more info, see guess.matchtree.to_string)
Second, it tries to merge all this information into a single object
@@ -333,266 +68,89 @@ class IterativeMatcher(object):
resolution when they arise.
"""
- if filetype not in ('autodetect', 'subtitle', 'video',
+ valid_filetypes = ('autodetect', 'subtitle', 'video',
'movie', 'moviesubtitle',
- 'episode', 'episodesubtitle'):
- raise ValueError, "filetype needs to be one of ('autodetect', 'subtitle', 'video', 'movie', 'moviesubtitle', 'episode', 'episodesubtitle')"
+ 'episode', 'episodesubtitle')
+ if filetype not in valid_filetypes:
+ raise ValueError("filetype needs to be one of %s" % valid_filetypes)
if not isinstance(filename, unicode):
log.debug('WARNING: given filename to matcher is not unicode...')
- match_tree = []
- result = [] # list of found metadata
-
- def guessed(match_dict, confidence):
- guess = format_guess(Guess(match_dict, confidence = confidence))
- result.append(guess)
- log.debug('Found with confidence %.2f: %s' % (confidence, guess))
- return guess
-
- def update_found(leftover, group_pos, guess):
- pidx, eidx, gidx = group_pos
- group = match_tree[pidx][eidx][gidx]
- match_tree[pidx][eidx][gidx] = (group[0],
- deleted * len(group[0]),
- guess)
- return [ g for g in leftover if g[1] != group_pos ]
+ self.match_tree = MatchTree(filename)
+ mtree = self.match_tree
+ mtree.guess.set('type', filetype, confidence=1.0)
+ def apply_transfo(transfo_name, *args, **kwargs):
+ transfo = __import__('guessit.transfo.' + transfo_name,
+ globals=globals(), locals=locals(),
+ fromlist=['process'], level=-1)
+ transfo.process(mtree, *args, **kwargs)
# 1- first split our path into dirs + basename + ext
- match_tree = split_path_components(filename)
+ apply_transfo('split_path_components')
- # try to detect the file type
- filetype, other = guess_filetype(filename, filetype)
- guessed({ 'type': filetype }, confidence = 1.0)
- extguess = guessed(other, confidence = 1.0)
+ # 2- guess the file type now (will be useful later)
+ apply_transfo('guess_filetype', filetype)
+ if mtree.guess['type'] == 'unknown':
+ return
- # guess the mimetype of the filename
- # TODO: handle other mimetypes not found on the default type_maps
- # mimetypes.types_map['.srt']='text/subtitle'
- mime, _ = mimetypes.guess_type(filename, strict=False)
- if mime is not None:
- guessed({ 'mimetype': mime }, confidence = 1.0)
+ # 3- split each of those into explicit groups (separated by parentheses
+ # or square brackets)
+ apply_transfo('split_explicit_groups')
- # remove the extension from the match tree, as all indices relative
- # the the filename groups assume the basename is the last one
- fileext = match_tree.pop(-1)[1:].lower()
+ # 4- try to match information for specific patterns
+ if mtree.guess['type'] in ('episode', 'episodesubtitle'):
+ strategy = ['guess_date', 'guess_video_rexps',
+ 'guess_episodes_rexps', 'guess_website',
+ 'guess_release_group', 'guess_properties',
+ 'guess_weak_episodes_rexps', 'guess_language']
+ else:
+ strategy = ['guess_date', 'guess_year', 'guess_video_rexps',
+ 'guess_website', 'guess_release_group',
+ 'guess_properties', 'guess_language']
+ for name in strategy:
+ apply_transfo(name)
- # 2- split each of those into explicit groups, if any
- # note: be careful, as this might split some regexps with more confidence such as
- # Alfleni-Team, or [XCT] or split a date such as (14-01-2008)
- match_tree = [ split_explicit_groups(part) for part in match_tree ]
+ # more guessers for both movies and episodes
+ for name in ['guess_bonus_features']:
+ apply_transfo(name)
+ # split into '-' separated subgroups (with required separator chars
+ # around the dash)
+ apply_transfo('split_on_dash')
- # 3- try to match information in decreasing order of confidence and
- # blank the matching group in the string if we found something
- for pathpart in match_tree:
- for gidx, explicit_group in enumerate(pathpart):
- pathpart[gidx] = guess_groups(explicit_group, result, filetype = filetype)
+ # 5- try to identify the remaining unknown groups by looking at their
+ # position relative to other known elements
+ if mtree.guess['type'] in ('episode', 'episodesubtitle'):
+ apply_transfo('guess_episode_info_from_position')
+ else:
+ apply_transfo('guess_movie_title_from_position')
- # 4- try to identify the remaining unknown groups by looking at their position
- # relative to other known elements
-
- if filetype in ('episode', 'episodesubtitle'):
- eps = find_group(match_tree, 'episodeNumber')
- if eps:
- match_tree = match_from_epnum_position(match_tree, eps[0], guessed, update_found)
-
- leftover = leftover_valid_groups(match_tree)
-
- if not eps:
- # if we don't have the episode number, but at least 2 groups in the
- # last path group, then it's probably series - eptitle
- title_candidates = filter(lambda g:g[0].lower() not in non_episode_title,
- filter(lambda g: g[1][0] == len(match_tree)-1,
- leftover_valid_groups(match_tree)))
- if len(title_candidates) >= 2:
- guess = guessed({ 'series': title_candidates[0][0] }, confidence = 0.4)
- leftover = update_found(leftover, title_candidates[0][1], guess)
- guess = guessed({ 'title': title_candidates[1][0] }, confidence = 0.4)
- leftover = update_found(leftover, title_candidates[1][1], guess)
-
-
- # if there's a path group that only contains the season info, then the previous one
- # is most likely the series title (ie: .../series/season X/...)
- eps = [ gpos for gpos in find_group(match_tree, 'season')
- if 'episodeNumber' not in get_group(match_tree, gpos)[2] ]
-
- if eps:
- pidx, eidx, gidx = eps[0]
- previous = [ group for group in leftover if group[1][0] == pidx - 1 ]
- if len(previous) == 1:
- guess = guessed({ 'series': previous[0][0] }, confidence = 0.5)
- leftover = update_found(leftover, previous[0][1], guess)
-
- # reduce the confidence of unlikely series
- for guess in result:
- if 'series' in guess:
- if guess['series'].lower() in unlikely_series:
- guess.set_confidence('series', guess.confidence('series') * 0.5)
-
-
- elif filetype in ('movie', 'moviesubtitle'):
- leftover_all = leftover_valid_groups(match_tree)
-
- # specific cases:
- # - movies/tttttt (yyyy)/tttttt.ccc
- try:
- if match_tree[-3][0][0][0].lower() == 'movies':
- # Note:too generic, might solve all the unittests as they all contain 'movies'
- # in their path
- #
- #if len(match_tree[-2][0]) == 1:
- # title = match_tree[-2][0][0]
- # guess = guessed({ 'title': clean_string(title[0]) }, confidence = 0.7)
- # update_found(leftover_all, title, guess)
-
- year_group = filter(lambda gpos: gpos[0] == len(match_tree)-2,
- find_group(match_tree, 'year'))[0]
- leftover = leftover_valid_groups(match_tree,
- valid = lambda g: ((g[0] and g[0][0] not in sep) and
- g[1][0] == len(match_tree) - 2))
- if len(match_tree[-2]) == 2 and year_group[1] == 1:
- title = leftover[0]
- guess = guessed({ 'title': clean_string(title[0]) },
- confidence = 0.8)
- update_found(leftover_all, title[1], guess)
- raise Exception # to exit the try catch now
-
- leftover = [ g for g in leftover_all if (g[1][0] == year_group[0] and
- g[1][1] < year_group[1] and
- g[1][2] < year_group[2]) ]
- leftover = sorted(leftover, key = lambda x:x[1])
- title = leftover[0]
- guess = guessed({ 'title': title[0] }, confidence = 0.8)
- leftover = update_found(leftover, title[1], guess)
- except:
- pass
-
- # if we have either format or videoCodec in the folder containing the file
- # or one of its parents, then we should probably look for the title in
- # there rather than in the basename
- props = filter(lambda g: g[0] <= len(match_tree) - 2,
- find_group(match_tree, 'videoCodec') +
- find_group(match_tree, 'format') +
- find_group(match_tree, 'language'))
- leftover = None
- if props and all(g[0] == props[0][0] for g in props):
- leftover = [ g for g in leftover_all if g[1][0] == props[0][0] ]
-
- if props and leftover:
- guess = guessed({ 'title': leftover[0][0] }, confidence = 0.7)
- leftover = update_found(leftover, leftover[0][1], guess)
-
- else:
- # first leftover group in the last path part sounds like a good candidate for title,
- # except if it's only one word and that the first group before has at least 3 words in it
- # (case where the filename contains an 8 chars short name and the movie title is
- # actually in the parent directory name)
- leftover = [ g for g in leftover_all if g[1][0] == len(match_tree)-1 ]
- if leftover:
- title, (pidx, eidx, gidx) = leftover[0]
- previous_pgroup_leftover = filter(lambda g: g[1][0] == pidx-1, leftover_all)
-
- if (title.count(' ') == 0 and
- previous_pgroup_leftover and
- previous_pgroup_leftover[0][0].count(' ') >= 2):
-
- guess = guessed({ 'title': previous_pgroup_leftover[0][0] }, confidence = 0.6)
- leftover = update_found(leftover, previous_pgroup_leftover[0][1], guess)
-
- else:
- guess = guessed({ 'title': title }, confidence = 0.6)
- leftover = update_found(leftover, leftover[0][1], guess)
- else:
- # if there were no leftover groups in the last path part, look in the one before that
- previous_pgroup_leftover = filter(lambda g: g[1][0] == len(match_tree)-2, leftover_all)
- if previous_pgroup_leftover:
- guess = guessed({ 'title': previous_pgroup_leftover[0][0] }, confidence = 0.6)
- leftover = update_found(leftover, previous_pgroup_leftover[0][1], guess)
-
-
-
-
-
-
- # 5- perform some post-processing steps
-
- # 5.1- try to promote language to subtitle language where it makes sense
- for pidx, eidx, gidx in find_group(match_tree, 'language'):
- string, remaining, guess = get_group(match_tree, (pidx, eidx, gidx))
-
- def promote_subtitle():
- guess.set('subtitleLanguage', guess['language'], confidence = guess.confidence('language'))
- del guess['language']
-
- # - if we matched a language in a file with a sub extension and that the group
- # is the last group of the filename, it is probably the language of the subtitle
- # (eg: 'xxx.english.srt')
- if (fileext in subtitle_exts and
- pidx == len(match_tree) - 1 and
- eidx == len(match_tree[pidx]) - 1):
- promote_subtitle()
-
- # - if a language is in an explicit group just preceded by "st", it is a subtitle
- # language (eg: '...st[fr-eng]...')
- if eidx > 0:
- previous = get_group(match_tree, (pidx, eidx-1, -1))
- if previous[0][-2:].lower() == 'st':
- promote_subtitle()
-
-
-
- # re-append the extension now
- match_tree.append([[(fileext, deleted*len(fileext), extguess)]])
-
- self.parts = result
- self.match_tree = match_tree
-
- if filename.startswith('/'):
- filename = ' ' + filename
-
- log.debug('Found match tree:\n%s\n%s' % (to_utf8(tree_to_string(match_tree)),
- to_utf8(filename)))
+ # 6- perform some post-processing steps
+ apply_transfo('post_process')
+ log.debug('Found match tree:\n%s' % (to_utf8(unicode(mtree))))
def matched(self):
# we need to make a copy here, as the merge functions work in place and
# calling them on the match tree would modify it
- parts = copy.deepcopy(self.parts)
- # 1- start by doing some common preprocessing tasks
+ parts = [node.guess for node in self.match_tree.nodes() if node.guess]
+ parts = copy.deepcopy(parts)
- # 1.1- ", the" at the end of a series title should be prepended to it
- for part in parts:
- if 'series' not in part:
- continue
-
- series = part['series']
- lseries = series.lower()
-
- if lseries[-4:] == ',the':
- part['series'] = 'The ' + series[:-4]
-
- if lseries[-5:] == ', the':
- part['series'] = 'The ' + series[:-5]
-
-
- # 2- try to merge similar information together and give it a higher confidence
+ # 1- try to merge similar information together and give it a higher
+ # confidence
for int_part in ('year', 'season', 'episodeNumber'):
merge_similar_guesses(parts, int_part, choose_int)
- for string_part in ('title', 'series', 'container', 'format', 'releaseGroup', 'website',
- 'audioCodec', 'videoCodec', 'screenSize', 'episodeFormat'):
+ for string_part in ('title', 'series', 'container', 'format',
+ 'releaseGroup', 'website', 'audioCodec',
+ 'videoCodec', 'screenSize', 'episodeFormat'):
merge_similar_guesses(parts, string_part, choose_string)
- result = merge_all(parts, append = ['language', 'subtitleLanguage', 'other'])
-
- # 3- some last minute post-processing
- if (result['type'] == 'episode' and
- 'season' not in result and
- result.get('episodeFormat', '') == 'Minisode'):
- result['season'] = 0
+ result = merge_all(parts,
+ append=['language', 'subtitleLanguage', 'other'])
log.debug('Final result: ' + result.nice_string())
return result
diff --git a/libs/guessit/matchtree.py b/libs/guessit/matchtree.py
index 944cab03..634cbf7a 100644
--- a/libs/guessit/matchtree.py
+++ b/libs/guessit/matchtree.py
@@ -18,136 +18,243 @@
# along with this program. If not, see .
#
-from guessit.patterns import deleted
-from guessit.textutils import clean_string
+from guessit import Guess
+from guessit.textutils import clean_string, str_fill, to_utf8
+from guessit.patterns import group_delimiters
import logging
log = logging.getLogger("guessit.matchtree")
+class BaseMatchTree(object):
+ """A MatchTree represents the hierarchical split of a string into its
+ constituent semantic groups."""
-def tree_to_string(tree):
- """Return a string representation for the given tree.
+ def __init__(self, string='', span=None, parent=None):
+ self.string = string
+ self.span = span or (0, len(string))
+ self.parent = parent
+ self.children = []
+ self.guess = Guess()
- The lines convey the following information:
- - line 1: path idx
- - line 2: explicit group idx
- - line 3: group index
- - line 4: remaining info
- - line 5: meaning conveyed
+ @property
+ def value(self):
+ return self.string[self.span[0]:self.span[1]]
- Meaning is a letter indicating what type of info was matched by this group,
- for instance 't' = title, 'f' = format, 'l' = language, etc...
+ @property
+ def clean_value(self):
+ return clean_string(self.value)
- An example is the following:
+ @property
+ def offset(self):
+ return self.span[0]
- 0000000000000000000000000000000000000000000000000000000000000000000000000000000000 111
- 0000011111111111112222222222222233333333444444444444444455555555666777777778888888 000
- 0000000000000000000000000000000001111112011112222333333401123334000011233340000000 000
- __________________(The.Prestige).______.[____.HP.______.{__-___}.St{__-___}.Chaps].___
- xxxxxttttttttttttt ffffff vvvv xxxxxx ll lll xx xxx ccc
- [XCT].Le.Prestige.(The.Prestige).DVDRip.[x264.HP.He-Aac.{Fr-Eng}.St{Fr-Eng}.Chaps].mkv
+ @property
+ def info(self):
+ result = dict(self.guess)
- (note: the last line representing the filename is not pat of the tree representation)
- """
- m_tree = [ '', # path level index
- '', # explicit group index
- '', # matched regexp and dash-separated
- '', # groups leftover that couldn't be matched
- '', # meaning conveyed: E = episodenumber, S = season, ...
- ]
+ for c in self.children:
+ result.update(c.info)
+
+ return result
+
+ @property
+ def root(self):
+ if not self.parent:
+ return self
+
+ return self.parent.root
+
+ @property
+ def depth(self):
+ if self.is_leaf():
+ return 0
+
+ return 1 + max(c.depth for c in self.children)
+
+ def is_leaf(self):
+ return self.children == []
+
+ def add_child(self, span):
+ child = MatchTree(self.string, span=span, parent=self)
+ self.children.append(child)
+
+ def partition(self, indices):
+ indices = sorted(indices)
+ if indices[0] != 0:
+ indices.insert(0, 0)
+ if indices[-1] != len(self.value):
+ indices.append(len(self.value))
+
+ for start, end in zip(indices[:-1], indices[1:]):
+ self.add_child(span=(self.offset + start,
+ self.offset + end))
+
+ def split_on_components(self, components):
+ offset = 0
+ for c in components:
+ start = self.value.find(c, offset)
+ end = start + len(c)
+ self.add_child(span=(self.offset + start,
+ self.offset + end))
+ offset = end
+
+ def nodes_at_depth(self, depth):
+ if depth == 0:
+ yield self
+
+ for child in self.children:
+ for node in child.nodes_at_depth(depth - 1):
+ yield node
+
+ @property
+ def node_idx(self):
+ if self.parent is None:
+ return ()
+ return self.parent.node_idx + (self.parent.children.index(self),)
+
+ def node_at(self, idx):
+ if not idx:
+ return self
+
+ try:
+ return self.children[idx[0]].node_at(idx[1:])
+ except:
+ raise ValueError('Non-existent node index: %s' % (idx,))
+
+ def nodes(self):
+ yield self
+ for child in self.children:
+ for node in child.nodes():
+ yield node
+
+ def _leaves(self):
+ if self.is_leaf():
+ yield self
+ else:
+ for child in self.children:
+ # pylint: disable=W0212
+ for leaf in child._leaves():
+ yield leaf
+
+ def leaves(self):
+ return list(self._leaves())
+
+ def to_string(self):
+ empty_line = ' ' * len(self.string)
- def add_char(pidx, eidx, gidx, remaining, meaning = None):
- nr = len(remaining)
def to_hex(x):
if isinstance(x, int):
- return str(x) if x < 10 else chr(55+x)
+ return str(x) if x < 10 else chr(55 + x)
return x
- m_tree[0] = m_tree[0] + to_hex(pidx) * nr
- m_tree[1] = m_tree[1] + to_hex(eidx) * nr
- m_tree[2] = m_tree[2] + to_hex(gidx) * nr
- m_tree[3] = m_tree[3] + remaining
- m_tree[4] = m_tree[4] + str(meaning or ' ') * nr
- def meaning(result):
- mmap = { 'episodeNumber': 'E',
- 'season': 'S',
- 'extension': 'e',
- 'format': 'f',
- 'language': 'l',
- 'videoCodec': 'v',
- 'audioCodec': 'a',
- 'website': 'w',
- 'container': 'c',
- 'series': 'T',
- 'title': 't',
- 'date': 'd',
- 'year': 'y',
- 'releaseGroup': 'r',
- 'screenSize': 's'
- }
+ def meaning(result):
+ mmap = { 'episodeNumber': 'E',
+ 'season': 'S',
+ 'extension': 'e',
+ 'format': 'f',
+ 'language': 'l',
+ 'videoCodec': 'v',
+ 'audioCodec': 'a',
+ 'website': 'w',
+ 'container': 'c',
+ 'series': 'T',
+ 'title': 't',
+ 'date': 'd',
+ 'year': 'y',
+ 'releaseGroup': 'r',
+ 'screenSize': 's'
+ }
- if result is None:
- return ' '
+ if result is None:
+ return ' '
- for prop, l in mmap.items():
- if prop in result:
- return l
+ for prop, l in mmap.items():
+ if prop in result:
+ return l
- return 'x'
+ return 'x'
- for pidx, pathpart in enumerate(tree):
- for eidx, explicit_group in enumerate(pathpart):
- for gidx, (group, remaining, result) in enumerate(explicit_group):
- add_char(pidx, eidx, gidx, remaining, meaning(result))
+ lines = [ empty_line ] * (self.depth + 2) # +2: remaining, meaning
+ lines[-2] = self.string
- # special conditions for the path separator
- if pidx < len(tree) - 2:
- add_char(' ', ' ', ' ', '/')
- elif pidx == len(tree) - 2:
- add_char(' ', ' ', ' ', '.')
+ for node in self.nodes():
+ if node == self:
+ continue
- return '\n'.join(m_tree)
+ idx = node.node_idx
+ depth = len(idx) - 1
+ if idx:
+ lines[depth] = str_fill(lines[depth], node.span,
+ to_hex(idx[-1]))
+ if node.guess:
+ lines[-2] = str_fill(lines[-2], node.span, '_')
+ lines[-1] = str_fill(lines[-1], node.span, meaning(node.guess))
+
+ lines.append(self.string)
+
+ return '\n'.join(lines)
+
+ def __unicode__(self):
+ return self.to_string()
+
+ def __str__(self):
+ return to_utf8(unicode(self))
+class MatchTree(BaseMatchTree):
+ """The MatchTree contains a few "utility" methods which are not necessary
+ for the BaseMatchTree, but add a lot of convenience for writing
+ higher-level rules."""
-def iterate_groups(match_tree):
- """Iterate over all the groups in a match_tree and return them as pairs
- of (group_pos, group) where:
- - group_pos = (pidx, eidx, gidx)
- - group = (string, remaining, guess)
- """
- for pidx, pathpart in enumerate(match_tree):
- for eidx, explicit_group in enumerate(pathpart):
- for gidx, group in enumerate(explicit_group):
- yield (pidx, eidx, gidx), group
+ def _unidentified_leaves(self,
+ valid=lambda leaf: len(leaf.clean_value) >= 2):
+ for leaf in self._leaves():
+ if not leaf.guess and valid(leaf):
+ yield leaf
+ def unidentified_leaves(self,
+ valid=lambda leaf: len(leaf.clean_value) >= 2):
+ return list(self._unidentified_leaves(valid))
-def find_group(match_tree, prop):
- """Find the list of groups that resulted in a guess that contains the
- asked property."""
- result = []
- for gpos, (string, remaining, guess) in iterate_groups(match_tree):
- if guess and prop in guess:
- result.append(gpos)
- return result
+ def _leaves_containing(self, property_name):
+ if isinstance(property_name, basestring):
+ property_name = [ property_name ]
-def get_group(match_tree, gpos):
- pidx, eidx, gidx = gpos
- return match_tree[pidx][eidx][gidx]
+ for leaf in self._leaves():
+ for prop in property_name:
+ if prop in leaf.guess:
+ yield leaf
+ break
+ def leaves_containing(self, property_name):
+ return list(self._leaves_containing(property_name))
-def leftover_valid_groups(match_tree, valid = lambda s: len(s[0]) > 3):
- """Return the list of valid string groups (eg: len(s) > 3) that could not be
- matched to anything as a list of pairs (cleaned_str, group_pos)."""
- leftover = []
- for gpos, (group, remaining, guess) in iterate_groups(match_tree):
- if not guess:
- clean_str = clean_string(remaining)
- if valid((clean_str, gpos)):
- leftover.append((clean_str, gpos))
+ def first_leaf_containing(self, property_name):
+ try:
+ return next(self._leaves_containing(property_name))
+ except StopIteration:
+ return None
- return leftover
+ def _previous_unidentified_leaves(self, node):
+ node_idx = node.node_idx
+ for leaf in self._unidentified_leaves():
+ if leaf.node_idx < node_idx:
+ yield leaf
+ def previous_unidentified_leaves(self, node):
+ return list(self._previous_unidentified_leaves(node))
+ def _previous_leaves_containing(self, node, property_name):
+ node_idx = node.node_idx
+ for leaf in self._leaves_containing(property_name):
+ if leaf.node_idx < node_idx:
+ yield leaf
+ def previous_leaves_containing(self, node, property_name):
+ return list(self._previous_leaves_containing(node, property_name))
+
+ def is_explicit(self):
+ """Return whether the group was explicitly enclosed by
+ parentheses/square brackets/etc."""
+ return (self.value[0] + self.value[-1]) in group_delimiters
diff --git a/libs/guessit/patterns.py b/libs/guessit/patterns.py
old mode 100644
new mode 100755
index 8331723e..4125fb7b
--- a/libs/guessit/patterns.py
+++ b/libs/guessit/patterns.py
@@ -22,10 +22,13 @@
subtitle_exts = [ 'srt', 'idx', 'sub', 'ssa', 'txt' ]
-video_exts = [ 'avi', 'mkv', 'mpg', 'mp4', 'm4v', 'mov', 'ogg', 'ogm', 'ogv', 'wmv', 'divx' ]
+video_exts = [ 'avi', 'mkv', 'mpg', 'mp4', 'm4v', 'mov', 'ogg', 'ogm', 'ogv',
+ 'wmv', 'divx' ]
+
+group_delimiters = [ '()', '[]', '{}' ]
# separator character regexp
-sep = r'[][)(}{+ \._-]' # regexp art, hehe :D
+sep = r'[][)(}{+ /\._-]' # regexp art, hehe :D
# character used to represent a deleted char (when matching groups)
deleted = '_'
@@ -35,28 +38,33 @@ episode_rexps = [ # ... Season 2 ...
(r'season (?P[0-9]+)', 1.0, (0, 0)),
(r'saison (?P[0-9]+)', 1.0, (0, 0)),
- # ... s02-x01 ...
- (r's(?P[0-9]{1,2})-x(?P[0-9]{1,2})[^0-9]', 1.0, (0, -1)),
-
# ... s02e13 ...
(r'[Ss](?P[0-9]{1,2}).{,3}[EeXx](?P[0-9]{1,2})[^0-9]', 1.0, (0, -1)),
# ... 2x13 ...
- (r'[^0-9](?P[0-9]{1,2})[x\.](?P[0-9]{2})[^0-9]', 0.8, (1, -1)),
+ (r'[^0-9](?P[0-9]{1,2})x(?P[0-9]{2})[^0-9]', 0.8, (1, -1)),
# ... s02 ...
- (sep + r's(?P[0-9]{1,2})' + sep + '?', 0.6, (1, -1)),
+ #(sep + r's(?P[0-9]{1,2})' + sep, 0.6, (1, -1)),
+ (r's(?P[0-9]{1,2})[^0-9]', 0.6, (0, -1)),
# v2 or v3 for some mangas which have multiples rips
- (sep + r'(?P[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)),
+ (r'(?P[0-9]{1,3})v[23]' + sep, 0.6, (0, 0)),
+
+ # ... ep 23 ...
+ ('ep' + sep + r'(?P[0-9]{1,2})[^0-9]', 0.7, (0, -1))
]
weak_episode_rexps = [ # ... 213 or 0106 ...
- (sep + r'(?P[0-9]{1,4})' + sep, 0.3, (1, -1)),
+ (sep + r'(?P[0-9]{1,4})' + sep, (1, -1)),
+
+ # ... 2x13 ...
+ (sep + r'[^0-9](?P[0-9]{1,2})\.(?P[0-9]{2})[^0-9]' + sep, (1, -1)),
+
]
-non_episode_title = [ 'extras' ]
+non_episode_title = [ 'extras', 'rip' ]
video_rexps = [ # cd number
@@ -76,7 +84,13 @@ video_rexps = [ # cd number
(r'(?P[0-9]{3,4})x(?P[0-9]{3,4})', 0.9, (0, 0)),
# website
- (r'(?Pwww(\.[a-zA-Z0-9]+){2,3})', 0.8, (0, 0))
+ (r'(?Pwww(\.[a-zA-Z0-9]+){2,3})', 0.8, (0, 0)),
+
+ # bonusNumber: ... x01 ...
+ (r'x(?P[0-9]{1,2})', 1.0, (0, 0)),
+
+ # filmNumber: ... f01 ...
+ (r'f(?P[0-9]{1,2})', 1.0, (0, 0))
]
websites = [ 'tvu.org.ru', 'emule-island.com', 'UsaBit.com', 'www.divx-overnet.com', 'sharethefiles.com' ]
@@ -87,7 +101,7 @@ properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'B
'HDRip', 'DVD', 'DVDivX', 'HDTV', 'DVB', 'DVBRip', 'PDTV', 'WEBRip',
'DVDSCR', 'Screener', 'VHS', 'VIDEO_TS' ],
- 'screenSize': [ '720p', '720' ],
+ 'screenSize': [ '720p', '720', '1080p', '1080' ],
'videoCodec': [ 'XviD', 'DivX', 'x264', 'h264', 'Rv10' ],
@@ -98,18 +112,18 @@ properties = { 'format': [ 'DVDRip', 'HD-DVD', 'HDDVD', 'HDDVDRip', 'BluRay', 'B
'releaseGroup': [ 'ESiR', 'WAF', 'SEPTiC', '[XCT]', 'iNT', 'PUKKA',
'CHD', 'ViTE', 'TLF', 'DEiTY', 'FLAiTE',
'MDX', 'GM4F', 'DVL', 'SVD', 'iLUMiNADOS', ' FiNaLe',
- 'UnSeeN', 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL' ],
+ 'UnSeeN', 'aXXo', 'KLAXXON', 'NoTV', 'ZeaL', 'LOL',
+ 'HDBRiSe' ],
'episodeFormat': [ 'Minisode', 'Minisodes' ],
'other': [ '5ch', 'PROPER', 'REPACK', 'LIMITED', 'DualAudio', 'iNTERNAL', 'Audiofixed', 'R5',
'complete', 'classic', # not so sure about these ones, could appear in a title
'ws', # widescreen
- #'SE', # special edition
- # TODO: director's cut
],
}
+
def find_properties(filename):
result = []
clow = filename.lower()
@@ -119,7 +133,7 @@ def find_properties(filename):
if pos != -1:
end = pos + len(value)
# make sure our word is always surrounded by separators
- if ((pos > 0 and clow[pos-1] not in sep) or
+ if ((pos > 0 and clow[pos - 1] not in sep) or
(end < len(clow) and clow[end] not in sep)):
# note: sep is a regexp, but in this case using it as
# a sequence achieves the same goal
@@ -137,6 +151,7 @@ property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ],
'DivX': [ 'DVDivX' ],
'h264': [ 'x264' ],
'720p': [ '720' ],
+ '1080p': [ '1080' ],
'AAC': [ 'He-AAC', 'AAC-He' ],
'Special Edition': [ 'Special' ],
'Collector Edition': [ 'Collector' ],
@@ -145,14 +160,21 @@ property_synonyms = { 'DVD': [ 'DVDRip', 'VIDEO_TS' ],
}
-reverse_synonyms = {}
-for prop, values in properties.items():
- for value in values:
- reverse_synonyms[value.lower()] = value
+def revert_synonyms():
+ reverse = {}
+
+ for _, values in properties.items():
+ for value in values:
+ reverse[value.lower()] = value
+
+ for canonical, synonyms in property_synonyms.items():
+ for synonym in synonyms:
+ reverse[synonym.lower()] = canonical
+
+ return reverse
+
+reverse_synonyms = revert_synonyms()
-for canonical, synonyms in property_synonyms.items():
- for synonym in synonyms:
- reverse_synonyms[synonym.lower()] = canonical
def canonical_form(string):
return reverse_synonyms.get(string.lower(), string)
diff --git a/libs/guessit/slogging.py b/libs/guessit/slogging.py
index 308ef887..34d9af79 100644
--- a/libs/guessit/slogging.py
+++ b/libs/guessit/slogging.py
@@ -28,8 +28,8 @@ RED_FONT = "\x1B[0;31m"
RESET_FONT = "\x1B[0m"
-def setupLogging(colored = True):
- """Sets up a nice colored logger as the main application logger (not only smewt itself)."""
+def setupLogging(colored=True):
+ """Set up a nice colored logger as the main application logger."""
class SimpleFormatter(logging.Formatter):
def __init__(self):
@@ -38,7 +38,9 @@ def setupLogging(colored = True):
class ColoredFormatter(logging.Formatter):
def __init__(self):
- self.fmt = '%(levelname)-8s ' + BLUE_FONT + '%(module)s:%(funcName)s' + RESET_FONT + ' -- %(message)s'
+ self.fmt = ('%(levelname)-8s ' +
+ BLUE_FONT + '%(name)s:%(funcName)s' +
+ RESET_FONT + ' -- %(message)s')
logging.Formatter.__init__(self, self.fmt)
def format(self, record):
@@ -50,11 +52,9 @@ def setupLogging(colored = True):
else:
return RED_FONT + result
-
ch = logging.StreamHandler()
if colored and sys.platform != 'win32':
ch.setFormatter(ColoredFormatter())
else:
ch.setFormatter(SimpleFormatter())
logging.getLogger().addHandler(ch)
-
diff --git a/libs/guessit/textutils.py b/libs/guessit/textutils.py
index 2ab88815..cfe31c4d 100644
--- a/libs/guessit/textutils.py
+++ b/libs/guessit/textutils.py
@@ -18,47 +18,59 @@
# along with this program. If not, see .
#
-from guessit.patterns import sep, deleted
+from guessit.patterns import sep
import copy
# string-related functions
+
def strip_brackets(s):
if not s:
return s
- if s[0] == '[' and s[-1] == ']': return s[1:-1]
- if s[0] == '(' and s[-1] == ')': return s[1:-1]
- if s[0] == '{' and s[-1] == '}': return s[1:-1]
+
+ if ((s[0] == '[' and s[-1] == ']') or
+ (s[0] == '(' and s[-1] == ')') or
+ (s[0] == '{' and s[-1] == '}')):
+ return s[1:-1]
+
return s
def clean_string(s):
- for c in sep:
+ for c in sep[:-2]: # do not remove dashes ('-')
s = s.replace(c, ' ')
parts = s.split()
- return ' '.join(p for p in parts if p != '')
+ result = ' '.join(p for p in parts if p != '')
+
+ # now also remove dashes on the outer part of the string
+ while result and result[0] in sep:
+ result = result[1:]
+ while result and result[-1] in sep:
+ result = result[:-1]
+
+ return result
def str_replace(string, pos, c):
return string[:pos] + c + string[pos+1:]
-def blank_region(string, region, blank_sep = deleted):
+
+def str_fill(string, region, c):
start, end = region
- return string[:start] + blank_sep * (end - start) + string[end:]
+ return string[:start] + c * (end - start) + string[end:]
-def between(s, left, right):
- return s.split(left)[1].split(right)[0]
-
def to_utf8(o):
- '''converts all unicode strings found in the given object to utf-8 strings'''
+ """Convert all unicode strings found in the given object to utf-8
+ strings."""
if isinstance(o, unicode):
return o.encode('utf-8')
elif isinstance(o, list):
return [ to_utf8(i) for i in o ]
elif isinstance(o, dict):
- result = copy.deepcopy(o) # need to do it like that to handle Guess instances correctly
+ # need to do it like that to handle Guess instances correctly
+ result = copy.deepcopy(o)
for key, value in o.items():
result[to_utf8(key)] = to_utf8(value)
return result
@@ -68,8 +80,10 @@ def to_utf8(o):
def levenshtein(a, b):
- if not a: return len(b)
- if not b: return len(a)
+ if not a:
+ return len(b)
+ if not b:
+ return len(a)
m = len(a)
n = len(b)
@@ -160,14 +174,13 @@ def split_on_groups(string, groups):
if boundaries[-1] != len(string):
boundaries.append(len(string))
- groups = [ string[start:end] for start, end in zip(boundaries[:-1], boundaries[1:]) ]
+ groups = [ string[start:end] for start, end in zip(boundaries[:-1],
+ boundaries[1:]) ]
- return filter(bool, groups) # return only non-empty groups
+ return [ g for g in groups if g ] # return only non-empty groups
-
-
-def find_first_level_groups(string, enclosing, blank_sep = None):
+def find_first_level_groups(string, enclosing, blank_sep=None):
"""Return a list of groups that could be split because of explicit grouping.
The groups are delimited by the given enclosing characters.
@@ -203,8 +216,3 @@ def find_first_level_groups(string, enclosing, blank_sep = None):
string = str_replace(string, end-1, blank_sep)
return split_on_groups(string, groups)
-
-
-
-
-
diff --git a/libs/guessit/transfo/__init__.py b/libs/guessit/transfo/__init__.py
new file mode 100644
index 00000000..eb72bebd
--- /dev/null
+++ b/libs/guessit/transfo/__init__.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit import Guess
+from guessit.patterns import canonical_form
+from guessit.textutils import clean_string
+import logging
+
+log = logging.getLogger('guessit.transfo')
+
+
+def found_property(node, name, confidence):
+ node.guess = Guess({name: node.clean_value}, confidence=confidence)
+ log.debug('Found with confidence %.2f: %s' % (confidence, node.guess))
+
+
+def format_guess(guess):
+ """Format all the found values to their natural type.
+ For instance, a year would be stored as an int value, etc...
+
+ Note that this modifies the dictionary given as input.
+ """
+ for prop, value in guess.items():
+ if prop in ('season', 'episodeNumber', 'year', 'cdNumber',
+ 'cdNumberTotal', 'bonusNumber', 'filmNumber'):
+ guess[prop] = int(guess[prop])
+ elif isinstance(value, basestring):
+ if prop in ('edition',):
+ value = clean_string(value)
+ guess[prop] = canonical_form(value)
+
+ return guess
+
+
+def find_and_split_node(node, strategy, logger):
+ string = ' %s ' % node.value # add sentinels
+ for matcher, confidence in strategy:
+ if getattr(matcher, 'use_node', False):
+ result, span = matcher(string, node)
+ else:
+ result, span = matcher(string)
+
+ if result:
+ # readjust span to compensate for sentinels
+ span = (span[0] - 1, span[1] - 1)
+
+ if isinstance(result, Guess):
+ if confidence is None:
+ confidence = result.confidence(result.keys()[0])
+ else:
+ if confidence is None:
+ confidence = 1.0
+
+ guess = format_guess(Guess(result, confidence=confidence))
+ msg = 'Found with confidence %.2f: %s' % (confidence, guess)
+ (logger or log).debug(msg)
+
+ node.partition(span)
+ absolute_span = (span[0] + node.offset, span[1] + node.offset)
+ for child in node.children:
+ if child.span == absolute_span:
+ child.guess = guess
+ else:
+ find_and_split_node(child, strategy, logger)
+ return
+
+
+class SingleNodeGuesser(object):
+ def __init__(self, guess_func, confidence, logger=None):
+ self.guess_func = guess_func
+ self.confidence = confidence
+ self.logger = logger
+
+ def process(self, mtree):
+ # strategy is a list of pairs (guesser, confidence)
+ # - if the guesser returns a guessit.Guess and confidence is specified,
+ # it will override it, otherwise it will leave the guess confidence
+ # - if the guesser returns a simple dict as a guess and confidence is
+ # specified, it will use it, or 1.0 otherwise
+ strategy = [ (self.guess_func, self.confidence) ]
+
+ for node in mtree.unidentified_leaves():
+ find_and_split_node(node, strategy, self.logger)
diff --git a/libs/guessit/transfo/guess_bonus_features.py b/libs/guessit/transfo/guess_bonus_features.py
new file mode 100644
index 00000000..dcb90b3f
--- /dev/null
+++ b/libs/guessit/transfo/guess_bonus_features.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit.transfo import found_property
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_bonus_features")
+
+
+def process(mtree):
+ def previous_group(g):
+ for leaf in mtree.unidentified_leaves()[::-1]:
+ if leaf.node_idx < g.node_idx:
+ return leaf
+
+ def next_group(g):
+ for leaf in mtree.unidentified_leaves():
+ if leaf.node_idx > g.node_idx:
+ return leaf
+
+ def same_group(g1, g2):
+ return g1.node_idx[:2] == g2.node_idx[:2]
+
+ bonus = [ node for node in mtree.leaves() if 'bonusNumber' in node.guess ]
+ if bonus:
+ bonusTitle = next_group(bonus[0])
+ if same_group(bonusTitle, bonus[0]):
+ found_property(bonusTitle, 'bonusTitle', 0.8)
+
+ filmNumber = [ node for node in mtree.leaves()
+ if 'filmNumber' in node.guess ]
+ if filmNumber:
+ filmSeries = previous_group(filmNumber[0])
+ found_property(filmSeries, 'filmSeries', 0.9)
+
+ title = next_group(filmNumber[0])
+ found_property(title, 'title', 0.9)
+
+ season = [ node for node in mtree.leaves() if 'season' in node.guess ]
+ if season and 'bonusNumber' in mtree.info:
+ series = previous_group(season[0])
+ if same_group(series, season[0]):
+ found_property(series, 'series', 0.9)
diff --git a/libs/guessit/transfo/guess_date.py b/libs/guessit/transfo/guess_date.py
new file mode 100644
index 00000000..c72d66ad
--- /dev/null
+++ b/libs/guessit/transfo/guess_date.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit.transfo import SingleNodeGuesser
+from guessit.date import search_date
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_date")
+
+
+def guess_date(string):
+ date, span = search_date(string)
+ if date:
+ return { 'date': date }, span
+ else:
+ return None, None
+
+
+def process(mtree):
+ SingleNodeGuesser(guess_date, 1.0, log).process(mtree)
diff --git a/libs/guessit/transfo/guess_episode_info_from_position.py b/libs/guessit/transfo/guess_episode_info_from_position.py
new file mode 100644
index 00000000..fe1a7525
--- /dev/null
+++ b/libs/guessit/transfo/guess_episode_info_from_position.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit.transfo import found_property
+from guessit.patterns import non_episode_title, unlikely_series
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_episode_info_from_position")
+
+
+def match_from_epnum_position(mtree, node):
+ epnum_idx = node.node_idx
+
+ # a few helper functions to be able to filter using high-level semantics
+ def before_epnum_in_same_pathgroup():
+ return [ leaf for leaf in mtree.unidentified_leaves()
+ if (leaf.node_idx[0] == epnum_idx[0] and
+ leaf.node_idx[1:] < epnum_idx[1:]) ]
+
+ def after_epnum_in_same_pathgroup():
+ return [ leaf for leaf in mtree.unidentified_leaves()
+ if (leaf.node_idx[0] == epnum_idx[0] and
+ leaf.node_idx[1:] > epnum_idx[1:]) ]
+
+ def after_epnum_in_same_explicitgroup():
+ return [ leaf for leaf in mtree.unidentified_leaves()
+ if (leaf.node_idx[:2] == epnum_idx[:2] and
+ leaf.node_idx[2:] > epnum_idx[2:]) ]
+
+ # epnumber is the first group and there are only 2 after it in same
+ # path group
+ # -> series title - episode title
+ title_candidates = [ n for n in after_epnum_in_same_pathgroup()
+ if n.clean_value.lower() not in non_episode_title ]
+ if ('title' not in mtree.info and # no title
+ before_epnum_in_same_pathgroup() == [] and # no groups before
+ len(title_candidates) == 2): # only 2 groups after
+
+ found_property(title_candidates[0], 'series', confidence=0.4)
+ found_property(title_candidates[1], 'title', confidence=0.4)
+ return
+
+ # if we have at least 1 valid group before the episodeNumber, then it's
+ # probably the series name
+ series_candidates = before_epnum_in_same_pathgroup()
+ if len(series_candidates) >= 1:
+ found_property(series_candidates[0], 'series', confidence=0.7)
+
+ # only 1 group after (in the same path group) and it's probably the
+ # episode title
+ title_candidates = [ n for n in after_epnum_in_same_pathgroup()
+ if n.clean_value.lower() not in non_episode_title ]
+
+ if len(title_candidates) == 1:
+ found_property(title_candidates[0], 'title', confidence=0.5)
+ return
+ else:
+ # try in the same explicit group, with lower confidence
+ title_candidates = [ n for n in after_epnum_in_same_explicitgroup()
+ if n.clean_value.lower() not in non_episode_title
+ ]
+ if len(title_candidates) == 1:
+ found_property(title_candidates[0], 'title', confidence=0.4)
+ return
+ elif len(title_candidates) > 1:
+ found_property(title_candidates[0], 'title', confidence=0.3)
+ return
+
+ # get the one with the longest value
+ title_candidates = [ n for n in after_epnum_in_same_pathgroup()
+ if n.clean_value.lower() not in non_episode_title ]
+ if title_candidates:
+ maxidx = -1
+ maxv = -1
+ for i, c in enumerate(title_candidates):
+ if len(c.clean_value) > maxv:
+ maxidx = i
+ maxv = len(c.clean_value)
+ found_property(title_candidates[maxidx], 'title', confidence=0.3)
+
+
+def process(mtree):
+ eps = [node for node in mtree.leaves() if 'episodeNumber' in node.guess]
+ if eps:
+ match_from_epnum_position(mtree, eps[0])
+
+ else:
+ # if we don't have the episode number, but at least 2 groups in the
+ # basename, then it's probably series - eptitle
+ basename = mtree.node_at((-2,))
+ title_candidates = [ n for n in basename.unidentified_leaves()
+ if n.clean_value.lower() not in non_episode_title
+ ]
+
+ if len(title_candidates) >= 2:
+ found_property(title_candidates[0], 'series', 0.4)
+ found_property(title_candidates[1], 'title', 0.4)
+
+ # if we only have 1 remaining valid group in the folder containing the
+ # file, then it's likely that it is the series name
+ try:
+ series_candidates = mtree.node_at((-3,)).unidentified_leaves()
+ except ValueError:
+ series_candidates = []
+
+ if len(series_candidates) == 1:
+ found_property(series_candidates[0], 'series', 0.3)
+
+ # if there's a path group that only contains the season info, then the
+ # previous one is most likely the series title (ie: ../series/season X/..)
+ eps = [ node for node in mtree.nodes()
+ if 'season' in node.guess and 'episodeNumber' not in node.guess ]
+
+ if eps:
+ previous = [ node for node in mtree.unidentified_leaves()
+ if node.node_idx[0] == eps[0].node_idx[0] - 1 ]
+ if len(previous) == 1:
+ found_property(previous[0], 'series', 0.5)
+
+ # reduce the confidence of unlikely series
+ for node in mtree.nodes():
+ if 'series' in node.guess:
+ if node.guess['series'].lower() in unlikely_series:
+ new_confidence = node.guess.confidence('series') * 0.5
+ node.guess.set_confidence('series', new_confidence)
diff --git a/libs/guessit/transfo/guess_episodes_rexps.py b/libs/guessit/transfo/guess_episodes_rexps.py
new file mode 100644
index 00000000..46dbc590
--- /dev/null
+++ b/libs/guessit/transfo/guess_episodes_rexps.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit import Guess
+from guessit.transfo import SingleNodeGuesser
+from guessit.patterns import episode_rexps
+import re
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_episodes_rexps")
+
+
+def guess_episodes_rexps(string):
+ for rexp, confidence, span_adjust in episode_rexps:
+ match = re.search(rexp, string, re.IGNORECASE)
+ if match:
+ return (Guess(match.groupdict(), confidence=confidence),
+ (match.start() + span_adjust[0],
+ match.end() + span_adjust[1]))
+
+ return None, None
+
+
+def process(mtree):
+ SingleNodeGuesser(guess_episodes_rexps, None, log).process(mtree)
diff --git a/libs/guessit/transfo/guess_filetype.py b/libs/guessit/transfo/guess_filetype.py
new file mode 100644
index 00000000..32bdc136
--- /dev/null
+++ b/libs/guessit/transfo/guess_filetype.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit import Guess
+from guessit.patterns import (subtitle_exts, video_exts, episode_rexps,
+ find_properties, canonical_form)
+import os.path
+import re
+import mimetypes
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_filetype")
+
+
+def guess_filetype(filename, filetype):
+ other = {}
+
+ # look at the extension first
+ fileext = os.path.splitext(filename)[1][1:].lower()
+ if fileext in subtitle_exts:
+ if 'movie' in filetype:
+ filetype = 'moviesubtitle'
+ elif 'episode' in filetype:
+ filetype = 'episodesubtitle'
+ else:
+ filetype = 'subtitle'
+ other = { 'container': fileext }
+ elif fileext in video_exts:
+ if filetype == 'autodetect':
+ filetype = 'video'
+ other = { 'container': fileext }
+ else:
+ if filetype == 'autodetect':
+ filetype = 'unknown'
+ other = { 'extension': fileext }
+
+ # put the filetype inside a dummy container to be able to have the
+ # following functions work correctly as closures
+ # this is a workaround for python 2 which doesn't have the
+ # 'nonlocal' keyword (python 3 does have it)
+ filetype_container = [filetype]
+
+ def upgrade_episode():
+ if filetype_container[0] == 'video':
+ filetype_container[0] = 'episode'
+ elif filetype_container[0] == 'subtitle':
+ filetype_container[0] = 'episodesubtitle'
+
+ def upgrade_movie():
+ if filetype_container[0] == 'video':
+ filetype_container[0] = 'movie'
+ elif filetype_container[0] == 'subtitle':
+ filetype_container[0] = 'moviesubtitle'
+
+ # now look whether there are some specific hints for episode vs movie
+ if filetype in ('video', 'subtitle'):
+ for rexp, _, _ in episode_rexps:
+ match = re.search(rexp, filename, re.IGNORECASE)
+ if match:
+ upgrade_episode()
+ break
+
+ for prop, value, _, _ in find_properties(filename):
+ log.debug('prop: %s = %s' % (prop, value))
+ if prop == 'episodeFormat':
+ upgrade_episode()
+ break
+
+ elif canonical_form(value) == 'DVB':
+ upgrade_episode()
+ break
+
+ # if no episode info found, assume it's a movie
+ upgrade_movie()
+
+ filetype = filetype_container[0]
+ return filetype, other
+
+
+def process(mtree, filetype='autodetect'):
+ filetype, other = guess_filetype(mtree.string, filetype)
+
+ mtree.guess.set('type', filetype, confidence=1.0)
+ log.debug('Found with confidence %.2f: %s' % (1.0, mtree.guess))
+
+ filetype_info = Guess(other, confidence=1.0)
+ # guess the mimetype of the filename
+ # TODO: handle other mimetypes not found on the default type_maps
+ # mimetypes.types_map['.srt']='text/subtitle'
+ mime, _ = mimetypes.guess_type(mtree.string, strict=False)
+ if mime is not None:
+ filetype_info.update({'mimetype': mime}, confidence=1.0)
+
+ node_ext = mtree.node_at((-1,))
+ node_ext.guess = filetype_info
+ log.debug('Found with confidence %.2f: %s' % (1.0, node_ext.guess))
diff --git a/libs/guessit/transfo/guess_language.py b/libs/guessit/transfo/guess_language.py
new file mode 100644
index 00000000..62f47d84
--- /dev/null
+++ b/libs/guessit/transfo/guess_language.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit import Guess
+from guessit.transfo import SingleNodeGuesser
+from guessit.language import search_language
+from guessit.textutils import clean_string
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_language")
+
+
+def guess_language(string):
+ language, span, confidence = search_language(string)
+ if language:
+ # is it a subtitle language?
+ if 'sub' in clean_string(string[:span[0]]).lower().split(' '):
+ return (Guess({'subtitleLanguage': language},
+ confidence=confidence),
+ span)
+ else:
+ return (Guess({'language': language},
+ confidence=confidence),
+ span)
+
+ return None, None
+
+
+def process(mtree):
+ SingleNodeGuesser(guess_language, None, log).process(mtree)
diff --git a/libs/guessit/transfo/guess_movie_title_from_position.py b/libs/guessit/transfo/guess_movie_title_from_position.py
new file mode 100644
index 00000000..dea56d63
--- /dev/null
+++ b/libs/guessit/transfo/guess_movie_title_from_position.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit import Guess
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_movie_title_from_position")
+
+
+def process(mtree):
+ def found_property(node, name, value, confidence):
+ node.guess = Guess({ name: value },
+ confidence=confidence)
+ log.debug('Found with confidence %.2f: %s' % (confidence, node.guess))
+
+ def found_title(node, confidence):
+ found_property(node, 'title', node.clean_value, confidence)
+
+ basename = mtree.node_at((-2,))
+ all_valid = lambda leaf: len(leaf.clean_value) > 0
+ basename_leftover = basename.unidentified_leaves(valid=all_valid)
+
+ try:
+ folder = mtree.node_at((-3,))
+ folder_leftover = folder.unidentified_leaves()
+ except ValueError:
+ folder = None
+ folder_leftover = []
+
+ log.debug('folder: %s' % folder_leftover)
+ log.debug('basename: %s' % basename_leftover)
+
+ # specific cases:
+ # if we find the same group both in the folder name and the filename,
+ # it's a good candidate for title
+ if (folder_leftover and basename_leftover and
+ folder_leftover[0].clean_value == basename_leftover[0].clean_value):
+
+ found_title(folder_leftover[0], confidence=0.8)
+ return
+
+ # specific cases:
+ # if the basename contains a number first followed by an unidentified
+ # group, and the folder only contains 1 unidentified one, then we have
+ # a series
+ # ex: Millenium Trilogy (2009)/(1)The Girl With The Dragon Tattoo(2009).mkv
+ try:
+ series = folder_leftover[0]
+ filmNumber = basename_leftover[0]
+ title = basename_leftover[1]
+
+ basename_leaves = basename.leaves()
+
+ num = int(filmNumber.clean_value)
+
+ log.debug('series: %s' % series.clean_value)
+ log.debug('title: %s' % title.clean_value)
+ if (series.clean_value != title.clean_value and
+ series.clean_value != filmNumber.clean_value and
+ basename_leaves.index(filmNumber) == 0 and
+ basename_leaves.index(title) == 1):
+
+ found_title(title, confidence=0.6)
+ found_property(series, 'filmSeries',
+ series.clean_value, confidence=0.6)
+ found_property(filmNumber, 'filmNumber',
+ num, confidence=0.6)
+ return
+ except Exception:
+ pass
+
+ # specific cases:
+ # - movies/tttttt (yyyy)/tttttt.ccc
+ try:
+ if mtree.node_at((-4, 0)).value.lower() == 'movies':
+ folder = mtree.node_at((-3,))
+
+ # Note:too generic, might solve all the unittests as they all
+ # contain 'movies' in their path
+ #
+ #if containing_folder.is_leaf() and not containing_folder.guess:
+ # containing_folder.guess =
+ # Guess({ 'title': clean_string(containing_folder.value) },
+ # confidence=0.7)
+
+ year_group = folder.first_leaf_containing('year')
+ groups_before = folder.previous_unidentified_leaves(year_group)
+
+ found_title(groups_before[0], confidence=0.8)
+ return
+
+ except Exception:
+ pass
+
+ # if we have either format or videoCodec in the folder containing the file
+ # or one of its parents, then we should probably look for the title in
+ # there rather than in the basename
+ try:
+ props = mtree.previous_leaves_containing(mtree.children[-2],
+ [ 'videoCodec', 'format',
+ 'language' ])
+ except IndexError:
+ props = []
+
+ if props:
+ group_idx = props[0].node_idx[0]
+ if all(g.node_idx[0] == group_idx for g in props):
+ # if they're all in the same group, take leftover info from there
+ leftover = mtree.node_at((group_idx,)).unidentified_leaves()
+
+ if leftover:
+ found_title(leftover[0], confidence=0.7)
+ return
+
+ # look for title in basename if there are some remaining undidentified
+ # groups there
+ if basename_leftover:
+ title_candidate = basename_leftover[0]
+
+ # if basename is only one word and the containing folder has at least
+ # 3 words in it, we should take the title from the folder name
+ # ex: Movies/Alice in Wonderland DVDRip.XviD-DiAMOND/dmd-aw.avi
+ # ex: Movies/Somewhere.2010.DVDRip.XviD-iLG/i-smwhr.avi <-- TODO: gets caught here?
+ if (title_candidate.clean_value.count(' ') == 0 and
+ folder_leftover and
+ folder_leftover[0].clean_value.count(' ') >= 2):
+
+ found_title(folder_leftover[0], confidence=0.7)
+ return
+
+ # if there are only 2 unidentified groups, the first of which is inside
+ # brackets or parentheses, we take the second one for the title:
+ # ex: Movies/[阿维达].Avida.2006.FRENCH.DVDRiP.XViD-PROD.avi
+ if len(basename_leftover) == 2 and basename_leftover[0].is_explicit():
+ found_title(basename_leftover[1], confidence=0.8)
+ return
+
+ # if all else fails, take the first remaining unidentified group in the
+ # basename as title
+ found_title(title_candidate, confidence=0.6)
+ return
+
+ # if there are no leftover groups in the basename, look in the folder name
+ if folder_leftover:
+ found_title(folder_leftover[0], confidence=0.5)
+ return
+
+ # if nothing worked, look if we have a very small group at the beginning
+ # of the basename
+ basename = mtree.node_at((-2,))
+ basename_leftover = basename.unidentified_leaves(valid=lambda leaf: True)
+ if basename_leftover:
+ found_title(basename_leftover[0], confidence=0.4)
+ return
diff --git a/libs/guessit/transfo/guess_properties.py b/libs/guessit/transfo/guess_properties.py
new file mode 100644
index 00000000..3822d22e
--- /dev/null
+++ b/libs/guessit/transfo/guess_properties.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit.transfo import SingleNodeGuesser
+from guessit.patterns import find_properties
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_properties")
+
+
+def guess_properties(string):
+ try:
+ prop, value, pos, end = find_properties(string)[0]
+ return { prop: value }, (pos, end)
+ except IndexError:
+ return None, None
+
+
+def process(mtree):
+ SingleNodeGuesser(guess_properties, 1.0, log).process(mtree)
diff --git a/libs/guessit/transfo/guess_release_group.py b/libs/guessit/transfo/guess_release_group.py
new file mode 100644
index 00000000..9ec609db
--- /dev/null
+++ b/libs/guessit/transfo/guess_release_group.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit.transfo import SingleNodeGuesser
+import re
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_release_group")
+
+
+def guess_release_group(string):
+ group_names = [ r'\.(Xvid)-(?P.*?)[ \.]',
+ r'\.(DivX)-(?P.*?)[\. ]',
+ r'\.(DVDivX)-(?P.*?)[\. ]',
+ ]
+ for rexp in group_names:
+ match = re.search(rexp, string, re.IGNORECASE)
+ if match:
+ metadata = match.groupdict()
+ metadata.update({ 'videoCodec': match.group(1) })
+ return metadata, (match.start() + 1, match.end() - 1)
+
+ return None, None
+
+
+def process(mtree):
+ SingleNodeGuesser(guess_release_group, 0.8, log).process(mtree)
diff --git a/libs/guessit/transfo/guess_video_rexps.py b/libs/guessit/transfo/guess_video_rexps.py
new file mode 100644
index 00000000..36723c82
--- /dev/null
+++ b/libs/guessit/transfo/guess_video_rexps.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit import Guess
+from guessit.transfo import SingleNodeGuesser
+from guessit.patterns import video_rexps, sep
+import re
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_video_rexps")
+
+
+def guess_video_rexps(string):
+ string = '-' + string + '-'
+ for rexp, confidence, span_adjust in video_rexps:
+ match = re.search(sep + rexp + sep, string, re.IGNORECASE)
+ if match:
+ metadata = match.groupdict()
+ # is this the better place to put it? (maybe, as it is at least
+ # the soonest that we can catch it)
+ if metadata.get('cdNumberTotal', -1) is None:
+ del metadata['cdNumberTotal']
+ return (Guess(metadata, confidence=confidence),
+ (match.start() + span_adjust[0],
+ match.end() + span_adjust[1] - 2))
+
+ return None, None
+
+
+def process(mtree):
+ SingleNodeGuesser(guess_video_rexps, None, log).process(mtree)
diff --git a/libs/guessit/transfo/guess_weak_episodes_rexps.py b/libs/guessit/transfo/guess_weak_episodes_rexps.py
new file mode 100644
index 00000000..8fffe17f
--- /dev/null
+++ b/libs/guessit/transfo/guess_weak_episodes_rexps.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit import Guess
+from guessit.transfo import SingleNodeGuesser
+from guessit.patterns import weak_episode_rexps
+import re
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_weak_episodes_rexps")
+
+
+def guess_weak_episodes_rexps(string, node):
+ if 'episodeNumber' in node.root.info:
+ return None, None
+
+ for rexp, span_adjust in weak_episode_rexps:
+ match = re.search(rexp, string, re.IGNORECASE)
+ if match:
+ metadata = match.groupdict()
+ span = (match.start() + span_adjust[0],
+ match.end() + span_adjust[1])
+
+ epnum = int(metadata['episodeNumber'])
+ if epnum > 100:
+ return Guess({ 'season': epnum // 100,
+ 'episodeNumber': epnum % 100 },
+ confidence=0.6), span
+ else:
+ return Guess(metadata, confidence=0.3), span
+
+ return None, None
+
+
+guess_weak_episodes_rexps.use_node = True
+
+
+def process(mtree):
+ SingleNodeGuesser(guess_weak_episodes_rexps, 0.6, log).process(mtree)
diff --git a/libs/guessit/transfo/guess_website.py b/libs/guessit/transfo/guess_website.py
new file mode 100644
index 00000000..a169f97b
--- /dev/null
+++ b/libs/guessit/transfo/guess_website.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit.transfo import SingleNodeGuesser
+from guessit.patterns import websites
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_website")
+
+
+def guess_website(string):
+ low = string.lower()
+ for site in websites:
+ pos = low.find(site.lower())
+ if pos != -1:
+ return {'website': site}, (pos, pos + len(site))
+ return None, None
+
+
+def process(mtree):
+ SingleNodeGuesser(guess_website, 1.0, log).process(mtree)
diff --git a/libs/guessit/transfo/guess_year.py b/libs/guessit/transfo/guess_year.py
new file mode 100644
index 00000000..7a47ecf7
--- /dev/null
+++ b/libs/guessit/transfo/guess_year.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit.transfo import SingleNodeGuesser
+from guessit.date import search_year
+import logging
+
+log = logging.getLogger("guessit.transfo.guess_year")
+
+
+def guess_year(string):
+ year, span = search_year(string)
+ if year:
+ return { 'year': year }, span
+ else:
+ return None, None
+
+
+def process(mtree):
+ SingleNodeGuesser(guess_year, 1.0, log).process(mtree)
diff --git a/libs/guessit/transfo/post_process.py b/libs/guessit/transfo/post_process.py
new file mode 100644
index 00000000..0b5a4dfb
--- /dev/null
+++ b/libs/guessit/transfo/post_process.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit.patterns import subtitle_exts
+import logging
+
+log = logging.getLogger("guessit.transfo.post_process")
+
+
+def process(mtree):
+ # 1- try to promote language to subtitle language where it makes sense
+ for node in mtree.nodes():
+ if 'language' not in node.guess:
+ continue
+
+ def promote_subtitle():
+ # pylint: disable=W0631
+ node.guess.set('subtitleLanguage', node.guess['language'],
+ confidence=node.guess.confidence('language'))
+ del node.guess['language']
+
+ # - if we matched a language in a file with a sub extension and that
+ # the group is the last group of the filename, it is probably the
+ # language of the subtitle
+ # (eg: 'xxx.english.srt')
+ if (mtree.node_at((-1,)).value.lower() in subtitle_exts and
+ node == mtree.leaves()[-2]):
+ promote_subtitle()
+
+ # - if a language is in an explicit group just preceded by "st",
+ # it is a subtitle language (eg: '...st[fr-eng]...')
+ try:
+ idx = node.node_idx
+ previous = mtree.node_at((idx[0], idx[1] - 1)).leaves()[-1]
+ if previous.value.lower()[-2:] == 'st':
+ promote_subtitle()
+ except IndexError:
+ pass
+
+ # 2- ", the" at the end of a series title should be prepended to it
+ for node in mtree.nodes():
+ if 'series' not in node.guess:
+ continue
+
+ series = node.guess['series']
+ lseries = series.lower()
+
+ if lseries[-4:] == ',the':
+ node.guess['series'] = 'The ' + series[:-4]
+
+ if lseries[-5:] == ', the':
+ node.guess['series'] = 'The ' + series[:-5]
diff --git a/libs/guessit/transfo/split_explicit_groups.py b/libs/guessit/transfo/split_explicit_groups.py
new file mode 100644
index 00000000..797a886c
--- /dev/null
+++ b/libs/guessit/transfo/split_explicit_groups.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit.textutils import find_first_level_groups
+from guessit.patterns import group_delimiters
+import logging
+
+log = logging.getLogger("guessit.transfo.split_explicit_groups")
+
+
+def process(mtree):
+ """return the string split into explicit groups, that is, those either
+ between parenthese, square brackets or curly braces, and those separated
+ by a dash."""
+ for c in mtree.children:
+ groups = find_first_level_groups(c.value, group_delimiters[0])
+ for delimiters in group_delimiters:
+ flatten = lambda l, x: l + find_first_level_groups(x, delimiters)
+ groups = reduce(flatten, groups, [])
+
+ # do not do this at this moment, it is not strong enough and can break other
+ # patterns, such as dates, etc...
+ #groups = reduce(lambda l, x: l + x.split('-'), groups, [])
+
+ c.split_on_components(groups)
diff --git a/libs/guessit/transfo/split_on_dash.py b/libs/guessit/transfo/split_on_dash.py
new file mode 100644
index 00000000..fc10c49d
--- /dev/null
+++ b/libs/guessit/transfo/split_on_dash.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit.patterns import sep
+import re
+import logging
+
+log = logging.getLogger("guessit.transfo.split_on_dash")
+
+
+def process(mtree):
+ for node in mtree.unidentified_leaves():
+ indices = []
+
+ didx = 0
+ pattern = re.compile(sep + '-' + sep)
+ match = pattern.search(node.value)
+ while match:
+ span = match.span()
+ indices.extend([ span[0], span[1] ])
+ match = pattern.search(node.value, span[1])
+
+ didx = node.value.find('-')
+ while didx > 0:
+ if (didx > 10 and
+ (didx - 1 not in indices and
+ didx + 2 not in indices)):
+
+ indices.extend([ didx, didx + 1 ])
+
+ didx = node.value.find('-', didx + 1)
+
+ if indices:
+ node.partition(indices)
diff --git a/libs/guessit/transfo/split_path_components.py b/libs/guessit/transfo/split_path_components.py
new file mode 100644
index 00000000..0f8d1a5a
--- /dev/null
+++ b/libs/guessit/transfo/split_path_components.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# GuessIt - A library for guessing information from filenames
+# Copyright (c) 2012 Nicolas Wack
+#
+# GuessIt is free software; you can redistribute it and/or modify it under
+# the terms of the Lesser GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# GuessIt is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# Lesser GNU General Public License for more details.
+#
+# You should have received a copy of the Lesser GNU General Public License
+# along with this program. If not, see .
+#
+
+from guessit import fileutils
+import os.path
+import logging
+
+log = logging.getLogger("guessit.transfo.split_path_components")
+
+
+def process(mtree):
+ """Returns the filename split into [ dir*, basename, ext ]."""
+ components = fileutils.split_path(mtree.value)
+ basename = components.pop(-1)
+ components += list(os.path.splitext(basename))
+ components[-1] = components[-1][1:] # remove the '.' from the extension
+
+ mtree.split_on_components(components)
diff --git a/libs/requests/__init__.py b/libs/requests/__init__.py
index 48fb3899..0ace12b6 100644
--- a/libs/requests/__init__.py
+++ b/libs/requests/__init__.py
@@ -15,14 +15,13 @@ requests
"""
__title__ = 'requests'
-__version__ = '0.10.1'
-__build__ = 0x001001
+__version__ = '0.11.1'
+__build__ = 0x001101
__author__ = 'Kenneth Reitz'
__license__ = 'ISC'
__copyright__ = 'Copyright 2012 Kenneth Reitz'
-
from . import utils
from .models import Request, Response
from .api import request, get, head, post, patch, put, delete, options
diff --git a/libs/requests/api.py b/libs/requests/api.py
index b7d4158d..e0bf346c 100644
--- a/libs/requests/api.py
+++ b/libs/requests/api.py
@@ -33,6 +33,7 @@ def request(method, url, **kwargs):
:param config: (optional) A configuration dictionary.
:param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
:param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
+ :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
"""
s = kwargs.pop('session') if 'session' in kwargs else sessions.session()
@@ -44,7 +45,7 @@ def get(url, **kwargs):
"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
@@ -55,7 +56,7 @@ def options(url, **kwargs):
"""Sends a OPTIONS request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
@@ -66,10 +67,10 @@ def head(url, **kwargs):
"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
- kwargs.setdefault('allow_redirects', True)
+ kwargs.setdefault('allow_redirects', False)
return request('head', url, **kwargs)
@@ -78,7 +79,7 @@ def post(url, data=None, **kwargs):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return request('post', url, data=data, **kwargs)
@@ -89,7 +90,7 @@ def put(url, data=None, **kwargs):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return request('put', url, data=data, **kwargs)
@@ -100,7 +101,7 @@ def patch(url, data=None, **kwargs):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return request('patch', url, data=data, **kwargs)
@@ -110,7 +111,7 @@ def delete(url, **kwargs):
"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return request('delete', url, **kwargs)
diff --git a/libs/requests/async.py b/libs/requests/async.py
index 94884478..f12cf268 100644
--- a/libs/requests/async.py
+++ b/libs/requests/async.py
@@ -17,13 +17,13 @@ except ImportError:
raise RuntimeError('Gevent is required for requests.async.')
# Monkey-patch.
-curious_george.patch_all(thread=False)
+curious_george.patch_all(thread=False, select=False)
from . import api
__all__ = (
- 'map',
+ 'map', 'imap',
'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request'
)
@@ -46,15 +46,15 @@ def patched(f):
return wrapped
-def send(r, pool=None):
- """Sends the request object using the specified pool. If a pool isn't
+def send(r, pool=None, prefetch=False):
+ """Sends the request object using the specified pool. If a pool isn't
specified this method blocks. Pools are useful because you can specify size
and can hence limit concurrency."""
if pool != None:
- return pool.spawn(r.send)
+ return pool.spawn(r.send, prefetch=prefetch)
- return gevent.spawn(r.send)
+ return gevent.spawn(r.send, prefetch=prefetch)
# Patched requests.api functions.
@@ -79,10 +79,28 @@ def map(requests, prefetch=True, size=None):
requests = list(requests)
pool = Pool(size) if size else None
- jobs = [send(r, pool) for r in requests]
+ jobs = [send(r, pool, prefetch=prefetch) for r in requests]
gevent.joinall(jobs)
- if prefetch:
- [r.response.content for r in requests]
+ return [r.response for r in requests]
- return [r.response for r in requests]
\ No newline at end of file
+
+def imap(requests, prefetch=True, size=2):
+ """Concurrently converts a generator object of Requests to
+ a generator of Responses.
+
+ :param requests: a generator of Request objects.
+ :param prefetch: If False, the content will not be downloaded immediately.
+ :param size: Specifies the number of requests to make at a time. default is 2
+ """
+
+ pool = Pool(size)
+
+ def send(r):
+ r.send(prefetch)
+ return r.response
+
+ for r in pool.imap_unordered(send, requests):
+ yield r
+
+ pool.join()
\ No newline at end of file
diff --git a/libs/requests/auth.py b/libs/requests/auth.py
index 183731bd..385dd27d 100644
--- a/libs/requests/auth.py
+++ b/libs/requests/auth.py
@@ -7,13 +7,11 @@ requests.auth
This module contains the authentication handlers for Requests.
"""
-from __future__ import unicode_literals
-
import time
import hashlib
from base64 import b64encode
-from .compat import urlparse, str, bytes
+from .compat import urlparse, str
from .utils import randombytes, parse_dict_header
@@ -21,7 +19,7 @@ from .utils import randombytes, parse_dict_header
def _basic_auth_str(username, password):
"""Returns a Basic Auth string."""
- return 'Basic ' + b64encode(("%s:%s" % (username, password)).encode('utf-8')).strip().decode('utf-8')
+ return 'Basic ' + b64encode(('%s:%s' % (username, password)).encode('latin1')).strip().decode('latin1')
class AuthBase(object):
diff --git a/libs/requests/compat.py b/libs/requests/compat.py
index 224bfd02..fec7a01d 100644
--- a/libs/requests/compat.py
+++ b/libs/requests/compat.py
@@ -86,8 +86,10 @@ if is_py2:
from .packages.oreos.monkeys import SimpleCookie
from StringIO import StringIO
- str = unicode
bytes = str
+ str = unicode
+ basestring = basestring
+
elif is_py3:
@@ -99,4 +101,5 @@ elif is_py3:
str = str
bytes = bytes
+ basestring = (str,bytes)
diff --git a/libs/requests/defaults.py b/libs/requests/defaults.py
index 424d3731..6a7ea270 100644
--- a/libs/requests/defaults.py
+++ b/libs/requests/defaults.py
@@ -15,10 +15,16 @@ Configurations:
:max_retries: The number of times a request should be retried in the event of a connection failure.
:danger_mode: If true, Requests will raise errors immediately.
:safe_mode: If true, Requests will catch all errors.
+:strict_mode: If true, Requests will do its best to follow RFCs (e.g. POST redirects).
:pool_maxsize: The maximium size of an HTTP connection pool.
:pool_connections: The number of active HTTP connection pools to use.
+:encode_uri: If true, URIs will automatically be percent-encoded.
+:trust_env: If true, the surrouding environment will be trusted (environ, netrc).
+
"""
+SCHEMAS = ['http', 'https']
+
from . import __version__
defaults = dict()
@@ -37,4 +43,9 @@ defaults['pool_maxsize'] = 10
defaults['max_retries'] = 0
defaults['danger_mode'] = False
defaults['safe_mode'] = False
+defaults['strict_mode'] = False
defaults['keep_alive'] = True
+defaults['encode_uri'] = True
+defaults['trust_env'] = True
+
+
diff --git a/libs/requests/exceptions.py b/libs/requests/exceptions.py
index c7b98e6f..3c262e36 100644
--- a/libs/requests/exceptions.py
+++ b/libs/requests/exceptions.py
@@ -8,12 +8,13 @@ This module contains the set of Requests' exceptions.
"""
-class RequestException(Exception):
+class RequestException(RuntimeError):
"""There was an ambiguous exception that occurred while handling your
request."""
class HTTPError(RequestException):
"""An HTTP error occurred."""
+ response = None
class ConnectionError(RequestException):
"""A Connection error occurred."""
@@ -29,3 +30,9 @@ class URLRequired(RequestException):
class TooManyRedirects(RequestException):
"""Too many redirects."""
+
+class MissingSchema(RequestException, ValueError):
+ """The URL schema (e.g. http or https) is missing."""
+
+class InvalidSchema(RequestException, ValueError):
+ """See defaults.py for valid schemas."""
\ No newline at end of file
diff --git a/libs/requests/models.py b/libs/requests/models.py
index c200896c..70e35036 100644
--- a/libs/requests/models.py
+++ b/libs/requests/models.py
@@ -21,14 +21,16 @@ from .packages.urllib3.exceptions import SSLError as _SSLError
from .packages.urllib3.exceptions import HTTPError as _HTTPError
from .packages.urllib3 import connectionpool, poolmanager
from .packages.urllib3.filepost import encode_multipart_formdata
+from .defaults import SCHEMAS
from .exceptions import (
ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects,
- URLRequired, SSLError)
+ URLRequired, SSLError, MissingSchema, InvalidSchema)
from .utils import (
- get_encoding_from_headers, stream_decode_response_unicode,
- stream_decompress, guess_filename, requote_path, dict_from_string)
-
-from .compat import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, str, bytes, SimpleCookie, is_py3, is_py2
+ get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri,
+ dict_from_string, stream_decode_response_unicode, get_netrc_auth)
+from .compat import (
+ urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,
+ SimpleCookie, is_py2)
# Import chardet if it is available.
try:
@@ -39,7 +41,6 @@ except ImportError:
REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
-
class Request(object):
"""The :class:`Request ` object. It carries out all functionality of
Requests. Recommended interface is with the Requests functions.
@@ -62,18 +63,17 @@ class Request(object):
config=None,
_poolmanager=None,
verify=None,
- session=None):
+ session=None,
+ cert=None):
+
+ #: Dictionary of configurations for this request.
+ self.config = dict(config or [])
#: Float describes the timeout of the request.
# (Use socket.setdefaulttimeout() as fallback)
self.timeout = timeout
#: Request URL.
-
- # if isinstance(url, str):
- # url = url.encode('utf-8')
- # print(dir(url))
-
self.url = url
#: Dictionary of HTTP Headers to attach to the :class:`Request `.
@@ -103,6 +103,14 @@ class Request(object):
# Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
self.proxies = dict(proxies or [])
+ # If no proxies are given, allow configuration by environment variables
+ # HTTP_PROXY and HTTPS_PROXY.
+ if not self.proxies and self.config.get('trust_env'):
+ if 'HTTP_PROXY' in os.environ:
+ self.proxies['http'] = os.environ['HTTP_PROXY']
+ if 'HTTPS_PROXY' in os.environ:
+ self.proxies['https'] = os.environ['HTTPS_PROXY']
+
self.data, self._enc_data = self._encode_params(data)
self.params, self._enc_params = self._encode_params(params)
@@ -116,9 +124,6 @@ class Request(object):
#: CookieJar to attach to :class:`Request `.
self.cookies = dict(cookies or [])
- #: Dictionary of configurations for this request.
- self.config = dict(config or [])
-
#: True if Request has been sent.
self.sent = False
@@ -139,6 +144,9 @@ class Request(object):
#: SSL Verification.
self.verify = verify
+ #: SSL Certificate
+ self.cert = cert
+
if headers:
headers = CaseInsensitiveDict(self.headers)
else:
@@ -152,15 +160,9 @@ class Request(object):
self.headers = headers
self._poolmanager = _poolmanager
- # Pre-request hook.
- r = dispatch_hook('pre_request', hooks, self)
- self.__dict__.update(r.__dict__)
-
-
def __repr__(self):
return '' % (self.method)
-
def _build_response(self, resp):
"""Build internal :class:`Response ` object
from given response.
@@ -200,29 +202,36 @@ class Request(object):
# Save original response for later.
response.raw = resp
- response.url = self.full_url
+ if isinstance(self.full_url, bytes):
+ response.url = self.full_url.decode('utf-8')
+ else:
+ response.url = self.full_url
return response
history = []
r = build(resp)
- cookies = self.cookies
+
self.cookies.update(r.cookies)
if r.status_code in REDIRECT_STATI and not self.redirect:
- while (
- ('location' in r.headers) and
- ((r.status_code is codes.see_other) or (self.allow_redirects))
- ):
+ while (('location' in r.headers) and
+ ((r.status_code is codes.see_other) or (self.allow_redirects))):
+
+ r.content # Consume socket so it can be released
if not len(history) < self.config.get('max_redirects'):
raise TooManyRedirects()
+ # Release the connection back into the pool.
+ r.raw.release_conn()
+
history.append(r)
url = r.headers['location']
+ data = self.data
# Handle redirection without scheme (see: RFC 1808 Section 4)
if url.startswith('//'):
@@ -232,14 +241,29 @@ class Request(object):
# Facilitate non-RFC2616-compliant 'location' headers
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
if not urlparse(url).netloc:
- url = urljoin(r.url, url)
+ url = urljoin(r.url,
+ # Compliant with RFC3986, we percent
+ # encode the url.
+ requote_uri(url))
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
if r.status_code is codes.see_other:
method = 'GET'
+ data = None
else:
method = self.method
+ # Do what the browsers do if strict_mode is off...
+ if (not self.config.get('strict_mode')):
+
+ if r.status_code in (codes.moved, codes.found) and self.method == 'POST':
+ method = 'GET'
+ data = None
+
+ if (r.status_code == 303) and self.method != 'HEAD':
+ method = 'GET'
+ data = None
+
# Remove the cookie headers that were sent.
headers = self.headers
try:
@@ -254,18 +278,19 @@ class Request(object):
method=method,
params=self.session.params,
auth=self.auth,
- cookies=cookies,
+ cookies=self.cookies,
redirect=True,
+ data=data,
config=self.config,
timeout=self.timeout,
_poolmanager=self._poolmanager,
- proxies = self.proxies,
- verify = self.verify,
- session = self.session
+ proxies=self.proxies,
+ verify=self.verify,
+ session=self.session,
+ cert=self.cert
)
request.send()
- cookies.update(request.response.cookies)
r = request.response
self.cookies.update(r.cookies)
@@ -275,7 +300,6 @@ class Request(object):
self.response.request = self
self.response.cookies.update(self.cookies)
-
@staticmethod
def _encode_params(data):
"""Encode parameters in a piece of data.
@@ -288,10 +312,12 @@ class Request(object):
returns it twice.
"""
+ if isinstance(data, bytes):
+ return data, data
+
if hasattr(data, '__iter__') and not isinstance(data, str):
data = dict(data)
-
if hasattr(data, 'items'):
result = []
for k, vs in list(data.items()):
@@ -314,30 +340,44 @@ class Request(object):
# Support for unicode domain names and paths.
scheme, netloc, path, params, query, fragment = urlparse(url)
-
if not scheme:
- raise ValueError("Invalid URL %r: No schema supplied" % url)
+ raise MissingSchema("Invalid URL %r: No schema supplied" % url)
+
+ if not scheme in SCHEMAS:
+ raise InvalidSchema("Invalid scheme %r" % scheme)
netloc = netloc.encode('idna').decode('utf-8')
+ if not path:
+ path = '/'
+
+
if is_py2:
+ if isinstance(scheme, str):
+ scheme = scheme.encode('utf-8')
+ if isinstance(netloc, str):
+ netloc = netloc.encode('utf-8')
if isinstance(path, str):
path = path.encode('utf-8')
+ if isinstance(params, str):
+ params = params.encode('utf-8')
+ if isinstance(query, str):
+ query = query.encode('utf-8')
+ if isinstance(fragment, str):
+ fragment = fragment.encode('utf-8')
- path = requote_path(path)
-
- # print([ scheme, netloc, path, params, query, fragment ])
- # print('---------------------')
-
- url = (urlunparse([ scheme, netloc, path, params, query, fragment ]))
+ url = (urlunparse([scheme, netloc, path, params, query, fragment]))
if self._enc_params:
if urlparse(url).query:
- return '%s&%s' % (url, self._enc_params)
+ url = '%s&%s' % (url, self._enc_params)
else:
- return '%s?%s' % (url, self._enc_params)
- else:
- return url
+ url = '%s?%s' % (url, self._enc_params)
+
+ if self.config.get('encode_uri', True):
+ url = requote_uri(url)
+
+ return url
@property
def path_url(self):
@@ -355,9 +395,6 @@ class Request(object):
if not path:
path = '/'
- # if is_py3:
- path = quote(path.encode('utf-8'))
-
url.append(path)
query = p.query
@@ -365,19 +402,15 @@ class Request(object):
url.append('?')
url.append(query)
- # print(url)
-
return ''.join(url)
-
def register_hook(self, event, hook):
"""Properly register a hook."""
return self.hooks[event].append(hook)
-
def send(self, anyway=False, prefetch=False):
- """Sends the request. Returns True of successful, false if not.
+ """Sends the request. Returns True of successful, False if not.
If there was an HTTPError during transmission,
self.response.status_code will contain the HTTPError code.
@@ -435,6 +468,10 @@ class Request(object):
if (content_type) and (not 'content-type' in self.headers):
self.headers['Content-Type'] = content_type
+ # Use .netrc auth if none was provided.
+ if not self.auth and self.config.get('trust_env'):
+ self.auth = get_netrc_auth(url)
+
if self.auth:
if isinstance(self.auth, tuple) and len(self.auth) == 2:
# special-case basic HTTP auth
@@ -472,13 +509,12 @@ class Request(object):
if self.verify is not True:
cert_loc = self.verify
-
# Look for configuration.
- if not cert_loc:
+ if not cert_loc and self.config.get('trust_env'):
cert_loc = os.environ.get('REQUESTS_CA_BUNDLE')
# Curl compatiblity.
- if not cert_loc:
+ if not cert_loc and self.config.get('trust_env'):
cert_loc = os.environ.get('CURL_CA_BUNDLE')
# Use the awesome certifi list.
@@ -491,6 +527,13 @@ class Request(object):
conn.cert_reqs = 'CERT_NONE'
conn.ca_certs = None
+ if self.cert and self.verify:
+ if len(self.cert) == 2:
+ conn.cert_file = self.cert[0]
+ conn.key_file = self.cert[1]
+ else:
+ conn.cert_file = self.cert
+
if not self.sent or anyway:
if self.cookies:
@@ -509,6 +552,10 @@ class Request(object):
# Attach Cookie header to request.
self.headers['Cookie'] = cookie_header
+ # Pre-request hook.
+ r = dispatch_hook('pre_request', self.hooks, self)
+ self.__dict__.update(r.__dict__)
+
try:
# The inner try .. except re-raises certain exceptions as
# internal exception types; the outer suppresses exceptions
@@ -523,7 +570,7 @@ class Request(object):
redirect=False,
assert_same_host=False,
preload_content=False,
- decode_content=True,
+ decode_content=False,
retries=self.config.get('max_retries', 0),
timeout=self.timeout,
)
@@ -613,7 +660,6 @@ class Response(object):
#: Dictionary of configurations for this request.
self.config = {}
-
def __repr__(self):
return '' % (self.status_code)
@@ -629,11 +675,10 @@ class Response(object):
def ok(self):
try:
self.raise_for_status()
- except HTTPError:
+ except RequestException:
return False
return True
-
def iter_content(self, chunk_size=10 * 1024, decode_unicode=False):
"""Iterates over the response data. This avoids reading the content
at once into memory for large responses. The chunk size is the number
@@ -653,81 +698,39 @@ class Response(object):
yield chunk
self._content_consumed = True
- def generate_chunked():
- resp = self.raw._original_response
- fp = resp.fp
- if resp.chunk_left is not None:
- pending_bytes = resp.chunk_left
- while pending_bytes:
- chunk = fp.read(min(chunk_size, pending_bytes))
- pending_bytes-=len(chunk)
- yield chunk
- fp.read(2) # throw away crlf
- while 1:
- #XXX correct line size? (httplib has 64kb, seems insane)
- pending_bytes = fp.readline(40).strip()
- pending_bytes = int(pending_bytes, 16)
- if pending_bytes == 0:
- break
- while pending_bytes:
- chunk = fp.read(min(chunk_size, pending_bytes))
- pending_bytes-=len(chunk)
- yield chunk
- fp.read(2) # throw away crlf
- self._content_consumed = True
- fp.close()
-
-
- if getattr(getattr(self.raw, '_original_response', None), 'chunked', False):
- gen = generate_chunked()
- else:
- gen = generate()
-
- if 'gzip' in self.headers.get('content-encoding', ''):
- gen = stream_decompress(gen, mode='gzip')
- elif 'deflate' in self.headers.get('content-encoding', ''):
- gen = stream_decompress(gen, mode='deflate')
+ gen = stream_untransfer(generate(), self)
if decode_unicode:
gen = stream_decode_response_unicode(gen, self)
return gen
-
def iter_lines(self, chunk_size=10 * 1024, decode_unicode=None):
"""Iterates over the response data, one line at a time. This
avoids reading the content at once into memory for large
responses.
"""
- #TODO: why rstrip by default
pending = None
- for chunk in self.iter_content(chunk_size, decode_unicode=decode_unicode):
+ for chunk in self.iter_content(
+ chunk_size=chunk_size,
+ decode_unicode=decode_unicode):
if pending is not None:
chunk = pending + chunk
- lines = chunk.splitlines(True)
+ lines = chunk.splitlines()
- for line in lines[:-1]:
- yield line.rstrip()
-
- # Save the last part of the chunk for next iteration, to keep full line together
- # lines may be empty for the last chunk of a chunked response
-
- if lines:
- pending = lines[-1]
- #if pending is a complete line, give it baack
- if pending[-1] == '\n':
- yield pending.rstrip()
- pending = None
+ if lines[-1][-1] == chunk[-1]:
+ pending = lines.pop()
else:
pending = None
- # Yield the last line
- if pending is not None:
- yield pending.rstrip()
+ for line in lines:
+ yield line
+ if pending is not None:
+ yield pending
@property
def content(self):
@@ -740,13 +743,26 @@ class Response(object):
raise RuntimeError(
'The content for this response was already consumed')
- self._content = self.raw.read()
+ if self.status_code is 0:
+ self._content = None
+ else:
+ self._content = bytes().join(self.iter_content()) or bytes()
+
except AttributeError:
self._content = None
self._content_consumed = True
return self._content
+ def _detected_encoding(self):
+ try:
+ detected = chardet.detect(self.content) or {}
+ return detected.get('encoding')
+
+ # Trust that chardet isn't available or something went terribly wrong.
+ except Exception:
+ pass
+
@property
def text(self):
@@ -762,43 +778,40 @@ class Response(object):
# Fallback to auto-detected encoding if chardet is available.
if self.encoding is None:
- try:
- detected = chardet.detect(self.content) or {}
- encoding = detected.get('encoding')
-
- # Trust that chardet isn't available or something went terribly wrong.
- except Exception:
- pass
+ encoding = self._detected_encoding()
# Decode unicode from given encoding.
try:
- content = str(self.content, encoding)
+ content = str(self.content, encoding, errors='replace')
+ except LookupError:
+ # A LookupError is raised if the encoding was not found which could
+ # indicate a misspelling or similar mistake.
+ #
+ # So we try blindly encoding.
+ content = str(self.content, errors='replace')
except (UnicodeError, TypeError):
pass
- # Try to fall back:
- if not content:
- try:
- content = str(content, encoding, errors='replace')
- except (UnicodeError, TypeError):
- pass
-
return content
-
- def raise_for_status(self):
+ def raise_for_status(self, allow_redirects=True):
"""Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred."""
if self.error:
raise self.error
- if (self.status_code >= 300) and (self.status_code < 400):
- raise HTTPError('%s Redirection' % self.status_code)
+ if (self.status_code >= 300) and (self.status_code < 400) and not allow_redirects:
+ http_error = HTTPError('%s Redirection' % self.status_code)
+ http_error.response = self
+ raise http_error
elif (self.status_code >= 400) and (self.status_code < 500):
- raise HTTPError('%s Client Error' % self.status_code)
+ http_error = HTTPError('%s Client Error' % self.status_code)
+ http_error.response = self
+ raise http_error
+
elif (self.status_code >= 500) and (self.status_code < 600):
- raise HTTPError('%s Server Error' % self.status_code)
-
-
+ http_error = HTTPError('%s Server Error' % self.status_code)
+ http_error.response = self
+ raise http_error
diff --git a/libs/requests/packages/oreos/monkeys.py b/libs/requests/packages/oreos/monkeys.py
index 2269e30b..2cf90163 100644
--- a/libs/requests/packages/oreos/monkeys.py
+++ b/libs/requests/packages/oreos/monkeys.py
@@ -249,10 +249,13 @@ class CookieError(Exception):
# quoted with a preceeding '\' slash.
#
# These are taken from RFC2068 and RFC2109.
+# _RFC2965Forbidden is the list of forbidden chars we accept anyway
# _LegalChars is the list of chars which don't require "'s
# _Translator hash-table for fast quoting
#
-_LegalChars = string.ascii_letters + string.digits + "!#$%&'*+-.^_`|~[]_"
+_RFC2965Forbidden = "[]:{}="
+_LegalChars = ( string.ascii_letters + string.digits +
+ "!#$%&'*+-.^_`|~_@" + _RFC2965Forbidden )
_Translator = {
'\000' : '\\000', '\001' : '\\001', '\002' : '\\002',
'\003' : '\\003', '\004' : '\\004', '\005' : '\\005',
diff --git a/libs/requests/packages/oreos/structures.py b/libs/requests/packages/oreos/structures.py
index 063d5f96..83292777 100644
--- a/libs/requests/packages/oreos/structures.py
+++ b/libs/requests/packages/oreos/structures.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
"""
-oreos.sructures
-~~~~~~~~~~~~~~~
+oreos.structures
+~~~~~~~~~~~~~~~~
The plastic blue packaging.
@@ -362,7 +362,7 @@ class MultiDict(TypeConversionDict):
"""
try:
return dict.pop(self, key)[0]
- except KeyError, e:
+ except KeyError as e:
if default is not _missing:
return default
raise KeyError(str(e))
@@ -372,7 +372,7 @@ class MultiDict(TypeConversionDict):
try:
item = dict.popitem(self)
return (item[0], item[1][0])
- except KeyError, e:
+ except KeyError as e:
raise KeyError(str(e))
def poplist(self, key):
@@ -389,7 +389,7 @@ class MultiDict(TypeConversionDict):
"""Pop a ``(key, list)`` tuple from the dict."""
try:
return dict.popitem(self)
- except KeyError, e:
+ except KeyError as e:
raise KeyError(str(e))
def __copy__(self):
diff --git a/libs/requests/packages/urllib3/__init__.py b/libs/requests/packages/urllib3/__init__.py
index 5f70c563..2d6fecec 100644
--- a/libs/requests/packages/urllib3/__init__.py
+++ b/libs/requests/packages/urllib3/__init__.py
@@ -10,26 +10,20 @@ urllib3 - Thread-safe connection pooling and re-using.
__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
__license__ = 'MIT'
-__version__ = '1.1'
+__version__ = '1.3'
from .connectionpool import (
HTTPConnectionPool,
HTTPSConnectionPool,
- connection_from_url,
- get_host,
- make_headers)
-
-
-from .exceptions import (
- HTTPError,
- MaxRetryError,
- SSLError,
- TimeoutError)
+ connection_from_url
+)
+from . import exceptions
+from .filepost import encode_multipart_formdata
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
from .response import HTTPResponse
-from .filepost import encode_multipart_formdata
+from .util import make_headers, get_host
# Set default logging handler to avoid "No handler found" warnings.
diff --git a/libs/requests/packages/urllib3/connectionpool.py b/libs/requests/packages/urllib3/connectionpool.py
index 52b1802b..c3cb3b12 100644
--- a/libs/requests/packages/urllib3/connectionpool.py
+++ b/libs/requests/packages/urllib3/connectionpool.py
@@ -9,45 +9,51 @@ import socket
from socket import error as SocketError, timeout as SocketTimeout
-try:
- from select import poll, POLLIN
-except ImportError: # Doesn't exist on OSX and other platforms
- from select import select
- poll = False
-
try: # Python 3
- from http.client import HTTPConnection, HTTPSConnection, HTTPException
+ from http.client import HTTPConnection, HTTPException
from http.client import HTTP_PORT, HTTPS_PORT
except ImportError:
- from httplib import HTTPConnection, HTTPSConnection, HTTPException
+ from httplib import HTTPConnection, HTTPException
from httplib import HTTP_PORT, HTTPS_PORT
try: # Python 3
- from queue import Queue, Empty, Full
+ from queue import LifoQueue, Empty, Full
except ImportError:
- from Queue import Queue, Empty, Full
+ from Queue import LifoQueue, Empty, Full
+
try: # Compiled with SSL?
+ HTTPSConnection = object
+ BaseSSLError = None
+ ssl = None
+
+ try: # Python 3
+ from http.client import HTTPSConnection
+ except ImportError:
+ from httplib import HTTPSConnection
+
import ssl
BaseSSLError = ssl.SSLError
-except ImportError:
- ssl = None
- BaseSSLError = None
+
+except (ImportError, AttributeError):
+ pass
-from .packages.ssl_match_hostname import match_hostname, CertificateError
from .request import RequestMethods
from .response import HTTPResponse
-from .exceptions import (SSLError,
- MaxRetryError,
- TimeoutError,
- HostChangedError,
+from .util import get_host, is_connection_dropped
+from .exceptions import (
EmptyPoolError,
+ HostChangedError,
+ MaxRetryError,
+ SSLError,
+ TimeoutError,
)
from .packages.ssl_match_hostname import match_hostname, CertificateError
from .packages import six
+
xrange = six.moves.xrange
log = logging.getLogger(__name__)
@@ -59,6 +65,7 @@ port_by_scheme = {
'https': HTTPS_PORT,
}
+
## Connection objects (extension of httplib)
class VerifiedHTTPSConnection(HTTPSConnection):
@@ -94,6 +101,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
if self.ca_certs:
match_hostname(self.sock.getpeercert(), self.host)
+
## Pool objects
class ConnectionPool(object):
@@ -103,6 +111,7 @@ class ConnectionPool(object):
"""
scheme = None
+ QueueCls = LifoQueue
def __init__(self, host, port=None):
self.host = host
@@ -156,11 +165,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1,
block=False, headers=None):
- self.host = host
- self.port = port
+ super(HTTPConnectionPool, self).__init__(host, port)
+
self.strict = strict
self.timeout = timeout
- self.pool = Queue(maxsize)
+ self.pool = self.QueueCls(maxsize)
self.block = block
self.headers = headers or {}
@@ -198,7 +207,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
conn = self.pool.get(block=self.block, timeout=timeout)
# If this is a persistent connection, check if it got disconnected
- if conn and conn.sock and is_connection_dropped(conn):
+ if conn and is_connection_dropped(conn):
log.info("Resetting dropped connection: %s" % self.host)
conn.close()
@@ -242,9 +251,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
timeout = self.timeout
conn.timeout = timeout # This only does anything in Py26+
-
conn.request(method, url, **httplib_request_kw)
- conn.sock.settimeout(timeout)
+
+ # Set timeout
+ sock = getattr(conn, 'sock', False) # AppEngine doesn't have sock attr.
+ if sock:
+ sock.settimeout(timeout)
+
httplib_response = conn.getresponse()
log.debug("\"%s %s %s\" %s %s" %
@@ -281,7 +294,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
.. note::
More commonly, it's appropriate to use a convenience method provided
- by :class:`.RequestMethods`, such as :meth:`.request`.
+ by :class:`.RequestMethods`, such as :meth:`request`.
.. note::
@@ -468,7 +481,11 @@ class HTTPSConnectionPool(HTTPConnectionPool):
log.info("Starting new HTTPS connection (%d): %s"
% (self.num_connections, self.host))
- if not ssl:
+ if not ssl: # Platform-specific: Python compiled without +ssl
+ if not HTTPSConnection or HTTPSConnection is object:
+ raise SSLError("Can't connect to HTTPS URL because the SSL "
+ "module is not available.")
+
return HTTPSConnection(host=self.host, port=self.port)
connection = VerifiedHTTPSConnection(host=self.host, port=self.port)
@@ -477,87 +494,6 @@ class HTTPSConnectionPool(HTTPConnectionPool):
return connection
-## Helpers
-
-def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
- basic_auth=None):
- """
- Shortcuts for generating request headers.
-
- :param keep_alive:
- If ``True``, adds 'connection: keep-alive' header.
-
- :param accept_encoding:
- Can be a boolean, list, or string.
- ``True`` translates to 'gzip,deflate'.
- List will get joined by comma.
- String will be used as provided.
-
- :param user_agent:
- String representing the user-agent you want, such as
- "python-urllib3/0.6"
-
- :param basic_auth:
- Colon-separated username:password string for 'authorization: basic ...'
- auth header.
-
- Example: ::
-
- >>> make_headers(keep_alive=True, user_agent="Batman/1.0")
- {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
- >>> make_headers(accept_encoding=True)
- {'accept-encoding': 'gzip,deflate'}
- """
- headers = {}
- if accept_encoding:
- if isinstance(accept_encoding, str):
- pass
- elif isinstance(accept_encoding, list):
- accept_encoding = ','.join(accept_encoding)
- else:
- accept_encoding = 'gzip,deflate'
- headers['accept-encoding'] = accept_encoding
-
- if user_agent:
- headers['user-agent'] = user_agent
-
- if keep_alive:
- headers['connection'] = 'keep-alive'
-
- if basic_auth:
- headers['authorization'] = 'Basic ' + \
- basic_auth.encode('base64').strip()
-
- return headers
-
-
-def get_host(url):
- """
- Given a url, return its scheme, host and port (None if it's not there).
-
- For example: ::
-
- >>> get_host('http://google.com/mail/')
- ('http', 'google.com', None)
- >>> get_host('google.com:80')
- ('http', 'google.com', 80)
- """
- # This code is actually similar to urlparse.urlsplit, but much
- # simplified for our needs.
- port = None
- scheme = 'http'
- if '://' in url:
- scheme, url = url.split('://', 1)
- if '/' in url:
- url, _path = url.split('/', 1)
- if '@' in url:
- _auth, url = url.split('@', 1)
- if ':' in url:
- url, port = url.split(':', 1)
- port = int(port)
- return scheme, url, port
-
-
def connection_from_url(url, **kw):
"""
Given a url, return an :class:`.ConnectionPool` instance of its host.
@@ -583,22 +519,3 @@ def connection_from_url(url, **kw):
return HTTPSConnectionPool(host, port=port, **kw)
else:
return HTTPConnectionPool(host, port=port, **kw)
-
-
-def is_connection_dropped(conn):
- """
- Returns True if the connection is dropped and should be closed.
-
- :param conn:
- ``HTTPConnection`` object.
- """
- if not poll:
- return select([conn.sock], [], [], 0.0)[0]
-
- # This version is better on platforms that support it.
- p = poll()
- p.register(conn.sock, POLLIN)
- for (fno, ev) in p.poll(0.0):
- if fno == conn.sock.fileno():
- # Either data is buffered (bad), or the connection is dropped.
- return True
diff --git a/libs/requests/packages/urllib3/exceptions.py b/libs/requests/packages/urllib3/exceptions.py
index 0bffeb47..15c9699e 100644
--- a/libs/requests/packages/urllib3/exceptions.py
+++ b/libs/requests/packages/urllib3/exceptions.py
@@ -4,6 +4,7 @@
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
+
## Base Exceptions
class HTTPError(Exception):
@@ -27,18 +28,20 @@ class SSLError(HTTPError):
class MaxRetryError(PoolError):
"Raised when the maximum number of retries is exceeded."
+
def __init__(self, pool, url):
- PoolError.__init__(self, pool,
- "Max retries exceeded with url: %s" % url)
+ message = "Max retries exceeded with url: %s" % url
+ PoolError.__init__(self, pool, message)
self.url = url
class HostChangedError(PoolError):
"Raised when an existing pool gets a request for a foreign host."
+
def __init__(self, pool, url, retries=3):
- PoolError.__init__(self, pool,
- "Tried to open a foreign host with url: %s" % url)
+ message = "Tried to open a foreign host with url: %s" % url
+ PoolError.__init__(self, pool, message)
self.url = url
self.retries = retries
@@ -52,3 +55,13 @@ class TimeoutError(PoolError):
class EmptyPoolError(PoolError):
"Raised when a pool runs out of connections and no more are allowed."
pass
+
+
+class LocationParseError(ValueError, HTTPError):
+ "Raised when get_host or similar fails to parse the URL input."
+
+ def __init__(self, location):
+ message = "Failed to parse: %s" % location
+ super(LocationParseError, self).__init__(self, message)
+
+ self.location = location
diff --git a/libs/requests/packages/urllib3/filepost.py b/libs/requests/packages/urllib3/filepost.py
index e1ec8af7..344a1030 100644
--- a/libs/requests/packages/urllib3/filepost.py
+++ b/libs/requests/packages/urllib3/filepost.py
@@ -24,15 +24,29 @@ def get_content_type(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+def iter_fields(fields):
+ """
+ Iterate over fields.
+
+ Supports list of (k, v) tuples and dicts.
+ """
+ if isinstance(fields, dict):
+ return ((k, v) for k, v in six.iteritems(fields))
+
+ return ((k, v) for k, v in fields)
+
+
def encode_multipart_formdata(fields, boundary=None):
"""
Encode a dictionary of ``fields`` using the multipart/form-data mime format.
:param fields:
- Dictionary of fields. The key is treated as the field name, and the
- value as the body of the form-data. If the value is a tuple of two
- elements, then the first element is treated as the filename of the
- form-data section.
+ Dictionary of fields or list of (key, value) field tuples. The key is
+ treated as the field name, and the value as the body of the form-data
+ bytes. If the value is a tuple of two elements, then the first element
+ is treated as the filename of the form-data section.
+
+ Field names and filenames must be unicode.
:param boundary:
If not specified, then a random boundary will be generated using
@@ -42,7 +56,7 @@ def encode_multipart_formdata(fields, boundary=None):
if boundary is None:
boundary = choose_boundary()
- for fieldname, value in six.iteritems(fields):
+ for fieldname, value in iter_fields(fields):
body.write(b('--%s\r\n' % (boundary)))
if isinstance(value, tuple):
diff --git a/libs/requests/packages/urllib3/poolmanager.py b/libs/requests/packages/urllib3/poolmanager.py
index f194b2e8..310ea21d 100644
--- a/libs/requests/packages/urllib3/poolmanager.py
+++ b/libs/requests/packages/urllib3/poolmanager.py
@@ -39,11 +39,11 @@ class PoolManager(RequestMethods):
Example: ::
- >>> manager = PoolManager()
+ >>> manager = PoolManager(num_pools=2)
>>> r = manager.urlopen("http://google.com/")
>>> r = manager.urlopen("http://google.com/mail")
>>> r = manager.urlopen("http://yahoo.com/")
- >>> len(r.pools)
+ >>> len(manager.pools)
2
"""
@@ -117,9 +117,19 @@ class ProxyManager(RequestMethods):
def __init__(self, proxy_pool):
self.proxy_pool = proxy_pool
+ def _set_proxy_headers(self, headers=None):
+ headers = headers or {}
+
+ # Same headers are curl passes for --proxy1.0
+ headers['Accept'] = '*/*'
+ headers['Proxy-Connection'] = 'Keep-Alive'
+
+ return headers
+
def urlopen(self, method, url, **kw):
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
kw['assert_same_host'] = False
+ kw['headers'] = self._set_proxy_headers(kw.get('headers'))
return self.proxy_pool.urlopen(method, url, **kw)
diff --git a/libs/requests/packages/urllib3/request.py b/libs/requests/packages/urllib3/request.py
index 5ea26a0f..569ac966 100644
--- a/libs/requests/packages/urllib3/request.py
+++ b/libs/requests/packages/urllib3/request.py
@@ -44,7 +44,7 @@ class RequestMethods(object):
def urlopen(self, method, url, body=None, headers=None,
encode_multipart=True, multipart_boundary=None,
- **kw):
+ **kw): # Abstract
raise NotImplemented("Classes extending RequestMethods must implement "
"their own ``urlopen`` method.")
@@ -126,22 +126,3 @@ class RequestMethods(object):
return self.urlopen(method, url, body=body, headers=headers,
**urlopen_kw)
-
- # Deprecated:
-
- def get_url(self, url, fields=None, **urlopen_kw):
- """
- .. deprecated:: 1.0
- Use :meth:`request` instead.
- """
- return self.request_encode_url('GET', url, fields=fields,
- **urlopen_kw)
-
- def post_url(self, url, fields=None, headers=None, **urlopen_kw):
- """
- .. deprecated:: 1.0
- Use :meth:`request` instead.
- """
- return self.request_encode_body('POST', url, fields=fields,
- headers=headers,
- **urlopen_kw)
diff --git a/libs/requests/packages/urllib3/response.py b/libs/requests/packages/urllib3/response.py
index e0239703..5fab8243 100644
--- a/libs/requests/packages/urllib3/response.py
+++ b/libs/requests/packages/urllib3/response.py
@@ -11,12 +11,7 @@ import zlib
from io import BytesIO
from .exceptions import HTTPError
-
-
-try:
- basestring = basestring
-except NameError: # Python 3
- basestring = (str, bytes)
+from .packages.six import string_types as basestring
log = logging.getLogger(__name__)
@@ -176,11 +171,22 @@ class HTTPResponse(object):
with ``original_response=r``.
"""
+ # Normalize headers between different versions of Python
+ headers = {}
+ for k, v in r.getheaders():
+ # Python 3: Header keys are returned capitalised
+ k = k.lower()
+
+ has_value = headers.get(k)
+ if has_value: # Python 3: Repeating header keys are unmerged.
+ v = ', '.join([has_value, v])
+
+ headers[k] = v
+
# HTTPResponse objects in Python 3 don't have a .strict attribute
strict = getattr(r, 'strict', 0)
return ResponseCls(body=r,
- # In Python 3, the header keys are returned capitalised
- headers=dict((k.lower(), v) for k,v in r.getheaders()),
+ headers=headers,
status=r.status,
version=r.version,
reason=r.reason,
diff --git a/libs/requests/packages/urllib3/util.py b/libs/requests/packages/urllib3/util.py
new file mode 100644
index 00000000..2684a2fd
--- /dev/null
+++ b/libs/requests/packages/urllib3/util.py
@@ -0,0 +1,136 @@
+# urllib3/util.py
+# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+#
+# This module is part of urllib3 and is released under
+# the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+
+from base64 import b64encode
+
+try:
+ from select import poll, POLLIN
+except ImportError: # `poll` doesn't exist on OSX and other platforms
+ poll = False
+ try:
+ from select import select
+ except ImportError: # `select` doesn't exist on AppEngine.
+ select = False
+
+from .packages import six
+from .exceptions import LocationParseError
+
+
+def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
+ basic_auth=None):
+ """
+ Shortcuts for generating request headers.
+
+ :param keep_alive:
+ If ``True``, adds 'connection: keep-alive' header.
+
+ :param accept_encoding:
+ Can be a boolean, list, or string.
+ ``True`` translates to 'gzip,deflate'.
+ List will get joined by comma.
+ String will be used as provided.
+
+ :param user_agent:
+ String representing the user-agent you want, such as
+ "python-urllib3/0.6"
+
+ :param basic_auth:
+ Colon-separated username:password string for 'authorization: basic ...'
+ auth header.
+
+ Example: ::
+
+ >>> make_headers(keep_alive=True, user_agent="Batman/1.0")
+ {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
+ >>> make_headers(accept_encoding=True)
+ {'accept-encoding': 'gzip,deflate'}
+ """
+ headers = {}
+ if accept_encoding:
+ if isinstance(accept_encoding, str):
+ pass
+ elif isinstance(accept_encoding, list):
+ accept_encoding = ','.join(accept_encoding)
+ else:
+ accept_encoding = 'gzip,deflate'
+ headers['accept-encoding'] = accept_encoding
+
+ if user_agent:
+ headers['user-agent'] = user_agent
+
+ if keep_alive:
+ headers['connection'] = 'keep-alive'
+
+ if basic_auth:
+ headers['authorization'] = 'Basic ' + \
+ b64encode(six.b(basic_auth)).decode('utf-8')
+
+ return headers
+
+
+def get_host(url):
+ """
+ Given a url, return its scheme, host and port (None if it's not there).
+
+ For example: ::
+
+ >>> get_host('http://google.com/mail/')
+ ('http', 'google.com', None)
+ >>> get_host('google.com:80')
+ ('http', 'google.com', 80)
+ """
+
+ # This code is actually similar to urlparse.urlsplit, but much
+ # simplified for our needs.
+ port = None
+ scheme = 'http'
+
+ if '://' in url:
+ scheme, url = url.split('://', 1)
+ if '/' in url:
+ url, _path = url.split('/', 1)
+ if '@' in url:
+ _auth, url = url.split('@', 1)
+ if ':' in url:
+ url, port = url.split(':', 1)
+
+ if not port.isdigit():
+ raise LocationParseError("Failed to parse: %s" % url)
+
+ port = int(port)
+
+ return scheme, url, port
+
+
+
+def is_connection_dropped(conn):
+ """
+ Returns True if the connection is dropped and should be closed.
+
+ :param conn:
+ ``HTTPConnection`` object.
+
+ Note: For platforms like AppEngine, this will always return ``False`` to
+ let the platform handle connection recycling transparently for us.
+ """
+ sock = getattr(conn, 'sock', False)
+ if not sock: #Platform-specific: AppEngine
+ return False
+
+ if not poll: # Platform-specific
+ if not select: #Platform-specific: AppEngine
+ return False
+
+ return select([sock], [], [], 0.0)[0]
+
+ # This version is better on platforms that support it.
+ p = poll()
+ p.register(sock, POLLIN)
+ for (fno, ev) in p.poll(0.0):
+ if fno == sock.fileno():
+ # Either data is buffered (bad), or the connection is dropped.
+ return True
diff --git a/libs/requests/sessions.py b/libs/requests/sessions.py
index d9683b03..94c94bf9 100644
--- a/libs/requests/sessions.py
+++ b/libs/requests/sessions.py
@@ -40,7 +40,7 @@ def merge_kwargs(local_kwarg, default_kwarg):
kwargs.update(local_kwarg)
# Remove keys that are set to None.
- for (k,v) in list(local_kwarg.items()):
+ for (k, v) in list(local_kwarg.items()):
if v is None:
del kwargs[k]
@@ -52,7 +52,7 @@ class Session(object):
__attrs__ = [
'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
- 'params', 'config']
+ 'params', 'config', 'verify', 'cert']
def __init__(self,
@@ -64,7 +64,9 @@ class Session(object):
hooks=None,
params=None,
config=None,
- verify=True):
+ prefetch=False,
+ verify=True,
+ cert=None):
self.headers = headers or {}
self.cookies = cookies or {}
@@ -74,15 +76,14 @@ class Session(object):
self.hooks = hooks or {}
self.params = params or {}
self.config = config or {}
+ self.prefetch = prefetch
self.verify = verify
+ self.cert = cert
for (k, v) in list(defaults.items()):
self.config.setdefault(k, v)
- self.poolmanager = PoolManager(
- num_pools=self.config.get('pool_connections'),
- maxsize=self.config.get('pool_maxsize')
- )
+ self.init_poolmanager()
# Set up a CookieJar to be used by default
self.cookies = {}
@@ -91,6 +92,12 @@ class Session(object):
if cookies is not None:
self.cookies.update(cookies)
+ def init_poolmanager(self):
+ self.poolmanager = PoolManager(
+ num_pools=self.config.get('pool_connections'),
+ maxsize=self.config.get('pool_maxsize')
+ )
+
def __repr__(self):
return '' % (id(self))
@@ -108,13 +115,14 @@ class Session(object):
files=None,
auth=None,
timeout=None,
- allow_redirects=False,
+ allow_redirects=True,
proxies=None,
hooks=None,
return_response=True,
config=None,
prefetch=False,
- verify=None):
+ verify=None,
+ cert=None):
"""Constructs and sends a :class:`Request `.
Returns :class:`Response ` object.
@@ -128,12 +136,13 @@ class Session(object):
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
- :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
+ :param allow_redirects: (optional) Boolean. Set to True by default.
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
:param return_response: (optional) If False, an un-sent Request object will returned.
:param config: (optional) A configuration dictionary.
:param prefetch: (optional) if ``True``, the response content will be immediately downloaded.
:param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
+ :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair.
"""
method = str(method).upper()
@@ -145,9 +154,7 @@ class Session(object):
headers = {} if headers is None else headers
params = {} if params is None else params
hooks = {} if hooks is None else hooks
-
- if verify is None:
- verify = self.verify
+ prefetch = self.prefetch or prefetch
# use session's hooks as defaults
for key, cb in list(self.hooks.items()):
@@ -173,6 +180,7 @@ class Session(object):
proxies=proxies,
config=config,
verify=verify,
+ cert=cert,
_poolmanager=self.poolmanager
)
@@ -210,7 +218,7 @@ class Session(object):
"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
@@ -221,7 +229,7 @@ class Session(object):
"""Sends a OPTIONS request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
kwargs.setdefault('allow_redirects', True)
@@ -232,10 +240,10 @@ class Session(object):
"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
- kwargs.setdefault('allow_redirects', True)
+ kwargs.setdefault('allow_redirects', False)
return self.request('head', url, **kwargs)
@@ -244,7 +252,7 @@ class Session(object):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return self.request('post', url, data=data, **kwargs)
@@ -255,7 +263,7 @@ class Session(object):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return self.request('put', url, data=data, **kwargs)
@@ -266,7 +274,7 @@ class Session(object):
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return self.request('patch', url, data=data, **kwargs)
@@ -276,11 +284,20 @@ class Session(object):
"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
- :param **kwargs: Optional arguments that ``request`` takes.
+ :param \*\*kwargs: Optional arguments that ``request`` takes.
"""
return self.request('delete', url, **kwargs)
+ def __getstate__(self):
+ return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)
+
+ def __setstate__(self, state):
+ for attr, value in state.items():
+ setattr(self, attr, value)
+
+ self.init_poolmanager()
+
def session(**kwargs):
"""Returns a :class:`Session` for context-management."""
diff --git a/libs/requests/utils.py b/libs/requests/utils.py
index 0e0f69ee..b722d990 100644
--- a/libs/requests/utils.py
+++ b/libs/requests/utils.py
@@ -15,9 +15,54 @@ import os
import random
import re
import zlib
+from netrc import netrc, NetrcParseError
from .compat import parse_http_list as _parse_list_header
-from .compat import quote, unquote, cookielib, SimpleCookie, is_py2
+from .compat import quote, cookielib, SimpleCookie, is_py2, urlparse
+from .compat import basestring, bytes, str
+
+
+NETRC_FILES = ('.netrc', '_netrc')
+
+
+def dict_to_sequence(d):
+ """Returns an internal sequence dictionary update."""
+
+ if hasattr(d, 'items'):
+ d = d.items()
+
+ return d
+
+
+def get_netrc_auth(url):
+ """Returns the Requests tuple auth for a given url from netrc."""
+
+ locations = (os.path.expanduser('~/{0}'.format(f)) for f in NETRC_FILES)
+ netrc_path = None
+
+ for loc in locations:
+ if os.path.exists(loc) and not netrc_path:
+ netrc_path = loc
+
+ # Abort early if there isn't one.
+ if netrc_path is None:
+ return netrc_path
+
+ ri = urlparse(url)
+
+ # Strip port numbers from netloc
+ host = ri.netloc.split(':')[0]
+
+ try:
+ _netrc = netrc(netrc_path).authenticators(host)
+ if _netrc:
+ # Return with login / password
+ login_i = (0 if _netrc[0] else 1)
+ return (_netrc[login_i], _netrc[2])
+ except (NetrcParseError, IOError, AttributeError):
+ # If there was a parsing error or a permissions issue reading the file,
+ # we'll just skip netrc auth
+ pass
def dict_from_string(s):
@@ -25,20 +70,26 @@ def dict_from_string(s):
cookies = dict()
- c = SimpleCookie()
- c.load(s)
+ try:
+ c = SimpleCookie()
+ c.load(s)
- for k,v in list(c.items()):
- cookies.update({k: v.value})
+ for k, v in list(c.items()):
+ cookies.update({k: v.value})
+ # This stuff is not to be trusted.
+ except Exception:
+ pass
return cookies
+
def guess_filename(obj):
"""Tries to guess the filename of the given object."""
name = getattr(obj, 'name', None)
if name and name[0] != '<' and name[-1] != '>':
return name
+
# From mitsuhiko/werkzeug (used with permission).
def parse_list_header(value):
"""Parse lists as described by RFC 2068 Section 2.
@@ -145,8 +196,14 @@ def header_expand(headers):
if isinstance(headers, dict):
headers = list(headers.items())
-
+ elif isinstance(headers, basestring):
+ return headers
elif isinstance(headers, str):
+ # As discussed in https://github.com/kennethreitz/requests/issues/400
+ # latin-1 is the most conservative encoding used on the web. Anyone
+ # who needs more can encode to a byte-string before calling
+ return headers.encode("latin-1")
+ elif headers is None:
return headers
for i, (value, params) in enumerate(headers):
@@ -164,10 +221,9 @@ def header_expand(headers):
collector.append('; '.join(_params))
- if not len(headers) == i+1:
+ if not len(headers) == i + 1:
collector.append(', ')
-
# Remove trailing separators.
if collector[-1] in (', ', '; '):
del collector[-1]
@@ -175,7 +231,6 @@ def header_expand(headers):
return ''.join(collector)
-
def randombytes(n):
"""Return n random bytes."""
if is_py2:
@@ -286,23 +341,6 @@ def get_encoding_from_headers(headers):
return 'ISO-8859-1'
-def unicode_from_html(content):
- """Attempts to decode an HTML string into unicode.
- If unsuccessful, the original content is returned.
- """
-
- encodings = get_encodings_from_content(content)
-
- for encoding in encodings:
-
- try:
- return str(content, encoding)
- except (UnicodeError, TypeError):
- pass
-
- return content
-
-
def stream_decode_response_unicode(iterator, r):
"""Stream decodes a iterator."""
@@ -354,15 +392,6 @@ def get_unicode_from_response(r):
return r.content
-def decode_gzip(content):
- """Return gzip-decoded string.
-
- :param content: bytestring to gzip-decode.
- """
-
- return zlib.decompress(content, 16 + zlib.MAX_WBITS)
-
-
def stream_decompress(iterator, mode='gzip'):
"""
Stream decodes an iterator over compressed data
@@ -390,18 +419,53 @@ def stream_decompress(iterator, mode='gzip'):
yield chunk
else:
# Make sure everything has been returned from the decompression object
- buf = dec.decompress('')
+ buf = dec.decompress(bytes())
rv = buf + dec.flush()
if rv:
yield rv
-def requote_path(path):
- """Re-quote the given URL path component.
+def stream_untransfer(gen, resp):
+ if 'gzip' in resp.headers.get('content-encoding', ''):
+ gen = stream_decompress(gen, mode='gzip')
+ elif 'deflate' in resp.headers.get('content-encoding', ''):
+ gen = stream_decompress(gen, mode='deflate')
- This function passes the given path through an unquote/quote cycle to
+ return gen
+
+
+# The unreserved URI characters (RFC 3986)
+UNRESERVED_SET = frozenset(
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+ + "0123456789-._~")
+
+
+def unquote_unreserved(uri):
+ """Un-escape any percent-escape sequences in a URI that are unreserved
+ characters.
+ This leaves all reserved, illegal and non-ASCII bytes encoded.
+ """
+ parts = uri.split('%')
+ for i in range(1, len(parts)):
+ h = parts[i][0:2]
+ if len(h) == 2:
+ c = chr(int(h, 16))
+ if c in UNRESERVED_SET:
+ parts[i] = c + parts[i][2:]
+ else:
+ parts[i] = '%' + parts[i]
+ else:
+ parts[i] = '%' + parts[i]
+ return ''.join(parts)
+
+
+def requote_uri(uri):
+ """Re-quote the given URI.
+
+ This function passes the given URI through an unquote/quote cycle to
ensure that it is fully and consistently quoted.
"""
- parts = path.split(b"/")
- parts = (quote(unquote(part), safe=b"") for part in parts)
- return b"/".join(parts)
+ # Unquote only the unreserved characters
+ # Then quote only illegal characters (do not quote reserved, unreserved,
+ # or '%')
+ return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~")
diff --git a/libs/subliminal/__init__.py b/libs/subliminal/__init__.py
index 155cf244..666440b6 100755
--- a/libs/subliminal/__init__.py
+++ b/libs/subliminal/__init__.py
@@ -1,23 +1,34 @@
# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
#
-# Subliminal - Subtitles, faster than your thoughts
-# Copyright (c) 2011 Antoine Bertin
+# This file is part of subliminal.
#
-# This file is part of Subliminal.
-#
-# Subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Subliminal is distributed in the hope that it will be useful,
+# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
+# GNU Lesser General Public License for more details.
#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
-#
-__all__ = ['Subliminal']
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from .api import list_subtitles, download_subtitles
+from .async import Pool
+from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE,
+ MATCHING_CONFIDENCE)
+from .infos import __version__
+import logging
+try:
+ from logging import NullHandler
+except ImportError:
+ class NullHandler(logging.Handler):
+ def emit(self, record):
+ pass
-from core import Subliminal
+
+__all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE',
+ 'MATCHING_CONFIDENCE', 'list_subtitles', 'download_subtitles', 'Pool']
+logging.getLogger(__name__).addHandler(NullHandler())
diff --git a/libs/subliminal/api.py b/libs/subliminal/api.py
new file mode 100755
index 00000000..baff5f1a
--- /dev/null
+++ b/libs/subliminal/api.py
@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from .core import (SERVICES, LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE,
+ MATCHING_CONFIDENCE, create_list_tasks, consume_task, create_download_tasks,
+ group_by_video, key_subtitles)
+from .languages import list_languages
+import logging
+
+
+__all__ = ['list_subtitles', 'download_subtitles']
+logger = logging.getLogger(__name__)
+
+
+def list_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3):
+ """List subtitles in given paths according to the criteria
+
+ :param paths: path(s) to video file or folder
+ :type paths: string or list
+ :param list languages: languages to search for, in preferred order
+ :param list services: services to use for the search, in preferred order
+ :param bool force: force searching for subtitles even if some are detected
+ :param bool multi: search multiple languages for the same video
+ :param string cache_dir: path to the cache directory to use
+ :param int max_depth: maximum depth for scanning entries
+ :return: found subtitles
+ :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.ResultSubtitle`]
+
+ """
+ services = services or SERVICES
+ languages = set(languages or list_languages(1))
+ if isinstance(paths, basestring):
+ paths = [paths]
+ if any([not isinstance(p, unicode) for p in paths]):
+ logger.warning(u'Not all entries are unicode')
+ results = []
+ service_instances = {}
+ tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth)
+ for task in tasks:
+ try:
+ result = consume_task(task, service_instances)
+ results.append((task.video, result))
+ except:
+ logger.error(u'Error consuming task %r' % task, exc_info=True)
+ for service_instance in service_instances.itervalues():
+ service_instance.terminate()
+ return group_by_video(results)
+
+
+def download_subtitles(paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3, order=None):
+ """Download subtitles in given paths according to the criteria
+
+ :param paths: path(s) to video file or folder
+ :type paths: string or list
+ :param list languages: languages to search for, in preferred order
+ :param list services: services to use for the search, in preferred order
+ :param bool force: force searching for subtitles even if some are detected
+ :param bool multi: search multiple languages for the same video
+ :param string cache_dir: path to the cache directory to use
+ :param int max_depth: maximum depth for scanning entries
+ :param order: preferred order for subtitles sorting
+ :type list: list of :data:`~subliminal.core.LANGUAGE_INDEX`, :data:`~subliminal.core.SERVICE_INDEX`, :data:`~subliminal.core.SERVICE_CONFIDENCE`, :data:`~subliminal.core.MATCHING_CONFIDENCE`
+ :return: found subtitles
+ :rtype: list of (:class:`~subliminal.videos.Video`, [:class:`~subliminal.subtitles.ResultSubtitle`])
+
+ """
+ services = services or SERVICES
+ languages = languages or list_languages(1)
+ if isinstance(paths, basestring):
+ paths = [paths]
+ order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
+ subtitles_by_video = list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth)
+ for video, subtitles in subtitles_by_video.iteritems():
+ subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
+ results = []
+ service_instances = {}
+ tasks = create_download_tasks(subtitles_by_video, multi)
+ for task in tasks:
+ try:
+ result = consume_task(task, service_instances)
+ results.append(result)
+ except:
+ logger.error(u'Error consuming task %r' % task, exc_info=True)
+ for service_instance in service_instances.itervalues():
+ service_instance.terminate()
+ return results
diff --git a/libs/subliminal/async.py b/libs/subliminal/async.py
new file mode 100755
index 00000000..ce18a278
--- /dev/null
+++ b/libs/subliminal/async.py
@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from .core import (consume_task, LANGUAGE_INDEX, SERVICE_INDEX,
+ SERVICE_CONFIDENCE, MATCHING_CONFIDENCE, SERVICES, create_list_tasks,
+ create_download_tasks, group_by_video, key_subtitles)
+from .languages import list_languages
+from .tasks import StopTask
+import Queue
+import logging
+import threading
+
+
+logger = logging.getLogger(__name__)
+
+
+class Worker(threading.Thread):
+ """Consume tasks and put the result in the queue"""
+ def __init__(self, tasks, results):
+ super(Worker, self).__init__()
+ self.tasks = tasks
+ self.results = results
+ self.services = {}
+
+ def run(self):
+ while 1:
+ result = []
+ try:
+ task = self.tasks.get(block=True)
+ if isinstance(task, StopTask):
+ break
+ result = consume_task(task, self.services)
+ self.results.put((task.video, result))
+ except:
+ logger.error(u'Exception raised in worker %s' % self.name, exc_info=True)
+ finally:
+ self.tasks.task_done()
+ self.terminate()
+ logger.debug(u'Thread %s terminated' % self.name)
+
+ def terminate(self):
+ """Terminate instantiated services"""
+ for service_name, service in self.services.iteritems():
+ try:
+ service.terminate()
+ except:
+ logger.error(u'Exception raised when terminating service %s' % service_name, exc_info=True)
+
+
+class Pool(object):
+ """Pool of workers"""
+ def __init__(self, size):
+ self.tasks = Queue.Queue()
+ self.results = Queue.Queue()
+ self.workers = []
+ for _ in range(size):
+ self.workers.append(Worker(self.tasks, self.results))
+
+ def __enter__(self):
+ self.start()
+ return self
+
+ def __exit__(self, *args):
+ self.stop()
+ self.join()
+
+ def start(self):
+ """Start workers"""
+ for worker in self.workers:
+ worker.start()
+
+ def stop(self):
+ """Stop workers"""
+ for _ in self.workers:
+ self.tasks.put(StopTask())
+
+ def join(self):
+ """Join the task queue"""
+ self.tasks.join()
+
+ def collect(self):
+ """Collect available results
+
+ :return: results of tasks
+ :rtype: list of :class:`~subliminal.tasks.Task`
+
+ """
+ results = []
+ while 1:
+ try:
+ result = self.results.get(block=False)
+ results.append(result)
+ except Queue.Empty:
+ break
+ return results
+
+ def list_subtitles(self, paths, languages=None, services=None, force=True, multi=False, cache_dir=None, max_depth=3):
+ """See :meth:`subliminal.list_subtitles`"""
+ services = services or SERVICES
+ languages = set(languages or list_languages(1))
+ if isinstance(paths, basestring):
+ paths = [paths]
+ if any([not isinstance(p, unicode) for p in paths]):
+ logger.warning(u'Not all entries are unicode')
+ tasks = create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth)
+ for task in tasks:
+ self.tasks.put(task)
+ self.join()
+ results = self.collect()
+ return group_by_video(results)
+
+ def download_subtitles(self, paths, languages=None, services=None, cache_dir=None, max_depth=3, force=True, multi=False, order=None):
+ """See :meth:`subliminal.download_subtitles`"""
+ services = services or SERVICES
+ languages = languages or list_languages(1)
+ if isinstance(paths, basestring):
+ paths = [paths]
+ order = order or [LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE]
+ subtitles_by_video = self.list_subtitles(paths, set(languages), services, force, multi, cache_dir, max_depth)
+ for video, subtitles in subtitles_by_video.iteritems():
+ subtitles.sort(key=lambda s: key_subtitles(s, video, languages, services, order), reverse=True)
+ tasks = create_download_tasks(subtitles_by_video, multi)
+ for task in tasks:
+ self.tasks.put(task)
+ self.join()
+ results = self.collect()
+ return results
diff --git a/libs/subliminal/core.py b/libs/subliminal/core.py
index 8f762b48..56f4347e 100755
--- a/libs/subliminal/core.py
+++ b/libs/subliminal/core.py
@@ -1,346 +1,173 @@
# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
#
-# Subliminal - Subtitles, faster than your thoughts
-# Copyright (c) 2011 Antoine Bertin
+# This file is part of subliminal.
#
-# This file is part of Subliminal.
-#
-# Subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Subliminal is distributed in the hope that it will be useful,
+# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
+# GNU Lesser General Public License for more details.
#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
-#
-__all__ = ['PLUGINS', 'API_PLUGINS', 'IDLE', 'RUNNING', 'PAUSED', 'Subliminal', 'PluginWorker', 'matching_confidence',
- 'LANGUAGE_INDEX', 'PLUGIN_INDEX', 'PLUGIN_CONFIDENCE', 'MATCHING_CONFIDENCE']
-
-
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from .exceptions import DownloadFailedError
+from .services import ServiceConfig
+from .tasks import DownloadTask, ListTask
+from .utils import get_keywords
+from .videos import Episode, Movie, scan
from collections import defaultdict
-from exceptions import InvalidLanguageError, PluginError, BadStateError, \
- WrongTaskError, DownloadFailedError
from itertools import groupby
-from languages import list_languages
-from utils import NullHandler
-from tasks import Task, DownloadTask, ListTask, StopTask
-import Queue
import guessit
import logging
-import os
-import plugins
-import subtitles
-import threading
-import utils
-import videos
-
-# init logger
-logger = logging.getLogger('subliminal')
-logger.addHandler(NullHandler())
-
-# const
-PLUGINS = ['OpenSubtitles', 'BierDopje', 'TheSubDB', 'SubsWiki', 'Subtitulos']
-API_PLUGINS = filter(lambda p: getattr(plugins, p).api_based, PLUGINS)
-IDLE, RUNNING, PAUSED = range(3)
-LANGUAGE_INDEX, PLUGIN_INDEX, PLUGIN_CONFIDENCE, MATCHING_CONFIDENCE = range(4)
-class Subliminal(object):
- """Main Subliminal class"""
+__all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE', 'MATCHING_CONFIDENCE',
+ 'create_list_tasks', 'create_download_tasks', 'consume_task', 'matching_confidence',
+ 'key_subtitles', 'group_by_video']
+logger = logging.getLogger(__name__)
+SERVICES = ['opensubtitles', 'bierdopje', 'subswiki', 'subtitulos', 'thesubdb']
+LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE = range(4)
- def __init__(self, cache_dir=None, workers=None, multi=False, force=False,
- max_depth=None, filemode=None, sort_order=None, plugins=None, languages=None):
- self.multi = multi
- self.sort_order = sort_order or [LANGUAGE_INDEX, PLUGIN_INDEX, PLUGIN_CONFIDENCE, MATCHING_CONFIDENCE]
- self.force = force
- self.max_depth = max_depth or 3
- self.taskQueue = Queue.PriorityQueue()
- self.listResultQueue = Queue.Queue()
- self.downloadResultQueue = Queue.Queue()
- self.languages = languages or []
- self.plugins = plugins or API_PLUGINS
- self._workers = workers or 4
- self.filemode = filemode
- self.state = IDLE
- self.cache_dir = cache_dir
- try:
- if cache_dir and not os.path.isdir(cache_dir):
- os.makedirs(cache_dir)
- logger.debug(u'Creating cache directory: %r' % cache_dir)
- except:
- self.cache_dir = None
- logger.error(u'Failed to use the cache directory, continue without it')
- def __enter__(self):
- self.startWorkers()
- return self
+def create_list_tasks(paths, languages, services, force, multi, cache_dir, max_depth):
+ """Create a list of :class:`~subliminal.tasks.ListTask` from one or more paths using the given criteria
- def __exit__(self, *args):
- self.stopWorkers(0)
+ :param paths: path(s) to video file or folder
+ :type paths: string or list
+ :param set languages: languages to search for
+ :param list services: services to use for the search
+ :param bool force: force searching for subtitles even if some are detected
+ :param bool multi: search multiple languages for the same video
+ :param string cache_dir: path to the cache directory to use
+ :param int max_depth: maximum depth for scanning entries
+ :return: the created tasks
+ :rtype: list of :class:`~subliminal.tasks.ListTask`
- @property
- def workers(self):
- return self._workers
-
- @workers.setter
- def workers(self, value):
- if self.state == RUNNING:
- raise BadStateError(self.state, IDLE)
- self._workers = value
-
- @property
- def languages(self):
- """Getter for languages"""
- return self._languages
-
- @languages.setter
- def languages(self, languages):
- """Setter for languages"""
- logger.debug(u'Setting languages to %r' % languages)
- self._languages = []
- for l in languages:
- if l not in list_languages(1):
- raise InvalidLanguageError(l)
- if not l in self._languages:
- self._languages.append(l)
-
- @property
- def plugins(self):
- """Getter for plugins"""
- return self._plugins
-
- @plugins.setter
- def plugins(self, plugins):
- """Setter for plugins"""
- logger.debug(u'Setting plugins to %r' % plugins)
- self._plugins = []
- for p in plugins:
- if p not in PLUGINS:
- raise PluginError(p)
- if not p in self._plugins:
- self._plugins.append(p)
-
- def listSubtitles(self, entries, auto=False):
- """
- Search subtitles within the plugins and return all found subtitles in a list of Subtitle object.
-
- Attributes:
- entries -- filepath or folderpath of video file or a list of that
- auto -- automaticaly manage workers (default to False)"""
- if auto:
- if self.state != IDLE:
- raise BadStateError(self.state, IDLE)
- self.startWorkers()
- if isinstance(entries, basestring):
- entries = [entries]
- config = utils.PluginConfig(self.multi, self.cache_dir, self.filemode)
- scan_result = []
- for e in entries:
- if not isinstance(e, unicode):
- logger.warning(u'Entry %r is not unicode' % e)
- scan_result.extend(videos.scan(e))
- task_count = 0
- for video, subtitles in scan_result:
- languages = set([s.language for s in subtitles if s.language])
- wanted_languages = set(self._languages)
+ """
+ scan_result = []
+ for p in paths:
+ scan_result.extend(scan(p, max_depth))
+ logger.debug(u'Found %d videos in %r with maximum depth %d' % (len(scan_result), paths, max_depth))
+ tasks = []
+ config = ServiceConfig(multi, cache_dir)
+ for video, detected_subtitles in scan_result:
+ detected_languages = set([s.language for s in detected_subtitles])
+ wanted_languages = languages.copy()
+ if not force and multi:
+ wanted_languages -= detected_languages
if not wanted_languages:
- wanted_languages = list_languages(1)
- if not self.force and self.multi:
- wanted_languages = set(wanted_languages) - languages
- if not wanted_languages:
- logger.debug(u'No need to list multi subtitles %r for %r because %r subtitles detected' % (self._languages, video.path, languages))
- continue
- if not self.force and not self.multi and None in [s.language for s in subtitles]:
- logger.debug(u'No need to list single subtitles %r for %r because one detected' % (self._languages, video.path))
+ logger.debug(u'No need to list multi subtitles %r for %r because %r detected' % (languages, video, detected_languages))
continue
- logger.debug(u'Listing subtitles %r for %r with %r' % (wanted_languages, video.path, self._plugins))
- for plugin_name in self._plugins:
- plugin = getattr(plugins, plugin_name)
- to_list_languages = wanted_languages & plugin.availableLanguages()
- if not to_list_languages:
- logger.debug(u'Skipping %r: none of wanted languages %r available in %r for plugin %s' % (video.path, wanted_languages, plugin.availableLanguages(), plugin_name))
- continue
- if not plugin.isValidVideo(video):
- logger.debug(u'Skipping %r: video %r is not part of supported videos %r for plugin %s' % (video.path, video, plugin.videos, plugin_name))
- continue
- self.taskQueue.put((5, ListTask(video, to_list_languages, plugin_name, config)))
- task_count += 1
- subtitles = []
- for _ in range(task_count):
- subtitles.extend(self.listResultQueue.get())
- if auto:
- self.stopWorkers()
- return subtitles
-
- def downloadSubtitles(self, entries, auto=False):
- """
- Download subtitles using the plugins preferences and languages. Also use internal algorithm to find
- the best match inside a plugin.
-
- Attributes:
- entries -- filepath or folderpath of video file or a list of that
- auto -- automaticaly manage workers (default to False)"""
- if auto:
- if self.state != IDLE:
- raise BadStateError(self.state, IDLE)
- self.startWorkers()
- by_video = self.groupByVideo(self.listSubtitles(entries, False))
- # Define an order with LANGUAGE_INDEX first for multi sorting
- order = self.sort_order
- if self.multi:
- order.insert(0, LANGUAGE_INDEX)
- task_count = 0
- for video, subtitles in by_video.iteritems():
- ordered_subtitles = sorted(subtitles, key=lambda s: self.keySubtitles(s, video, order), reverse=True)
- if not self.multi:
- self.taskQueue.put((5, DownloadTask(video, list(ordered_subtitles))))
- task_count += 1
+ if not force and not multi and None in detected_languages:
+ logger.debug(u'No need to list single subtitles %r for %r because one detected' % (languages, video))
+ continue
+ logger.debug(u'Listing subtitles %r for %r with services %r' % (wanted_languages, video, services))
+ for service_name in services:
+ mod = __import__('services.' + service_name, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
+ service = mod.Service
+ service_languages = wanted_languages & service.available_languages()
+ if not service_languages:
+ logger.debug(u'Skipping %r: none of wanted languages %r available for service %s' % (video, wanted_languages, service_name))
continue
- for _, by_language in groupby(ordered_subtitles, lambda s: s.language):
- self.taskQueue.put((5, DownloadTask(video, list(by_language))))
- task_count += 1
- downloaded = []
- for _ in range(task_count):
- downloaded.extend(self.downloadResultQueue.get())
- if auto:
- self.stopWorkers()
- return downloaded
-
- def keySubtitles(self, subtitle, video, order):
- """Create a key to sort subtitle using preferences"""
- key = ''
- for sort_item in order:
- if sort_item == LANGUAGE_INDEX:
- key += '{0:03d}'.format(len(self._languages) - self._languages.index(subtitle.language) - 1)
- elif sort_item == PLUGIN_INDEX:
- key += '{0:02d}'.format(len(self._plugins) - self._plugins.index(subtitle.plugin) - 1)
- elif sort_item == PLUGIN_CONFIDENCE:
- key += '{0:04d}'.format(int(subtitle.confidence * 1000))
- elif sort_item == MATCHING_CONFIDENCE:
- confidence = 0
- if subtitle.release:
- confidence = matching_confidence(video, subtitle)
- key += '{0:04d}'.format(int(confidence * 1000))
- return int(key)
-
- def groupByVideo(self, list_result):
- '''Because list outputs a list of tuples from different plugins, we need to put them back
- together under a single video key'''
- result = defaultdict(list)
- for video, subtitles in list_result:
- result[video] += subtitles
- return result
-
- def startWorkers(self):
- """Create a pool of workers and start them"""
- if self.state == RUNNING:
- raise BadStateError(self.state, IDLE)
- self.pool = []
- for _ in range(self._workers):
- worker = PluginWorker(self.taskQueue, self.listResultQueue, self.downloadResultQueue)
- worker.start()
- self.pool.append(worker)
- logger.debug(u'Worker %s added to the pool' % worker.name)
- self.state = RUNNING
-
- def stopWorkers(self, priority=10):
- """Stop workers using a lowest priority stop signal and wait for them to terminate properly"""
- for _ in range(self._workers):
- self.taskQueue.put((priority, StopTask()))
- for worker in self.pool:
- worker.join()
- self.state = IDLE
- if not self.taskQueue.empty():
- self.state = PAUSED
-
- def pauseWorkers(self):
- """Pause workers using a highest priority stop signal and wait for them to terminate properly"""
- self.stopWorkers(0)
-
- def addTask(self, task):
- """Add a task with default priority"""
- if not isinstance(task, Task) or isinstance(task, StopTask):
- raise WrongTaskError()
- self.taskQueue.put((5, task))
+ if not service.is_valid_video(video):
+ logger.debug(u'Skipping %r: not part of supported videos %r for service %s' % (video, service.videos, service_name))
+ continue
+ task = ListTask(video, service_languages, service_name, config)
+ logger.debug(u'Created task %r' % task)
+ tasks.append(task)
+ return tasks
-class PluginWorker(threading.Thread):
- """Threaded plugin worker"""
- def __init__(self, taskQueue, listResultQueue, downloadResultQueue):
- threading.Thread.__init__(self)
- self.taskQueue = taskQueue
- self.listResultQueue = listResultQueue
- self.downloadResultQueue = downloadResultQueue
- self.logger = logging.getLogger('subliminal.worker')
- self.plugins = {}
+def create_download_tasks(subtitles_by_video, multi):
+ """Create a list of :class:`~subliminal.tasks.DownloadTask` from a list results grouped by video
- def run(self):
- while True:
- task = self.taskQueue.get()[1]
- if isinstance(task, StopTask):
- self.logger.debug(u'Poison pill received in thread %s' % self.name)
- self.taskQueue.task_done()
+ :param subtitles_by_video: :class:`~subliminal.tasks.ListTask` results grouped by video and sorted
+ :type subtitles_by_video: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`]
+ :param order: preferred order for subtitles sorting
+ :type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE`
+ :param bool multi: download multiple languages for the same video
+ :return: the created tasks
+ :rtype: list of :class:`~subliminal.tasks.DownloadTask`
+
+ """
+ tasks = []
+ for video, subtitles in subtitles_by_video.iteritems():
+ if not subtitles:
+ continue
+ if not multi:
+ task = DownloadTask(video, list(subtitles))
+ logger.debug(u'Created task %r' % task)
+ tasks.append(task)
+ continue
+ for _, by_language in groupby(subtitles, lambda s: s.language):
+ task = DownloadTask(video, list(by_language))
+ logger.debug(u'Created task %r' % task)
+ tasks.append(task)
+ return tasks
+
+
+def consume_task(task, services=None):
+ """Consume a task. If the ``services`` parameter is given, the function will attempt
+ to get the service from it. In case the service is not in ``services``, it will be initialized
+ and put in ``services``
+
+ :param task: task to consume
+ :type task: :class:`~subliminal.tasks.ListTask` or :class:`~subliminal.tasks.DownloadTask`
+ :param dict services: mapping between the service name and an instance of this service
+ :return: the result of the task
+ :rtype: list of :class:`~subliminal.subtitles.ResultSubtitle` or :class:`~subliminal.subtitles.Subtitle`
+
+ """
+ if services is None:
+ services = {}
+ logger.info(u'Consuming %r' % task)
+ result = None
+ if isinstance(task, ListTask):
+ if task.service not in services:
+ mod = __import__('services.' + task.service, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
+ services[task.service] = mod.Service(task.config)
+ services[task.service].init()
+ subtitles = services[task.service].list(task.video, task.languages)
+ result = subtitles
+ elif isinstance(task, DownloadTask):
+ for subtitle in task.subtitles:
+ if subtitle.service not in services:
+ mod = __import__('services.' + subtitle.service, globals=globals(), locals=locals(), fromlist=['Service'], level=-1)
+ services[subtitle.service] = mod.Service()
+ services[subtitle.service].init()
+ try:
+ services[subtitle.service].download(subtitle)
+ result = subtitle
break
- result = []
- try:
- if isinstance(task, ListTask):
- if task.plugin not in self.plugins: # init the plugin
- self.plugins[task.plugin] = getattr(plugins, task.plugin)()
- self.plugins[task.plugin].init()
- # Retrieve the plugin list subtitles and return [(video, [subtitle])]
- plugin = self.plugins[task.plugin]
- plugin.config = task.config
- subtitles = plugin.list(task.video, task.languages)
- result = [(task.video, subtitles)]
- elif isinstance(task, DownloadTask):
- # Attempt to download one subtitle from the given list
- for subtitle in task.subtitles:
- if subtitle.plugin not in self.plugins: # init the plugin
- self.plugins[subtitle.plugin] = getattr(plugins, subtitle.plugin)()
- self.plugins[subtitle.plugin].init()
- plugin = self.plugins[subtitle.plugin]
- try:
- result = [plugin.download(subtitle)]
- break
- except DownloadFailedError: # try the next one
- self.logger.warning(u'Could not download subtitle %r, trying next' % subtitle)
- continue
- if not result:
- self.logger.error(u'No subtitles could be downloaded for video %r' % task.video.path or task.video.release)
- except:
- self.logger.error(u'Exception raised in worker %s' % self.name, exc_info=True)
- finally:
- # Put the result in the correct queue
- if isinstance(task, ListTask):
- self.listResultQueue.put(result)
- elif isinstance(task, DownloadTask):
- self.downloadResultQueue.put(result)
- self.taskQueue.task_done()
- self.terminate()
- self.logger.debug(u'Thread %s terminated' % self.name)
-
- def terminate(self):
- """Terminate instanciated plugins"""
- for plugin_name, plugin in self.plugins.iteritems():
- try:
- plugin.terminate()
- except:
- self.logger.error(u'Exception raised when terminating plugin %s' % plugin_name, exc_info=True)
+ except DownloadFailedError:
+ logger.warning(u'Could not download subtitle %r, trying next' % subtitle)
+ continue
+ if result is None:
+ logger.error(u'No subtitles could be downloaded for video %r' % task.video)
+ return result
def matching_confidence(video, subtitle):
- '''Compute the confidence that the subtitle matches the video.
- Returns a float between 0 and 1. 1 being the perfect match.'''
+ """Compute the probability (confidence) that the subtitle matches the video
+
+ :param video: video to match
+ :type video: :class:`~subliminal.videos.Video`
+ :param subtitle: subtitle to match
+ :type subtitle: :class:`~subliminal.subtitles.Subtitle`
+ :return: the matching probability
+ :rtype: float
+
+ """
guess = guessit.guess_file_info(subtitle.release, 'autodetect')
- video_keywords = utils.get_keywords(video.guess)
- subtitle_keywords = utils.get_keywords(guess) | subtitle.keywords
+ video_keywords = get_keywords(video.guess)
+ subtitle_keywords = get_keywords(guess) | subtitle.keywords
replacement = {'keywords': len(video_keywords & subtitle_keywords)}
- if isinstance(video, videos.Episode):
+ if isinstance(video, Episode):
replacement.update({'series': 0, 'season': 0, 'episode': 0})
matching_format = '{series:b}{season:b}{episode:b}{keywords:03b}'
best = matching_format.format(series=1, season=1, episode=1, keywords=len(video_keywords))
@@ -351,7 +178,7 @@ def matching_confidence(video, subtitle):
replacement['season'] = 1
if 'episodeNumber' in guess and guess['episodeNumber'] == video.episode:
replacement['episode'] = 1
- elif isinstance(video, videos.Movie):
+ elif isinstance(video, Movie):
replacement.update({'title': 0, 'year': 0})
matching_format = '{title:b}{year:b}{keywords:03b}'
best = matching_format.format(title=1, year=1, keywords=len(video_keywords))
@@ -364,3 +191,50 @@ def matching_confidence(video, subtitle):
return 0
confidence = float(int(matching_format.format(**replacement), 2)) / float(int(best, 2))
return confidence
+
+
+def key_subtitles(subtitle, video, languages, services, order):
+ """Create a key to sort subtitle using the given order
+
+ :param subtitle: subtitle to sort
+ :type subtitle: :class:`~subliminal.subtitles.ResultSubtitle`
+ :param video: video to match
+ :type video: :class:`~subliminal.videos.Video`
+ :param list languages: languages in preferred order
+ :param list services: services in preferred order
+ :param order: preferred order for subtitles sorting
+ :type list: list of :data:`LANGUAGE_INDEX`, :data:`SERVICE_INDEX`, :data:`SERVICE_CONFIDENCE`, :data:`MATCHING_CONFIDENCE`
+ :return: a key ready to use for subtitles sorting
+ :rtype: int
+
+ """
+ key = ''
+ for sort_item in order:
+ if sort_item == LANGUAGE_INDEX:
+ key += '{0:03d}'.format(len(languages) - languages.index(subtitle.language) - 1)
+ elif sort_item == SERVICE_INDEX:
+ key += '{0:02d}'.format(len(services) - services.index(subtitle.service) - 1)
+ elif sort_item == SERVICE_CONFIDENCE:
+ key += '{0:04d}'.format(int(subtitle.confidence * 1000))
+ elif sort_item == MATCHING_CONFIDENCE:
+ confidence = 0
+ if subtitle.release:
+ confidence = matching_confidence(video, subtitle)
+ key += '{0:04d}'.format(int(confidence * 1000))
+ return int(key)
+
+
+def group_by_video(list_results):
+ """Group the results of :class:`ListTasks ` into a
+ dictionary of :class:`~subliminal.videos.Video` => :class:`~subliminal.subtitles.Subtitle`
+
+ :param list_results:
+ :type list_results: list of result of :class:`~subliminal.tasks.ListTask`
+ :return: subtitles grouped by videos
+ :rtype: dict of :class:`~subliminal.videos.Video` => [:class:`~subliminal.subtitles.Subtitle`]
+
+ """
+ result = defaultdict(list)
+ for video, subtitles in list_results:
+ result[video] += subtitles
+ return result
diff --git a/libs/subliminal/exceptions.py b/libs/subliminal/exceptions.py
index 29b417f7..93001110 100755
--- a/libs/subliminal/exceptions.py
+++ b/libs/subliminal/exceptions.py
@@ -1,23 +1,20 @@
# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
#
-# Subliminal - Subtitles, faster than your thoughts
-# Copyright (c) 2011 Antoine Bertin
+# This file is part of subliminal.
#
-# This file is part of Subliminal.
-#
-# Subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Subliminal is distributed in the hope that it will be useful,
+# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
-#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
+# GNU Lesser General Public License for more details.
#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
class Error(Exception):
@@ -25,21 +22,6 @@ class Error(Exception):
pass
-class BadStateError(Error):
- """Exception raised when an invalid action is asked
-
- Attributes:
- current -- current state of Subliminal instance
- expected -- expected state of Subliminal instance
- """
- def __init__(self, current, expected):
- self.current = current
- self.expected = expected
-
- def __str__(self):
- return 'Expected state %d but current state is %d' % (self.expected, self.current)
-
-
class InvalidLanguageError(Error):
"""Exception raised when invalid language is submitted
@@ -66,21 +48,21 @@ class MissingLanguageError(Error):
return self.language
-class InvalidPluginError(Error):
- """"Exception raised when invalid plugin is submitted
+class InvalidServiceError(Error):
+ """Exception raised when invalid service is submitted
+
+ :param string service: service that causes the error
- Attributes:
- plugin -- plugin that cause the error
"""
- def __init__(self, plugin):
- self.plugin = plugin
+ def __init__(self, service):
+ self.service = service
def __str__(self):
- return self.plugin
+ return self.service
-class PluginError(Error):
- """"Exception raised by plugins"""
+class ServiceError(Error):
+ """"Exception raised by services"""
pass
@@ -90,7 +72,7 @@ class WrongTaskError(Error):
class DownloadFailedError(Error):
- """"Exception raised when a download task has failed in plugin"""
+ """"Exception raised when a download task has failed in service"""
pass
diff --git a/libs/subliminal/infos.py b/libs/subliminal/infos.py
index 32ea82f1..b28fda00 100755
--- a/libs/subliminal/infos.py
+++ b/libs/subliminal/infos.py
@@ -1,26 +1,18 @@
# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
#
-# Subliminal - Subtitles, faster than your thoughts
-# Copyright (c) 2011 Antoine Bertin
+# This file is part of subliminal.
#
-# This file is part of Subliminal.
-#
-# Subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Subliminal is distributed in the hope that it will be useful,
+# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
+# GNU Lesser General Public License for more details.
#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
-#
-__title__ = 'subliminal'
-__description__ = 'Subtitles, faster than your thoughts'
-__version__ = '0.5'
-__author__ = 'Antoine Bertin'
-__license__ = 'LGPLv3'
-__copyright__ = 'Copyright 2010-2011 Antoine Bertin'
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+__version__ = '0.5.1'
diff --git a/libs/subliminal/languages.py b/libs/subliminal/languages.py
index d38d48ba..f743953f 100755
--- a/libs/subliminal/languages.py
+++ b/libs/subliminal/languages.py
@@ -1,29 +1,34 @@
# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
#
-# Subliminal - Subtitles, faster than your thoughts
-# Copyright (c) 2011 Antoine Bertin
+# This file is part of subliminal.
#
-# This file is part of Subliminal.
-#
-# Subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Subliminal is distributed in the hope that it will be useful,
+# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
-#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
+# GNU Lesser General Public License for more details.
#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
__all__ = ['convert_language', 'list_languages', 'LANGUAGES']
def convert_language(language, to_iso, from_iso=None):
- # if no from_iso is given, try to guess it
- if from_iso == None:
+ """Convert a language into another format
+
+ :param string language: language
+ :param int to_iso: convert language to ISO-639-x
+ :param int from_iso: convert language from ISO-639-x
+ :return: converted language
+ :rtype: string
+
+ """
+ if from_iso == None: # if no from_iso is given, try to guess it
if language.startswith(language[:1].upper()):
from_iso = 0
elif len(language) == 2:
@@ -43,10 +48,17 @@ def convert_language(language, to_iso, from_iso=None):
def list_languages(iso):
+ """List languages in the given ISO-639-x format
+
+ :param int iso: ISO-639-x format to list
+ :return: languages in the requested format
+ :rtype: list
+
+ """
return [l[iso] for l in LANGUAGES if l[iso]]
-# ISO-639-2 languages list from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
-# + ('Brazilian', 'po', 'pob')
+#: ISO-639-2 languages list from http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
+#: + ('Brazilian', 'po', 'pob')
LANGUAGES = [('Afar', 'aa', 'aar'),
('Abkhazian', 'ab', 'abk'),
('Achinese', '', 'ace'),
diff --git a/libs/subliminal/plugins.py b/libs/subliminal/plugins.py
deleted file mode 100755
index 1b2dc04d..00000000
--- a/libs/subliminal/plugins.py
+++ /dev/null
@@ -1,890 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Subliminal - Subtitles, faster than your thoughts
-# Copyright (c) 2011 Antoine Bertin
-#
-# This file is part of Subliminal.
-#
-# Subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
-# the Free Software Foundation; either version 3 of the License, or
-# (at your option) any later version.
-#
-# Subliminal is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
-#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
-#
-__all__ = ['PluginBase', 'OpenSubtitles', 'BierDopje', 'TheSubDB', 'SubsWiki', 'Subtitulos']
-
-
-from exceptions import DownloadFailedError, MissingLanguageError, PluginError
-from utils import get_keywords, PluginConfig, split_keyword
-from videos import Episode, Movie, UnknownVideo
-from subtitles import ResultSubtitle, get_subtitle_path
-import BeautifulSoup
-import abc
-import gzip
-import logging
-import os
-import re
-import requests
-import suds.client
-import threading
-import unicodedata
-import urllib
-import xmlrpclib
-try:
- import cPickle as pickle
-except ImportError:
- import pickle
-
-
-#TODO: use ISO-639-2 in plugins instead of ISO-639-1
-class PluginBase(object):
- __metaclass__ = abc.ABCMeta
- site_url = ''
- site_name = ''
- server_url = ''
- user_agent = 'Subliminal v0.5'
- api_based = False
- timeout = 5
- lock = threading.Lock()
- languages = {}
- reverted_languages = False
- videos = []
- require_video = False
- shared_support = False
-
- @abc.abstractmethod
- def __init__(self, config=None):
- self.config = config or PluginConfig()
- self.logger = logging.getLogger('subliminal.%s' % self.__class__.__name__)
-
- @abc.abstractmethod
- def init(self):
- """Initiate connection"""
- self.session = requests.session(timeout=10, headers={'User-Agent': self.user_agent})
-
- @abc.abstractmethod
- def terminate(self):
- """Terminate connection"""
-
- @abc.abstractmethod
- def query(self, *args):
- """Make the actual query"""
-
- @abc.abstractmethod
- def list(self, video, languages):
- """List subtitles"""
-
- @abc.abstractmethod
- def download(self, subtitle):
- """Download a subtitle"""
-
- @classmethod
- def availableLanguages(cls):
- if not cls.reverted_languages:
- return set(cls.languages.keys())
- if cls.reverted_languages:
- return set(cls.languages.values())
-
- @classmethod
- def isValidVideo(cls, video):
- if cls.require_video and not video.exists:
- return False
- if not isinstance(video, tuple(cls.videos)):
- return False
- return True
-
- @classmethod
- def isValidLanguage(cls, language):
- if language in cls.availableLanguages():
- return True
- return False
-
- @classmethod
- def getRevertLanguage(cls, language):
- """ISO-639-1 language code from plugin language code"""
- if not cls.reverted_languages and language in cls.languages.values():
- return [k for k, v in cls.languages.iteritems() if v == language][0]
- if cls.reverted_languages and language in cls.languages.keys():
- return cls.languages[language]
- raise MissingLanguageError(language)
-
- @classmethod
- def getLanguage(cls, language):
- """Plugin language code from ISO-639-1 language code"""
- if not cls.reverted_languages and language in cls.languages.keys():
- return cls.languages[language]
- if cls.reverted_languages and language in cls.languages.values():
- return [k for k, v in cls.languages.iteritems() if v == language][0]
- raise MissingLanguageError(language)
-
- def adjustPermissions(self, filepath):
- if self.config.filemode != None:
- os.chmod(filepath, self.config.filemode)
-
- def downloadFile(self, url, filepath):
- """Download a subtitle file"""
- self.logger.info(u'Downloading %s' % url)
- try:
- r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
- with open(filepath, 'wb') as f:
- f.write(r.content)
- except Exception as e:
- self.logger.error(u'Download %s failed: %s' % (url, e))
- if os.path.exists(filepath):
- os.remove(filepath)
- raise DownloadFailedError(str(e))
- self.logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
-
-
-class OpenSubtitles(PluginBase):
- site_url = 'http://www.opensubtitles.org'
- site_name = 'OpenSubtitles'
- server_url = 'http://api.opensubtitles.org/xml-rpc'
- user_agent = 'Subliminal v0.5'
- api_based = True
- languages = {'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'alb', 'am': 'amh', 'ar': 'ara',
- 'an': 'arg', 'hy': 'arm', 'as': 'asm', 'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze',
- 'ba': 'bak', 'bm': 'bam', 'eu': 'baq', 'be': 'bel', 'bn': 'ben', 'bh': 'bih', 'bi': 'bis',
- 'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'bur', 'ca': 'cat', 'ch': 'cha', 'ce': 'che',
- 'zh': 'chi', 'cu': 'chu', 'cv': 'chv', 'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'cze',
- 'da': 'dan', 'dv': 'div', 'nl': 'dut', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo', 'et': 'est',
- 'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin', 'fr': 'fre', 'fy': 'fry', 'ff': 'ful',
- 'ka': 'geo', 'de': 'ger', 'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell',
- 'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb', 'hz': 'her', 'hi': 'hin',
- 'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun', 'ig': 'ibo', 'is': 'ice', 'io': 'ido', 'ii': 'iii',
- 'iu': 'iku', 'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita', 'jv': 'jav',
- 'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas', 'kr': 'kau', 'kk': 'kaz', 'km': 'khm',
- 'ki': 'kik', 'rw': 'kin', 'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua',
- 'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim', 'ln': 'lin', 'lt': 'lit',
- 'lb': 'ltz', 'lu': 'lub', 'lg': 'lug', 'mk': 'mac', 'mh': 'mah', 'ml': 'mal', 'mi': 'mao',
- 'mr': 'mar', 'ms': 'may', 'mg': 'mlg', 'mt': 'mlt', 'mo': 'mol', 'mn': 'mon', 'na': 'nau',
- 'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep', 'nn': 'nno', 'nb': 'nob',
- 'no': 'nor', 'ny': 'nya', 'oc': 'oci', 'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss',
- 'pa': 'pan', 'fa': 'per', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus', 'qu': 'que',
- 'rm': 'roh', 'rn': 'run', 'ru': 'rus', 'sg': 'sag', 'sa': 'san', 'sr': 'scc', 'si': 'sin',
- 'sk': 'slo', 'sl': 'slv', 'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som',
- 'st': 'sot', 'es': 'spa', 'sc': 'srd', 'ss': 'ssw', 'su': 'sun', 'sw': 'swa', 'sv': 'swe',
- 'ty': 'tah', 'ta': 'tam', 'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha',
- 'bo': 'tib', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso', 'tk': 'tuk', 'tr': 'tur',
- 'tw': 'twi', 'ug': 'uig', 'uk': 'ukr', 'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie',
- 'vo': 'vol', 'cy': 'wel', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid', 'yo': 'yor',
- 'za': 'zha', 'zu': 'zul', 'ro': 'rum', 'po': 'pob', 'un': 'unk', 'ay': 'ass'}
- reverted_languages = False
- videos = [Episode, Movie]
- require_video = False
- confidence_order = ['moviehash', 'imdbid', 'fulltext']
-
- def __init__(self, config=None):
- super(OpenSubtitles, self).__init__(config)
- self.server = xmlrpclib.ServerProxy(self.server_url)
- self.token = None
-
- def __enter__(self):
- self.init()
- return self
-
- def __exit__(self, *args):
- self.terminate()
-
- def init(self):
- self.logger.debug(u'Initializing')
- super(OpenSubtitles, self).init()
- result = self.server.LogIn('', '', 'eng', self.user_agent)
- if result['status'] != '200 OK':
- raise PluginError('Login failed')
- self.token = result['token']
-
- def terminate(self):
- self.logger.debug(u'Terminating')
- if self.token:
- self.server.LogOut(self.token)
-
- def query(self, filepath, languages, moviehash=None, size=None, imdbid=None, query=None):
- searches = []
- if moviehash and size:
- searches.append({'moviehash': moviehash, 'moviebytesize': size})
- if imdbid:
- searches.append({'imdbid': imdbid})
- if query:
- searches.append({'query': query})
- if not searches:
- raise PluginError('One or more parameter missing')
- for search in searches:
- search['sublanguageid'] = ','.join([self.getLanguage(l) for l in languages])
- self.logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token))
- results = self.server.SearchSubtitles(self.token, searches)
- if not results['data']:
- self.logger.debug(u'Could not find subtitles for %r with token %s' % (searches, self.token))
- return []
- subtitles = []
- for result in results['data']:
- language = self.getRevertLanguage(result['SubLanguageID'])
- path = get_subtitle_path(filepath, language, self.config.multi)
- confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
- subtitle = ResultSubtitle(path, language, self.__class__.__name__, result['SubDownloadLink'], result['SubFileName'], confidence)
- subtitles.append(subtitle)
- return subtitles
-
- def list(self, video, languages):
- languages = languages & self.availableLanguages()
- if not languages:
- self.logger.debug(u'No language available')
- return []
- if not self.isValidVideo(video):
- self.logger.debug(u'Not a valid video')
- return []
- results = []
- if video.exists:
- results = self.query(video.path or video.release, languages, moviehash=video.hashes['OpenSubtitles'], size=str(video.size))
- elif video.imdbid:
- results = self.query(video.path or video.release, languages, imdbid=video.imdbid)
- elif isinstance(video, Episode):
- results = self.query(video.path or video.release, languages, query=video.series)
- elif isinstance(video, Movie):
- results = self.query(video.path or video.release, languages, query=video.title)
- return results
-
- def download(self, subtitle):
- #TODO: Use OpenSubtitles DownloadSubtitles method
- try:
- self.downloadFile(subtitle.link, subtitle.path + '.gz')
- with open(subtitle.path, 'wb') as dump:
- gz = gzip.open(subtitle.path + '.gz')
- dump.write(gz.read())
- gz.close()
- self.adjustPermissions(subtitle.path)
- except Exception as e:
- if os.path.exists(subtitle.path):
- os.remove(subtitle.path)
- raise DownloadFailedError(str(e))
- finally:
- if os.path.exists(subtitle.path + '.gz'):
- os.remove(subtitle.path + '.gz')
- return subtitle
-
-
-class BierDopje(PluginBase):
- site_url = 'http://bierdopje.com'
- site_name = 'BierDopje'
- server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/'
- api_based = True
- languages = {'en': 'en', 'nl': 'nl'}
- reverted_languages = False
- videos = [Episode]
- require_video = False
-
- def __init__(self, config=None):
- super(BierDopje, self).__init__(config)
- self.showids = {}
- if self.config and self.config.cache_dir:
- self.initCache()
-
- def __enter__(self):
- self.init()
- return self
-
- def __exit__(self, *args):
- self.terminate()
-
- def init(self):
- self.logger.debug(u'Initializing')
- super(BierDopje, self).init()
-
- def terminate(self):
- self.logger.debug(u'Terminating')
-
- def initCache(self):
- self.logger.debug(u'Initializing cache...')
- if not self.config or not self.config.cache_dir:
- raise PluginError('Cache directory is required')
- self.showids_cache = os.path.join(self.config.cache_dir, 'bierdopje_showids.cache')
- if not os.path.exists(self.showids_cache):
- self.saveToCache()
-
- def saveToCache(self):
- self.logger.debug(u'Saving showids to cache...')
- with self.lock:
- with open(self.showids_cache, 'w') as f:
- pickle.dump(self.showids, f)
-
- def loadFromCache(self):
- self.logger.debug(u'Loading showids from cache...')
- with self.lock:
- with open(self.showids_cache, 'r') as f:
- self.showids = pickle.load(f)
-
- def query(self, season, episode, languages, filepath, tvdbid=None, series=None):
- self.initCache()
- self.loadFromCache()
- if series:
- if series.lower() in self.showids: # from cache
- request_id = self.showids[series.lower()]
- self.logger.debug(u'Retreived showid %d for %s from cache' % (request_id, series))
- else: # query to get showid
- self.logger.debug(u'Getting showid from show name %s...' % series)
- r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower())))
- if r.status_code != 200:
- self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
- return []
- soup = BeautifulSoup.BeautifulStoneSoup(r.content)
- if soup.status.contents[0] == 'false':
- self.logger.debug(u'Could not find show %s' % series)
- return []
- request_id = int(soup.showid.contents[0])
- self.showids[series.lower()] = request_id
- self.saveToCache()
- request_source = 'showid'
- request_is_tvdbid = 'false'
- elif tvdbid:
- request_id = tvdbid
- request_source = 'tvdbid'
- request_is_tvdbid = 'true'
- else:
- raise PluginError('One or more parameter missing')
- subtitles = []
- for language in languages:
- self.logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
- r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language, request_is_tvdbid))
- if r.status_code != 200:
- self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
- return []
- soup = BeautifulSoup.BeautifulStoneSoup(r.content)
- if soup.status.contents[0] == 'false':
- self.logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
- continue
- path = get_subtitle_path(filepath, language, self.config.multi)
- for result in soup.results('result'):
- subtitle = ResultSubtitle(path, language, self.__class__.__name__, result.downloadlink.contents[0], result.filename.contents[0])
- subtitles.append(subtitle)
- return subtitles
-
- def list(self, video, languages):
- languages = languages & self.availableLanguages()
- if not languages:
- self.logger.debug(u'No language available')
- return []
- if not self.isValidVideo(video):
- self.logger.debug(u'Not a valid video')
- return []
- results = self.query(video.season, video.episode, languages, video.path or video.release, video.tvdbid, video.series)
- return results
-
- def download(self, subtitle):
- self.downloadFile(subtitle.link, subtitle.path)
- return subtitle
-
-
-class TheSubDB(PluginBase):
- site_url = 'http://thesubdb.com'
- site_name = 'SubDB'
- server_url = 'http://api.thesubdb.com/' # for testing purpose, use http://sandbox.thesubdb.com/ instead
- api_based = True
- user_agent = 'SubDB/1.0 (Subliminal/0.5; https://github.com/Diaoul/subliminal)' # defined by the API
- languages = {'af': 'af', 'cs': 'cs', 'da': 'da', 'de': 'de', 'en': 'en', 'es': 'es', 'fi': 'fi',
- 'fr': 'fr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'la': 'la', 'nl': 'nl', 'no': 'no',
- 'oc': 'oc', 'pl': 'pl', 'pt': 'pt', 'ro': 'ro', 'ru': 'ru', 'sl': 'sl', 'sr': 'sr',
- 'sv': 'sv', 'tr': 'tr'} # list available with the API at http://sandbox.thesubdb.com/?action=languages
- reverted_languages = False
- videos = [Movie, Episode, UnknownVideo]
- require_video = True
-
- def __init__(self, config=None):
- super(TheSubDB, self).__init__(config)
-
- def __enter__(self):
- self.init()
- return self
-
- def __exit__(self, *args):
- self.terminate()
-
- def init(self):
- self.logger.debug(u'Initializing')
- super(TheSubDB, self).init()
-
- def terminate(self):
- self.logger.debug(u'Terminating')
-
- def list(self, video, languages):
- languages = languages & self.availableLanguages()
- if not languages:
- self.logger.debug(u'No language available')
- return []
- if not self.isValidVideo(video):
- self.logger.debug(u'Not a valid video')
- return []
- results = self.query(video.path, video.hashes['TheSubDB'], languages)
- return results
-
- def query(self, filepath, moviehash, languages):
- r = self.session.get(self.server_url, params={'action': 'search', 'hash': moviehash})
- if r.status_code == 404:
- self.logger.debug(u'Could not find subtitles for hash %s' % moviehash)
- return []
- if r.status_code != 200:
- self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
- return []
- available_languages = set([self.getRevertLanguage(l) for l in r.content.split(',')])
- filtered_languages = languages & available_languages
- if not filtered_languages:
- self.logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages))
- return []
- subtitles = []
- for language in filtered_languages:
- path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(path, language, self.__class__.__name__, '%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, self.getLanguage(language)))
- subtitles.append(subtitle)
- return subtitles
-
- def download(self, subtitle):
- self.downloadFile(subtitle.link, subtitle.path)
- return subtitle
-
-
-class SubsWiki(PluginBase):
- site_url = 'http://www.subswiki.com'
- site_name = 'SubsWiki'
- server_url = 'http://www.subswiki.com'
- api_based = False
- languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
- u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
- u'Italian': 'it', u'Català': 'ca'}
- reverted_languages = True
- videos = [Episode, Movie]
- require_video = False
- release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs')
-
- def __init__(self, config=None):
- super(SubsWiki, self).__init__(config)
-
- def __enter__(self):
- self.init()
- return self
-
- def __exit__(self, *args):
- self.terminate()
-
- def init(self):
- self.logger.debug(u'Initializing')
- super(SubsWiki, self).init()
-
- def terminate(self):
- self.logger.debug(u'Terminating')
-
- def list(self, video, languages):
- languages = languages & self.availableLanguages()
- if not languages:
- self.logger.debug(u'No language available')
- return []
- if not self.isValidVideo(video):
- self.logger.debug(u'Not a valid video')
- return []
- results = []
- if isinstance(video, Episode):
- results = self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode)
- elif isinstance(video, Movie) and video.year:
- results = self.query(video.path or video.release, languages, get_keywords(video.guess), movie=video.title, year=video.year)
- return results
-
- def query(self, filepath, languages, keywords=None, series=None, season=None, episode=None, movie=None, year=None):
- if series and season and episode:
- request_series = series.lower().replace(' ', '_')
- if isinstance(request_series, unicode):
- request_series = request_series.encode('utf-8')
- self.logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
- r = self.session.get('%s/serie/%s/%s/%s/' % (self.server_url, urllib.quote(request_series), season, episode))
- if r.status_code == 404:
- self.logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
- return []
- elif movie and year:
- request_movie = movie.title().replace(' ', '_')
- if isinstance(request_movie, unicode):
- request_movie = request_movie.encode('utf-8')
- self.logger.debug(u'Getting subtitles for %s (%d) with languages %r' % (movie, year, languages))
- r = self.session.get('%s/film/%s_(%d)' % (self.server_url, urllib.quote(request_movie), year))
- if r.status_code == 404:
- self.logger.debug(u'Could not find subtitles for %s (%d) with languages %r' % (movie, year, languages))
- return []
- else:
- raise PluginError('One or more parameter missing')
- if r.status_code != 200:
- self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
- return []
- soup = BeautifulSoup.BeautifulSoup(r.content)
- subtitles = []
- for sub in soup('td', {'class': 'NewsTitle'}):
- sub_keywords = split_keyword(self.release_pattern.search(sub.contents[1]).group(1).lower())
- if not keywords & sub_keywords:
- self.logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
- continue
- for html_language in sub.parent.parent.findAll('td', {'class': 'language'}):
- language = self.getRevertLanguage(html_language.string.strip())
- if not language in languages:
- self.logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
- continue
- html_status = html_language.findNextSibling('td')
- status = html_status.find('strong').string.strip()
- if status != 'Completed':
- self.logger.debug(u'Wrong subtitle status %s' % status)
- continue
- path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(path, language, self.__class__.__name__, '%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
- subtitles.append(subtitle)
- return subtitles
-
- def download(self, subtitle):
- self.downloadFile(subtitle.link, subtitle.path)
- return subtitle
-
-
-class Subtitulos(PluginBase):
- site_url = 'http://www.subtitulos.es/'
- site_name = 'Subtitulos'
- server_url = 'http://www.subtitulos.es'
- api_based = False
- languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
- u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
- u'Italian': 'it', u'Català': 'ca'}
- reverted_languages = True
- videos = [Episode]
- require_video = False
- release_pattern = re.compile('Versión (.+) ([0-9]+).([0-9])+ megabytes')
-
- def __init__(self, config=None):
- super(Subtitulos, self).__init__(config)
-
- def __enter__(self):
- self.init()
- return self
-
- def __exit__(self, *args):
- self.terminate()
-
- def init(self):
- self.logger.debug(u'Initializing')
- super(Subtitulos, self).init()
-
- def terminate(self):
- self.logger.debug(u'Terminating')
-
- def list(self, video, languages):
- languages = languages & self.availableLanguages()
- if not languages:
- self.logger.debug(u'No language available')
- return []
- if not self.isValidVideo(video):
- self.logger.debug(u'Not a valid video')
- return []
- results = self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
- return results
-
- def query(self, filepath, languages, keywords, series, season, episode):
- request_series = series.lower().replace(' ', '_')
- if isinstance(request_series, unicode):
- request_series = unicodedata.normalize('NFKD', request_series).encode('ascii', 'ignore')
- self.logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
- r = self.session.get('%s/%s/%sx%.2d' % (self.server_url, urllib.quote(request_series), season, episode))
- if r.status_code == 404:
- self.logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
- return []
- if r.status_code != 200:
- self.logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
- return []
- soup = BeautifulSoup.BeautifulSoup(r.content)
- subtitles = []
- for sub in soup('div', {'id': 'version'}):
- sub_keywords = split_keyword(self.release_pattern.search(sub.find('p', {'class': 'title-sub'}).contents[1]).group(1).lower())
- if not keywords & sub_keywords:
- self.logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
- continue
- for html_language in sub.findAllNext('ul', {'class': 'sslist'}):
- language = self.getRevertLanguage(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
- if not language in languages:
- self.logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
- continue
- html_status = html_language.findNext('li', {'class': 'li-estado green'})
- status = html_status.contents[0].string.strip()
- if status != 'Completado':
- self.logger.debug(u'Wrong subtitle status %s' % status)
- continue
- path = get_subtitle_path(filepath, language, self.config.multi)
- subtitle = ResultSubtitle(path, language, self.__class__.__name__, html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords)
- subtitles.append(subtitle)
- return subtitles
-
- def download(self, subtitle):
- self.downloadFile(subtitle.link, subtitle.path)
- return subtitle
-
-
-class GetSubtitle(PluginBase):
- site_url = 'http://www.subtitles.com.br/'
- site_name = 'GetSubtitle'
- server_url = 'http://api.getsubtitle.com/server.php?wsdl'
- api_based = True
- languages = {'sq': 'ALB', 'ar': 'ARA', 'hy': 'ARM', 'bs': 'BOS', 'bg': 'BUL', 'ca': 'CAT', 'zh': 'CHI', 'hr': 'HRV',
- 'cs': 'CZE', 'da': 'DAN', 'nl': 'NLD', 'en': 'ENG', 'eo': 'ESP', 'et': 'EST', 'fi': 'FIN', 'fr': 'FRA',
- 'gl': 'GLG', 'ka': 'GEO', 'de': 'DEU', 'el': 'GRC', 'he': 'ISR', 'hi': 'HIN', 'hu': 'HUN', 'is': 'ISL',
- 'id': 'IND', 'it': 'ITA', 'ja': 'JPN', 'kk': 'KAZ', 'ko': 'KOR', 'lv': 'LVA', 'lt': 'LIT', 'lb': 'LTZ',
- 'mk': 'MKD', 'ms': 'MAY', 'no': 'NOR', 'oc': 'OCC', 'pl': 'POL', 'pt': 'POR', 'ro': 'RUM', 'ru': 'RUS',
- 'sr': 'ZAF', 'sk': 'SLK', 'sl': 'SLV', 'es': 'SPA', 'sv': 'SWE', 'th': 'THA', 'tr': 'TUR', 'uk': 'UKR',
- 'ur': 'URD', 'vi': 'VTN'}
- reverted_languages = False
- videos = [Movie]
- require_video = False
- max_results = 100
-
- def __init__(self, config=None):
- super(GetSubtitle, self).__init__(config)
-
- def __enter__(self):
- self.init()
- return self
-
- def __exit__(self, *args):
- pass
-
- def init(self):
- self.logger.debug(u'Initializing')
- self.server = suds.client.Client(self.server_url)
-
- def terminate(self):
- self.logger.debug(u'Terminating')
-
- def query(self, *args):
- #TODO
- pass
-
- def list(self, video, languages):
- languages = languages & self.availableLanguages()
- if not languages:
- self.logger.debug(u'No language available')
- return []
- if not self.isValidVideo(video):
- self.logger.debug(u'Not a valid video')
- return []
- #TODO
-
- def download(self, subtitle):
- #TODO
- pass
-
-
-'''
-class Addic7ed(PluginBase.PluginBase):
- site_url = 'http://www.addic7ed.com'
- site_name = 'Addic7ed'
- server_url = 'http://www.addic7ed.com'
- api_based = False
- _plugin_languages = {u'English': 'en',
- u'English (US)': 'en',
- u'English (UK)': 'en',
- u'Italian': 'it',
- u'Portuguese': 'pt',
- u'Portuguese (Brazilian)': 'po',
- u'Romanian': 'ro',
- u'Español (Latinoamérica)': 'es',
- u'Español (España)': 'es',
- u'Spanish (Latin America)': 'es',
- u'Español': 'es',
- u'Spanish': 'es',
- u'Spanish (Spain)': 'es',
- u'French': 'fr',
- u'Greek': 'el',
- u'Arabic': 'ar',
- u'German': 'de',
- u'Croatian': 'hr',
- u'Indonesian': 'id',
- u'Hebrew': 'he',
- u'Russian': 'ru',
- u'Turkish': 'tr',
- u'Swedish': 'se',
- u'Czech': 'cs',
- u'Dutch': 'nl',
- u'Hungarian': 'hu',
- u'Norwegian': 'no',
- u'Polish': 'pl',
- u'Persian': 'fa'}
-
- def __init__(self, config_dict=None):
- super(Addic7ed, self).__init__(self._plugin_languages, config_dict, isRevert=True)
- #http://www.addic7ed.com/serie/Smallville/9/11/Absolute_Justice
- self.release_pattern = re.compile(' \nVersion (.+), ([0-9]+).([0-9])+ MBs')
-
- def list(self, filepath, languages):
- if not self.checkLanguages(languages):
- return []
- guess = guessit.guess_file_info(filepath, 'autodetect')
- if guess['type'] != 'episode':
- self.logger.debug(u'Not an episode')
- return []
- # add multiple things to the release group set
- release_group = set()
- if 'releaseGroup' in guess:
- release_group.add(guess['releaseGroup'].lower())
- else:
- if 'title' in guess:
- release_group.add(guess['title'].lower())
- if 'screenSize' in guess:
- release_group.add(guess['screenSize'].lower())
- if 'series' not in guess or len(release_group) == 0:
- self.logger.debug(u'Not enough information to proceed')
- return []
- self.release_group = release_group # used to sort results
- return self.query(guess['series'], guess['season'], guess['episodeNumber'], release_group, filepath, languages)
-
- def query(self, name, season, episode, release_group, filepath, languages=None):
- searchname = name.lower().replace(' ', '_')
- if isinstance(searchname, unicode):
- searchname = searchname.encode('utf-8')
- searchurl = '%s/serie/%s/%s/%s/%s' % (self.server_url, urllib2.quote(searchname), season, episode, urllib2.quote(searchname))
- self.logger.debug(u'Searching in %s' % searchurl)
- try:
- req = urllib2.Request(searchurl, headers={'User-Agent': self.user_agent})
- page = urllib2.urlopen(req, timeout=self.timeout)
- except urllib2.HTTPError as inst:
- self.logger.info(u'Error: %s - %s' % (searchurl, inst))
- return []
- except urllib2.URLError as inst:
- self.logger.info(u'TimeOut: %s' % inst)
- return []
- soup = BeautifulSoup(page.read())
- sublinks = []
- for html_sub in soup('td', {'class': 'NewsTitle', 'colspan': '3'}):
- if not self.release_pattern.match(str(html_sub.contents[1])): # On not needed soup td result
- continue
- sub_teams = self.listTeams([self.release_pattern.match(str(html_sub.contents[1])).groups()[0].lower()], ['.', '_', ' ', '/', '-'])
- if not release_group.intersection(sub_teams): # On wrong team
- continue
- html_language = html_sub.findNext('td', {'class': 'language'})
- sub_language = self.getRevertLanguage(html_language.contents[0].strip().replace(' ', ''))
- if languages and not sub_language in languages: # On wrong language
- continue
- html_status = html_language.findNextSibling('td')
- sub_status = html_status.find('b').string.strip()
- if not sub_status == 'Completed': # On not completed subtitles
- continue
- sub_link = self.server_url + html_status.findNextSibling('td', {'colspan': '3'}).find('a')['href']
- self.logger.debug(u'Found a match with teams: %s' % sub_teams)
- result = Subtitle(filepath, self.getSubtitlePath(filepath, sub_language), self.__class__.__name__, sub_language, sub_link, keywords=sub_teams)
- sublinks.append(result)
- sublinks.sort(self._cmpReleaseGroup)
- return sublinks
-
- def download(self, subtitle):
- self.downloadFile(subtitle.link, subtitle.path)
- return subtitle
-
-
-class Podnapisi(PluginBase.PluginBase):
- site_url = "http://www.podnapisi.net"
- site_name = "Podnapisi"
- server_url = 'http://ssp.podnapisi.net:8000'
- api_based = True
- _plugin_languages = {"sl": "1",
- "en": "2",
- "no": "3",
- "ko": "4",
- "de": "5",
- "is": "6",
- "cs": "7",
- "fr": "8",
- "it": "9",
- "bs": "10",
- "ja": "11",
- "ar": "12",
- "ro": "13",
- "es-ar": "14",
- "hu": "15",
- "el": "16",
- "zh": "17",
- "lt": "19",
- "et": "20",
- "lv": "21",
- "he": "22",
- "nl": "23",
- "da": "24",
- "se": "25",
- "pl": "26",
- "ru": "27",
- "es": "28",
- "sq": "29",
- "tr": "30",
- "fi": "31",
- "pt": "32",
- "bg": "33",
- "mk": "35",
- "sk": "37",
- "hr": "38",
- "zh": "40",
- "hi": "42",
- "th": "44",
- "uk": "46",
- "sr": "47",
- "po": "48",
- "ga": "49",
- "be": "50",
- "vi": "51",
- "fa": "52",
- "ca": "53",
- "id": "54"}
-
- def __init__(self, config_dict=None):
- super(Podnapisi, self).__init__(self._plugin_languages, config_dict)
- # Podnapisi uses two reference for latin serbian and cyrillic serbian (36 and 47)
- # add the 36 manually as cyrillic seems to be more used
- self.revertPluginLanguages["36"] = "sr"
-
- def list(self, filenames, languages):
- """Main method to call when you want to list subtitles"""
- filepath = filenames[0]
- if not os.path.isfile(filepath):
- return []
- return self.query(self.hashFile(filepath), languages)
-
- def download(self, subtitle):
- return []
-
- def query(self, moviehash, languages=None):
- """Makes a query on podnapisi and returns info (link, lang) about found subtitles"""
- # login
- self.server = xmlrpclib.ServerProxy(self.server_url)
- try:
- log_result = self.server.initiate(self.user_agent)
- self.logger.debug(u"Result: %s" % log_result)
- token = log_result["session"]
- nonce = log_result["nonce"]
- except Exception:
- self.logger.error(u"Cannot login" % log_result)
- return []
- username = 'getmesubs'
- password = '99D31$$'
- hash = md5()
- hash.update(password)
- password = hash.hexdigest()
- hash = sha256()
- hash.update(password)
- hash.update(nonce)
- password = hash.hexdigest()
- self.server.authenticate(token, username, password)
- self.logger.debug(u'Authenticated')
- #if languages:
- # self.logger.debug([self.getLanguage(l) for l in languages])
- # self.server.setFilters(token, [self.getLanguage(l) for l in languages])
- # self.logger.debug('Filers set for languages %s' % languages)
- self.logger.debug(u"Starting search with token %s and hashs %s" % (token, [moviehash]))
- results = self.server.search(token, [moviehash])
- return results
- subs = []
- for sub in results['results']:
- subs.append(sub)
- self.server.terminate(token)
- return subs
-'''
diff --git a/libs/subliminal/services/__init__.py b/libs/subliminal/services/__init__.py
new file mode 100755
index 00000000..67e457fe
--- /dev/null
+++ b/libs/subliminal/services/__init__.py
@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from ..exceptions import MissingLanguageError, DownloadFailedError
+import logging
+import os
+import requests
+import threading
+
+
+__all__ = ['ServiceBase', 'ServiceConfig']
+logger = logging.getLogger(__name__)
+
+
+class ServiceBase(object):
+ """Service base class
+
+ :param config: service configuration
+ :type config: :class:`ServiceConfig`
+
+ """
+ #: URL to the service server
+ server_url = ''
+
+ #: User Agent for any HTTP-based requests
+ user_agent = 'subliminal v0.5'
+
+ #: Whether based on an API or not
+ api_based = False
+
+ #: Timeout for web requests
+ timeout = 5
+
+ #: Lock for cache interactions
+ lock = threading.Lock()
+
+ #: Mapping to Service's language codes and subliminal's
+ languages = {}
+
+ #: Whether the mapping is reverted or not
+ reverted_languages = False
+
+ #: Accepted video classes (:class:`~subliminal.videos.Episode`, :class:`~subliminal.videos.Movie`, :class:`~subliminal.videos.UnknownVideo`)
+ videos = []
+
+ #: Whether the video has to exist or not
+ require_video = False
+
+ def __init__(self, config=None):
+ self.config = config or ServiceConfig()
+
+ def __enter__(self):
+ self.init()
+ return self
+
+ def __exit__(self, *args):
+ self.terminate()
+
+ def init(self):
+ """Initialize connection"""
+ logger.debug(u'Initializing %s' % self.__class__.__name__)
+ self.session = requests.session(timeout=10, headers={'User-Agent': self.user_agent})
+
+ def terminate(self):
+ """Terminate connection"""
+ logger.debug(u'Terminating %s' % self.__class__.__name__)
+
+ def query(self, *args):
+ """Make the actual query"""
+ pass
+
+ def list(self, video, languages):
+ """List subtitles"""
+ pass
+
+ def download(self, subtitle):
+ """Download a subtitle"""
+ self.download_file(subtitle.link, subtitle.path)
+
+ @classmethod
+ def available_languages(cls):
+ """Available languages in the Service
+
+ :return: available languages
+ :rtype: set
+
+ """
+ if not cls.reverted_languages:
+ return set(cls.languages.keys())
+ return set(cls.languages.values())
+
+ @classmethod
+ def check_validity(cls, video, languages):
+ """Check for video and languages validity in the Service
+
+ :param video: the video to check
+ :type video: :class:`~subliminal.videos.video`
+ :param set languages: languages to check
+ :rtype: bool
+
+ """
+ languages &= cls.available_languages()
+ if not languages:
+ logger.debug(u'No language available for service %s' % cls.__class__.__name__.lower())
+ return False
+ if not cls.is_valid_video(video):
+ logger.debug(u'%r is not valid for service %s' % (video, cls.__class__.__name__.lower()))
+ return False
+ return True
+
+ @classmethod
+ def is_valid_video(cls, video):
+ """Check if video is valid in the Service
+
+ :param video: the video to check
+ :type video: :class:`~subliminal.videos.Video`
+ :rtype: bool
+
+ """
+ if cls.require_video and not video.exists:
+ return False
+ if not isinstance(video, tuple(cls.videos)):
+ return False
+ return True
+
+ @classmethod
+ def is_valid_language(cls, language):
+ """Check if language is valid in the Service
+
+ :param string language: the language to check
+ :rtype: bool
+
+ """
+ if language in cls.available_languages():
+ return True
+ return False
+
+ @classmethod
+ def get_revert_language(cls, language):
+ """ISO-639-1 language code from service language code
+
+ :param string language: service language code
+ :return: ISO-639-1 language code
+ :rtype: string
+
+ """
+ if not cls.reverted_languages and language in cls.languages.values():
+ return [k for k, v in cls.languages.iteritems() if v == language][0]
+ if cls.reverted_languages and language in cls.languages.keys():
+ return cls.languages[language]
+ raise MissingLanguageError(language)
+
+ @classmethod
+ def get_language(cls, language):
+ """Service language code from ISO-639-1 language code
+
+ :param string language: ISO-639-1 language code
+ :return: service language code
+ :rtype: string
+
+ """
+ if not cls.reverted_languages and language in cls.languages.keys():
+ return cls.languages[language]
+ if cls.reverted_languages and language in cls.languages.values():
+ return [k for k, v in cls.languages.iteritems() if v == language][0]
+ raise MissingLanguageError(language)
+
+ def download_file(self, url, filepath):
+ """Attempt to download a file and remove it in case of failure
+
+ :param string url: URL to download
+ :param string filepath: destination path
+
+ """
+ logger.info(u'Downloading %s' % url)
+ try:
+ r = self.session.get(url, headers={'Referer': url, 'User-Agent': self.user_agent})
+ with open(filepath, 'wb') as f:
+ f.write(r.content)
+ except Exception as e:
+ logger.error(u'Download %s failed: %s' % (url, e))
+ if os.path.exists(filepath):
+ os.remove(filepath)
+ raise DownloadFailedError(str(e))
+ logger.debug(u'Download finished for file %s. Size: %s' % (filepath, os.path.getsize(filepath)))
+
+
+class ServiceConfig(object):
+ """Configuration for any :class:`Service`
+
+ :param bool multi: whether to download one subtitle per language or not
+ :param string cache_dir: cache directory
+
+ """
+ def __init__(self, multi=False, cache_dir=None):
+ self.multi = multi
+ self.cache_dir = cache_dir
+
+ def __repr__(self):
+ return 'ServiceConfig(%r, %s)' % (self.multi, self.cache_dir)
diff --git a/libs/subliminal/services/bierdopje.py b/libs/subliminal/services/bierdopje.py
new file mode 100755
index 00000000..15401ada
--- /dev/null
+++ b/libs/subliminal/services/bierdopje.py
@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from . import ServiceBase
+from ..exceptions import ServiceError
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..videos import Episode
+from ..utils import to_unicode
+import BeautifulSoup
+import logging
+import os.path
+import urllib
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+
+
+logger = logging.getLogger(__name__)
+
+
+class BierDopje(ServiceBase):
+ server_url = 'http://api.bierdopje.com/A2B638AC5D804C2E/'
+ api_based = True
+ languages = {'en': 'en', 'nl': 'nl'}
+ reverted_languages = False
+ videos = [Episode]
+ require_video = False
+
+ def __init__(self, config=None):
+ super(BierDopje, self).__init__(config)
+ self.showids = {}
+ if self.config and self.config.cache_dir:
+ self.init_cache()
+
+ def init_cache(self):
+ logger.debug(u'Initializing cache...')
+ if not self.config or not self.config.cache_dir:
+ raise ServiceError('Cache directory is required')
+ self.showids_cache = os.path.join(self.config.cache_dir, 'bierdopje_showids.cache')
+ if not os.path.exists(self.showids_cache):
+ self.save_cache()
+
+ def save_cache(self):
+ logger.debug(u'Saving showids to cache...')
+ with self.lock:
+ with open(self.showids_cache, 'w') as f:
+ pickle.dump(self.showids, f)
+
+ def load_cache(self):
+ logger.debug(u'Loading showids from cache...')
+ with self.lock:
+ with open(self.showids_cache, 'r') as f:
+ self.showids = pickle.load(f)
+
+ def query(self, season, episode, languages, filepath, tvdbid=None, series=None):
+ self.load_cache()
+ if series:
+ if series.lower() in self.showids: # from cache
+ request_id = self.showids[series.lower()]
+ logger.debug(u'Retreived showid %d for %s from cache' % (request_id, series))
+ else: # query to get showid
+ logger.debug(u'Getting showid from show name %s...' % series)
+ r = self.session.get('%sGetShowByName/%s' % (self.server_url, urllib.quote(series.lower())))
+ if r.status_code != 200:
+ logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
+ return []
+ soup = BeautifulSoup.BeautifulStoneSoup(r.content)
+ if soup.status.contents[0] == 'false':
+ logger.debug(u'Could not find show %s' % series)
+ return []
+ request_id = int(soup.showid.contents[0])
+ self.showids[series.lower()] = request_id
+ self.save_cache()
+ request_source = 'showid'
+ request_is_tvdbid = 'false'
+ elif tvdbid:
+ request_id = tvdbid
+ request_source = 'tvdbid'
+ request_is_tvdbid = 'true'
+ else:
+ raise ServiceError('One or more parameter missing')
+ subtitles = []
+ for language in languages:
+ logger.debug(u'Getting subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
+ r = self.session.get('%sGetAllSubsFor/%s/%s/%s/%s/%s' % (self.server_url, request_id, season, episode, language, request_is_tvdbid))
+ if r.status_code != 200:
+ logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
+ return []
+ soup = BeautifulSoup.BeautifulStoneSoup(r.content)
+ if soup.status.contents[0] == 'false':
+ logger.debug(u'Could not find subtitles for %s %d season %d episode %d with language %s' % (request_source, request_id, season, episode, language))
+ continue
+ path = get_subtitle_path(filepath, language, self.config.multi)
+ for result in soup.results('result'):
+ subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result.downloadlink.contents[0],
+ release=to_unicode(result.filename.contents[0]))
+ subtitles.append(subtitle)
+ return subtitles
+
+ def list(self, video, languages):
+ if not self.check_validity(video, languages):
+ return []
+ results = self.query(video.season, video.episode, languages, video.path or video.release, video.tvdbid, video.series)
+ return results
+
+
+Service = BierDopje
diff --git a/libs/subliminal/services/opensubtitles.py b/libs/subliminal/services/opensubtitles.py
new file mode 100755
index 00000000..9dee27b9
--- /dev/null
+++ b/libs/subliminal/services/opensubtitles.py
@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from . import ServiceBase
+from ..exceptions import ServiceError, DownloadFailedError
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..videos import Episode, Movie
+from ..utils import to_unicode
+import gzip
+import logging
+import os.path
+import xmlrpclib
+
+
+logger = logging.getLogger(__name__)
+
+
+class OpenSubtitles(ServiceBase):
+ server_url = 'http://api.opensubtitles.org/xml-rpc'
+ api_based = True
+ languages = {'aa': 'aar', 'ab': 'abk', 'af': 'afr', 'ak': 'aka', 'sq': 'alb', 'am': 'amh', 'ar': 'ara',
+ 'an': 'arg', 'hy': 'arm', 'as': 'asm', 'av': 'ava', 'ae': 'ave', 'ay': 'aym', 'az': 'aze',
+ 'ba': 'bak', 'bm': 'bam', 'eu': 'baq', 'be': 'bel', 'bn': 'ben', 'bh': 'bih', 'bi': 'bis',
+ 'bs': 'bos', 'br': 'bre', 'bg': 'bul', 'my': 'bur', 'ca': 'cat', 'ch': 'cha', 'ce': 'che',
+ 'zh': 'chi', 'cu': 'chu', 'cv': 'chv', 'kw': 'cor', 'co': 'cos', 'cr': 'cre', 'cs': 'cze',
+ 'da': 'dan', 'dv': 'div', 'nl': 'dut', 'dz': 'dzo', 'en': 'eng', 'eo': 'epo', 'et': 'est',
+ 'ee': 'ewe', 'fo': 'fao', 'fj': 'fij', 'fi': 'fin', 'fr': 'fre', 'fy': 'fry', 'ff': 'ful',
+ 'ka': 'geo', 'de': 'ger', 'gd': 'gla', 'ga': 'gle', 'gl': 'glg', 'gv': 'glv', 'el': 'ell',
+ 'gn': 'grn', 'gu': 'guj', 'ht': 'hat', 'ha': 'hau', 'he': 'heb', 'hz': 'her', 'hi': 'hin',
+ 'ho': 'hmo', 'hr': 'hrv', 'hu': 'hun', 'ig': 'ibo', 'is': 'ice', 'io': 'ido', 'ii': 'iii',
+ 'iu': 'iku', 'ie': 'ile', 'ia': 'ina', 'id': 'ind', 'ik': 'ipk', 'it': 'ita', 'jv': 'jav',
+ 'ja': 'jpn', 'kl': 'kal', 'kn': 'kan', 'ks': 'kas', 'kr': 'kau', 'kk': 'kaz', 'km': 'khm',
+ 'ki': 'kik', 'rw': 'kin', 'ky': 'kir', 'kv': 'kom', 'kg': 'kon', 'ko': 'kor', 'kj': 'kua',
+ 'ku': 'kur', 'lo': 'lao', 'la': 'lat', 'lv': 'lav', 'li': 'lim', 'ln': 'lin', 'lt': 'lit',
+ 'lb': 'ltz', 'lu': 'lub', 'lg': 'lug', 'mk': 'mac', 'mh': 'mah', 'ml': 'mal', 'mi': 'mao',
+ 'mr': 'mar', 'ms': 'may', 'mg': 'mlg', 'mt': 'mlt', 'mo': 'mol', 'mn': 'mon', 'na': 'nau',
+ 'nv': 'nav', 'nr': 'nbl', 'nd': 'nde', 'ng': 'ndo', 'ne': 'nep', 'nn': 'nno', 'nb': 'nob',
+ 'no': 'nor', 'ny': 'nya', 'oc': 'oci', 'oj': 'oji', 'or': 'ori', 'om': 'orm', 'os': 'oss',
+ 'pa': 'pan', 'fa': 'per', 'pi': 'pli', 'pl': 'pol', 'pt': 'por', 'ps': 'pus', 'qu': 'que',
+ 'rm': 'roh', 'rn': 'run', 'ru': 'rus', 'sg': 'sag', 'sa': 'san', 'sr': 'scc', 'si': 'sin',
+ 'sk': 'slo', 'sl': 'slv', 'se': 'sme', 'sm': 'smo', 'sn': 'sna', 'sd': 'snd', 'so': 'som',
+ 'st': 'sot', 'es': 'spa', 'sc': 'srd', 'ss': 'ssw', 'su': 'sun', 'sw': 'swa', 'sv': 'swe',
+ 'ty': 'tah', 'ta': 'tam', 'tt': 'tat', 'te': 'tel', 'tg': 'tgk', 'tl': 'tgl', 'th': 'tha',
+ 'bo': 'tib', 'ti': 'tir', 'to': 'ton', 'tn': 'tsn', 'ts': 'tso', 'tk': 'tuk', 'tr': 'tur',
+ 'tw': 'twi', 'ug': 'uig', 'uk': 'ukr', 'ur': 'urd', 'uz': 'uzb', 've': 'ven', 'vi': 'vie',
+ 'vo': 'vol', 'cy': 'wel', 'wa': 'wln', 'wo': 'wol', 'xh': 'xho', 'yi': 'yid', 'yo': 'yor',
+ 'za': 'zha', 'zu': 'zul', 'ro': 'rum', 'po': 'pob', 'un': 'unk', 'ay': 'ass'}
+ reverted_languages = False
+ videos = [Episode, Movie]
+ require_video = False
+ confidence_order = ['moviehash', 'imdbid', 'fulltext']
+
+ def __init__(self, config=None):
+ super(OpenSubtitles, self).__init__(config)
+ self.server = xmlrpclib.ServerProxy(self.server_url)
+ self.token = None
+
+ def init(self):
+ super(OpenSubtitles, self).init()
+ result = self.server.LogIn('', '', 'eng', self.user_agent)
+ if result['status'] != '200 OK':
+ raise ServiceError('Login failed')
+ self.token = result['token']
+
+ def terminate(self):
+ super(OpenSubtitles, self).terminate()
+ if self.token:
+ self.server.LogOut(self.token)
+
+ def query(self, filepath, languages, moviehash=None, size=None, imdbid=None, query=None):
+ searches = []
+ if moviehash and size:
+ searches.append({'moviehash': moviehash, 'moviebytesize': size})
+ if imdbid:
+ searches.append({'imdbid': imdbid})
+ if query:
+ searches.append({'query': query})
+ if not searches:
+ raise ServiceError('One or more parameter missing')
+ for search in searches:
+ search['sublanguageid'] = ','.join([self.get_language(l) for l in languages])
+ logger.debug(u'Getting subtitles %r with token %s' % (searches, self.token))
+ results = self.server.SearchSubtitles(self.token, searches)
+ if not results['data']:
+ logger.debug(u'Could not find subtitles for %r with token %s' % (searches, self.token))
+ return []
+ subtitles = []
+ for result in results['data']:
+ language = self.get_revert_language(result['SubLanguageID'])
+ path = get_subtitle_path(filepath, language, self.config.multi)
+ confidence = 1 - float(self.confidence_order.index(result['MatchedBy'])) / float(len(self.confidence_order))
+ subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=result['SubDownloadLink'],
+ release=to_unicode(result['SubFileName']), confidence=confidence)
+ subtitles.append(subtitle)
+ return subtitles
+
+ def list(self, video, languages):
+ if not self.check_validity(video, languages):
+ return []
+ results = []
+ if video.exists:
+ results = self.query(video.path or video.release, languages, moviehash=video.hashes['OpenSubtitles'], size=str(video.size))
+ elif video.imdbid:
+ results = self.query(video.path or video.release, languages, imdbid=video.imdbid)
+ elif isinstance(video, Episode):
+ results = self.query(video.path or video.release, languages, query=video.series)
+ elif isinstance(video, Movie):
+ results = self.query(video.path or video.release, languages, query=video.title)
+ return results
+
+ def download(self, subtitle):
+ #TODO: Use OpenSubtitles DownloadSubtitles method
+ try:
+ self.download_file(subtitle.link, subtitle.path + '.gz')
+ with open(subtitle.path, 'wb') as dump:
+ gz = gzip.open(subtitle.path + '.gz')
+ dump.write(gz.read())
+ gz.close()
+ except Exception as e:
+ if os.path.exists(subtitle.path):
+ os.remove(subtitle.path)
+ raise DownloadFailedError(str(e))
+ finally:
+ if os.path.exists(subtitle.path + '.gz'):
+ os.remove(subtitle.path + '.gz')
+ return subtitle
+
+
+Service = OpenSubtitles
diff --git a/libs/subliminal/services/subswiki.py b/libs/subliminal/services/subswiki.py
new file mode 100755
index 00000000..c5670c1c
--- /dev/null
+++ b/libs/subliminal/services/subswiki.py
@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from . import ServiceBase
+from ..exceptions import ServiceError
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..videos import Episode, Movie
+from subliminal.utils import get_keywords, split_keyword
+import BeautifulSoup
+import logging
+import re
+import urllib
+
+
+logger = logging.getLogger(__name__)
+
+
+class SubsWiki(ServiceBase):
+ server_url = 'http://www.subswiki.com'
+ api_based = False
+ languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
+ u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
+ u'Italian': 'it', u'Català': 'ca'}
+ reverted_languages = True
+ videos = [Episode, Movie]
+ require_video = False
+ release_pattern = re.compile('\nVersion (.+), ([0-9]+).([0-9])+ MBs')
+
+ def list(self, video, languages):
+ if not self.check_validity(video, languages):
+ return []
+ results = []
+ if isinstance(video, Episode):
+ results = self.query(video.path or video.release, languages, get_keywords(video.guess), series=video.series, season=video.season, episode=video.episode)
+ elif isinstance(video, Movie) and video.year:
+ results = self.query(video.path or video.release, languages, get_keywords(video.guess), movie=video.title, year=video.year)
+ return results
+
+ def query(self, filepath, languages, keywords=None, series=None, season=None, episode=None, movie=None, year=None):
+ if series and season and episode:
+ request_series = series.lower().replace(' ', '_')
+ if isinstance(request_series, unicode):
+ request_series = request_series.encode('utf-8')
+ logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
+ r = self.session.get('%s/serie/%s/%s/%s/' % (self.server_url, urllib.quote(request_series), season, episode))
+ if r.status_code == 404:
+ logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
+ return []
+ elif movie and year:
+ request_movie = movie.title().replace(' ', '_')
+ if isinstance(request_movie, unicode):
+ request_movie = request_movie.encode('utf-8')
+ logger.debug(u'Getting subtitles for %s (%d) with languages %r' % (movie, year, languages))
+ r = self.session.get('%s/film/%s_(%d)' % (self.server_url, urllib.quote(request_movie), year))
+ if r.status_code == 404:
+ logger.debug(u'Could not find subtitles for %s (%d) with languages %r' % (movie, year, languages))
+ return []
+ else:
+ raise ServiceError('One or more parameter missing')
+ if r.status_code != 200:
+ logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
+ return []
+ soup = BeautifulSoup.BeautifulSoup(r.content)
+ subtitles = []
+ for sub in soup('td', {'class': 'NewsTitle'}):
+ sub_keywords = split_keyword(self.release_pattern.search(sub.contents[1]).group(1).lower())
+ if not keywords & sub_keywords:
+ logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
+ continue
+ for html_language in sub.parent.parent.findAll('td', {'class': 'language'}):
+ language = self.get_revert_language(html_language.string.strip())
+ if not language in languages:
+ logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
+ continue
+ html_status = html_language.findNextSibling('td')
+ status = html_status.find('strong').string.strip()
+ if status != 'Completed':
+ logger.debug(u'Wrong subtitle status %s' % status)
+ continue
+ path = get_subtitle_path(filepath, language, self.config.multi)
+ subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s%s' % (self.server_url, html_status.findNext('td').find('a')['href']))
+ subtitles.append(subtitle)
+ return subtitles
+
+Service = SubsWiki
diff --git a/libs/subliminal/services/subtitulos.py b/libs/subliminal/services/subtitulos.py
new file mode 100755
index 00000000..44888e70
--- /dev/null
+++ b/libs/subliminal/services/subtitulos.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from . import ServiceBase
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..videos import Episode
+from subliminal.utils import get_keywords, split_keyword
+import BeautifulSoup
+import logging
+import re
+import unicodedata
+import urllib
+
+
+logger = logging.getLogger(__name__)
+
+
+class Subtitulos(ServiceBase):
+ server_url = 'http://www.subtitulos.es'
+ api_based = False
+ languages = {u'English (US)': 'en', u'English (UK)': 'en', u'English': 'en', u'French': 'fr', u'Brazilian': 'po',
+ u'Portuguese': 'pt', u'Español (Latinoamérica)': 'es', u'Español (España)': 'es', u'Español': 'es',
+ u'Italian': 'it', u'Català': 'ca'}
+ reverted_languages = True
+ videos = [Episode]
+ require_video = False
+ release_pattern = re.compile('Versión (.+) ([0-9]+).([0-9])+ megabytes')
+
+ def list(self, video, languages):
+ if not self.check_validity(video, languages):
+ return []
+ results = self.query(video.path or video.release, languages, get_keywords(video.guess), video.series, video.season, video.episode)
+ return results
+
+ def query(self, filepath, languages, keywords, series, season, episode):
+ request_series = series.lower().replace(' ', '_')
+ if isinstance(request_series, unicode):
+ request_series = unicodedata.normalize('NFKD', request_series).encode('ascii', 'ignore')
+ logger.debug(u'Getting subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
+ r = self.session.get('%s/%s/%sx%.2d' % (self.server_url, urllib.quote(request_series), season, episode))
+ if r.status_code == 404:
+ logger.debug(u'Could not find subtitles for %s season %d episode %d with languages %r' % (series, season, episode, languages))
+ return []
+ if r.status_code != 200:
+ logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
+ return []
+ soup = BeautifulSoup.BeautifulSoup(r.content)
+ subtitles = []
+ for sub in soup('div', {'id': 'version'}):
+ sub_keywords = split_keyword(self.release_pattern.search(sub.find('p', {'class': 'title-sub'}).contents[1]).group(1).lower())
+ if not keywords & sub_keywords:
+ logger.debug(u'None of subtitle keywords %r in %r' % (sub_keywords, keywords))
+ continue
+ for html_language in sub.findAllNext('ul', {'class': 'sslist'}):
+ language = self.get_revert_language(html_language.findNext('li', {'class': 'li-idioma'}).find('strong').contents[0].string.strip())
+ if not language in languages:
+ logger.debug(u'Language %r not in wanted languages %r' % (language, languages))
+ continue
+ html_status = html_language.findNext('li', {'class': 'li-estado green'})
+ status = html_status.contents[0].string.strip()
+ if status != 'Completado':
+ logger.debug(u'Wrong subtitle status %s' % status)
+ continue
+ path = get_subtitle_path(filepath, language, self.config.multi)
+ subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link=html_status.findNext('span', {'class': 'descargar green'}).find('a')['href'], keywords=sub_keywords)
+ subtitles.append(subtitle)
+ return subtitles
+
+Service = Subtitulos
diff --git a/libs/subliminal/services/thesubdb.py b/libs/subliminal/services/thesubdb.py
new file mode 100755
index 00000000..cccddd40
--- /dev/null
+++ b/libs/subliminal/services/thesubdb.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
+#
+# This file is part of subliminal.
+#
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# subliminal is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from . import ServiceBase
+from ..subtitles import get_subtitle_path, ResultSubtitle
+from ..videos import Episode, Movie, UnknownVideo
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+
+class TheSubDB(ServiceBase):
+ server_url = 'http://api.thesubdb.com/' # for testing purpose, use http://sandbox.thesubdb.com/ instead
+ user_agent = 'SubDB/1.0 (subliminal/0.5; https://github.com/Diaoul/subliminal)' # defined by the API
+ api_based = True
+ languages = {'af': 'af', 'cs': 'cs', 'da': 'da', 'de': 'de', 'en': 'en', 'es': 'es', 'fi': 'fi',
+ 'fr': 'fr', 'hu': 'hu', 'id': 'id', 'it': 'it', 'la': 'la', 'nl': 'nl', 'no': 'no',
+ 'oc': 'oc', 'pl': 'pl', 'pt': 'pt', 'ro': 'ro', 'ru': 'ru', 'sl': 'sl', 'sr': 'sr',
+ 'sv': 'sv', 'tr': 'tr'} # list available with the API at http://sandbox.thesubdb.com/?action=languages
+ reverted_languages = False
+ videos = [Movie, Episode, UnknownVideo]
+ require_video = True
+
+ def list(self, video, languages):
+ if not self.check_validity(video, languages):
+ return []
+ results = self.query(video.path, video.hashes['TheSubDB'], languages)
+ return results
+
+ def query(self, filepath, moviehash, languages):
+ r = self.session.get(self.server_url, params={'action': 'search', 'hash': moviehash})
+ if r.status_code == 404:
+ logger.debug(u'Could not find subtitles for hash %s' % moviehash)
+ return []
+ if r.status_code != 200:
+ logger.error(u'Request %s returned status code %d' % (r.url, r.status_code))
+ return []
+ available_languages = set([self.get_revert_language(l) for l in r.content.split(',')])
+ languages &= available_languages
+ if not languages:
+ logger.debug(u'Could not find subtitles for hash %s with languages %r (only %r available)' % (moviehash, languages, available_languages))
+ return []
+ subtitles = []
+ for language in languages:
+ path = get_subtitle_path(filepath, language, self.config.multi)
+ subtitle = ResultSubtitle(path, language, service=self.__class__.__name__.lower(), link='%s?action=download&hash=%s&language=%s' % (self.server_url, moviehash, self.get_language(language)))
+ subtitles.append(subtitle)
+ return subtitles
+
+Service = TheSubDB
diff --git a/libs/subliminal/subtitles.py b/libs/subliminal/subtitles.py
index 479d25c8..355046dc 100755
--- a/libs/subliminal/subtitles.py
+++ b/libs/subliminal/subtitles.py
@@ -1,50 +1,73 @@
# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
#
-# Subliminal - Subtitles, faster than your thoughts
-# Copyright (c) 2011 Antoine Bertin
+# This file is part of subliminal.
#
-# This file is part of Subliminal.
-#
-# Subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Subliminal is distributed in the hope that it will be useful,
+# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
+# GNU Lesser General Public License for more details.
#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
-#
-__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path']
-
-
-from languages import list_languages, convert_language
-import abc
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from .languages import list_languages, convert_language
import os.path
+__all__ = ['Subtitle', 'EmbeddedSubtitle', 'ExternalSubtitle', 'ResultSubtitle', 'get_subtitle_path']
+
+
+#: Subtitles extensions
EXTENSIONS = ['.srt', '.sub', '.txt']
class Subtitle(object):
- __metaclass__ = abc.ABCMeta
- """Base class for subtitles"""
+ """Base class for subtitles
+ :param string path: path to the subtitle
+ :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
+
+ """
def __init__(self, path, language):
self.path = path
self.language = language
@property
def exists(self):
+ """Whether the subtitle exists or not"""
if self.path:
return os.path.exists(self.path)
return False
+
+class EmbeddedSubtitle(Subtitle):
+ """Subtitle embedded in a container
+
+ :param string path: path to the subtitle
+ :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
+ :param int track_id: id of the subtitle track in the container
+
+ """
+ def __init__(self, path, language, track_id):
+ super(EmbeddedSubtitle, self).__init__(path, language)
+ self.track_id = track_id
+
@classmethod
- def fromPath(cls, path):
+ def from_enzyme(cls, path, subtitle):
+ language = convert_language(subtitle.language, 1, 2)
+ return cls(path, language, subtitle.trackno)
+
+
+class ExternalSubtitle(Subtitle):
+ """Subtitle in a file next to the video file"""
+ @classmethod
+ def from_path(cls, path):
+ """Create an :class:`ExternalSubtitle` from path"""
extension = ''
for e in EXTENSIONS:
if path.endswith(e):
@@ -58,25 +81,21 @@ class Subtitle(object):
return cls(path, language)
-class EmbeddedSubtitle(Subtitle):
- def __init__(self, path, language, track_id):
- super(EmbeddedSubtitle, self).__init__(path, language)
- self.track_id = track_id
-
- @classmethod
- def fromEnzyme(cls, path, subtitle):
- language = convert_language(subtitle.language, 1, 2)
- return cls(path, language, subtitle.trackno)
-
-
-class ExternalSubtitle(Subtitle):
- pass
-
-
class ResultSubtitle(ExternalSubtitle):
- def __init__(self, path, language, plugin, link, release=None, confidence=1, keywords=set()):
+ """Subtitle found using :mod:`~subliminal.services`
+
+ :param string path: path to the subtitle
+ :param string language: language of the subtitle (second element of :class:`~subliminal.languages.LANGUAGES`)
+ :param string service: name of the service
+ :param string link: download link for the subtitle
+ :param string release: release name of the video
+ :param float confidence: confidence that the subtitle matches the video according to the service
+ :param set keywords: keywords that describe the subtitle
+
+ """
+ def __init__(self, path, language, service, link, release=None, confidence=1, keywords=set()):
super(ResultSubtitle, self).__init__(path, language)
- self.plugin = plugin
+ self.service = service
self.link = link
self.release = release
self.confidence = confidence
@@ -84,19 +103,20 @@ class ResultSubtitle(ExternalSubtitle):
@property
def single(self):
+ """Whether this is a single subtitle or not. A single subtitle does not have
+ a language indicator in its file name
+
+ :rtype: bool
+
+ """
extension = os.path.splitext(self.path)[0]
language = os.path.splitext(self.path[:len(self.path) - len(extension)])[1][1:]
if not language in list_languages(1):
return True
return False
- def convert(self):
- converted = {'path': self.path, 'plugin': self.plugin, 'language': self.language, 'link': self.link, 'release': self.release,
- 'confidence': self.confidence, 'keywords': self.keywords}
- return converted
-
- def __str__(self):
- return repr(self.convert())
+ def __repr__(self):
+ return 'ResultSubtitle(%s, %s, %.2f, %s)' % (self.language, self.service, self.confidence, self.release)
def get_subtitle_path(video_path, language, multi):
diff --git a/libs/subliminal/tasks.py b/libs/subliminal/tasks.py
index 111a08d3..448a64f1 100755
--- a/libs/subliminal/tasks.py
+++ b/libs/subliminal/tasks.py
@@ -1,23 +1,20 @@
# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
#
-# Subliminal - Subtitles, faster than your thoughts
-# Copyright (c) 2011 Antoine Bertin
+# This file is part of subliminal.
#
-# This file is part of Subliminal.
-#
-# Subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Subliminal is distributed in the hope that it will be useful,
+# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
-#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
+# GNU Lesser General Public License for more details.
#
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
__all__ = ['Task', 'ListTask', 'DownloadTask', 'StopTask']
@@ -27,21 +24,43 @@ class Task(object):
class ListTask(Task):
- """List task to list subtitles"""
- def __init__(self, video, languages, plugin, config):
+ """List task used by the worker to search for subtitles
+
+ :param video: video to search subtitles for
+ :type video: :class:`~subliminal.videos.Video`
+ :param list languages: languages to search for
+ :param string service: name of the service to use
+ :param config: configuration for the service
+ :type config: :class:`~subliminal.services.ServiceConfig`
+
+ """
+ def __init__(self, video, languages, service, config):
self.video = video
- self.plugin = plugin
+ self.service = service
self.languages = languages
self.config = config
+ def __repr__(self):
+ return 'ListTask(%r, %r, %s, %r)' % (self.video, self.languages, self.service, self.config)
+
class DownloadTask(Task):
- """Download task to download subtitles"""
+ """Download task used by the worker to download subtitles
+
+ :param video: video to download subtitles for
+ :type video: :class:`~subliminal.videos.Video`
+ :param subtitles: subtitles to download in order of preference
+ :type subtitles: list of :class:`~subliminal.subtitles.Subtitle`
+
+ """
def __init__(self, video, subtitles):
self.video = video
self.subtitles = subtitles
+ def __repr__(self):
+ return 'DownloadTask(%r, %r)' % (self.video, self.subtitles)
+
class StopTask(Task):
- """Stop task to stop workers"""
+ """Stop task that will stop the worker"""
pass
diff --git a/libs/subliminal/utils.py b/libs/subliminal/utils.py
index 27dddb16..00cda322 100755
--- a/libs/subliminal/utils.py
+++ b/libs/subliminal/utils.py
@@ -1,44 +1,35 @@
# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
#
-# Subliminal - Subtitles, faster than your thoughts
-# Copyright (c) 2011 Antoine Bertin
+# This file is part of subliminal.
#
-# This file is part of Subliminal.
-#
-# Subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Subliminal is distributed in the hope that it will be useful,
+# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
+# GNU Lesser General Public License for more details.
#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
-#
-__all__ = ['PluginConfig', 'get_keywords', 'split_keyword', 'NullHandler']
-
-
-import logging
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
import re
-try:
- from logging import NullHandler
-except ImportError:
- class NullHandler(logging.Handler):
- def emit(self, record):
- pass
-class PluginConfig(object):
- def __init__(self, multi=None, cache_dir=None, filemode=None):
- self.multi = multi
- self.cache_dir = cache_dir
- self.filemode = filemode
+__all__ = ['get_keywords', 'split_keyword', 'to_unicode']
def get_keywords(guess):
+ """Retrieve keywords from guessed informations
+
+ :param guess: guessed informations
+ :type guess: :class:`guessit.guess.Guess`
+ :return: lower case alphanumeric keywords
+ :rtype: set
+
+ """
keywords = set()
for k in ['releaseGroup', 'screenSize', 'videoCodec', 'format']:
if k in guess:
@@ -47,5 +38,27 @@ def get_keywords(guess):
def split_keyword(keyword):
+ """Split a keyword in multiple ones on any non-alphanumeric character
+
+ :param string keyword: keyword
+ :return: keywords
+ :rtype: set
+
+ """
split = set(re.findall(r'\w+', keyword))
return split
+
+
+def to_unicode(data):
+ """Convert a basestring to unicode
+
+ :param basestring data: data to decode
+ :return: data as unicode
+ :rtype: unicode
+
+ """
+ if not isinstance(data, basestring):
+ raise ValueError('Basestring expected')
+ if isinstance(data, unicode):
+ return data
+ return unicode(data, 'utf-8', 'replace')
diff --git a/libs/subliminal/videos.py b/libs/subliminal/videos.py
index cb5a1520..79276b14 100755
--- a/libs/subliminal/videos.py
+++ b/libs/subliminal/videos.py
@@ -1,57 +1,73 @@
# -*- coding: utf-8 -*-
+# Copyright 2011-2012 Antoine Bertin
#
-# Subliminal - Subtitles, faster than your thoughts
-# Copyright (c) 2011 Antoine Bertin
+# This file is part of subliminal.
#
-# This file is part of Subliminal.
-#
-# Subliminal is free software; you can redistribute it and/or modify it under
-# the terms of the Lesser GNU General Public License as published by
+# subliminal is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
-# Subliminal is distributed in the hope that it will be useful,
+# subliminal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# Lesser GNU General Public License for more details.
+# GNU Lesser General Public License for more details.
#
-# You should have received a copy of the Lesser GNU General Public License
-# along with this program. If not, see .
-#
-__all__ = ['EXTENSIONS', 'MIMETYPES', 'Video', 'Episode', 'Movie', 'UnknownVideo', 'scan']
-
-
-from languages import list_languages
-import abc
+# You should have received a copy of the GNU Lesser General Public License
+# along with subliminal. If not, see .
+from . import subtitles
+from .languages import list_languages
import enzyme
import guessit
import hashlib
+import logging
import mimetypes
import os
import struct
-import subprocess
-import subtitles
-EXTENSIONS = ['.avi', '.mkv', '.mpg', '.mp4', '.m4v', '.mov', '.ogm', '.ogv', '.wmv', '.divx', '.asf']
-MIMETYPES = ['video/mpeg', 'video/mp4', 'video/quicktime', 'video/x-ms-wmv', 'video/x-msvideo', 'video/x-flv', 'video/x-matroska', 'video/x-matroska-3d']
+__all__ = ['EXTENSIONS', 'MIMETYPES', 'Video', 'Episode', 'Movie', 'UnknownVideo',
+ 'scan', 'hash_opensubtitles', 'hash_thesubdb']
+logger = logging.getLogger(__name__)
+
+#: Video extensions
+EXTENSIONS = ['.avi', '.mkv', '.mpg', '.mp4', '.m4v', '.mov', '.ogm', '.ogv', '.wmv',
+ '.divx', '.asf']
+
+#: Video mimetypes
+MIMETYPES = ['video/mpeg', 'video/mp4', 'video/quicktime', 'video/x-ms-wmv', 'video/x-msvideo',
+ 'video/x-flv', 'video/x-matroska', 'video/x-matroska-3d']
class Video(object):
- __metaclass__ = abc.ABCMeta
- """Base class for videos"""
- def __init__(self, release, guess, imdbid=None):
- self.release = release
+ """Base class for videos
+
+ :param string path: path
+ :param guess: guessed informations
+ :type guess: :class:`~guessit.guess.Guess`
+ :param string imdbid: imdbid
+
+ """
+ def __init__(self, path, guess, imdbid=None):
+ self.release = path
self.guess = guess
self.imdbid = imdbid
self._path = None
self.hashes = {}
- if os.path.exists(release):
- self.path = release
+ if os.path.exists(path):
+ self._path = path
+ self.size = os.path.getsize(self._path)
+ self._compute_hashes()
@classmethod
- def fromPath(cls, path):
- """Create a Video object guessing all informations from the given release/path"""
+ def from_path(cls, path):
+ """Create a :class:`Video` subclass guessing all informations from the given path
+
+ :param string path: path
+ :return: video object
+ :rtype: :class:`Episode` or :class:`Movie` or :class:`UnknownVideo`
+
+ """
guess = guessit.guess_file_info(path, 'autodetect')
result = None
if guess['type'] == 'episode' and 'series' in guess and 'season' in guess and 'episodeNumber' in guess:
@@ -72,12 +88,14 @@ class Video(object):
@property
def exists(self):
+ """Whether the video exists or not"""
if self._path:
return os.path.exists(self._path)
return False
@property
def path(self):
+ """Path to the video"""
return self._path
@path.setter
@@ -86,21 +104,137 @@ class Video(object):
raise ValueError('Path does not exists')
self._path = value
self.size = os.path.getsize(self._path)
- self._computeHashes()
+ self._compute_hashes()
- def _computeHashes(self):
- self.hashes['OpenSubtitles'] = self._computeHashOpenSubtitles()
- self.hashes['TheSubDB'] = self._computeHashTheSubDB()
+ def _compute_hashes(self):
+ """Compute different hashes"""
+ self.hashes['OpenSubtitles'] = hash_opensubtitles(self.path)
+ self.hashes['TheSubDB'] = hash_thesubdb(self.path)
- def _computeHashOpenSubtitles(self):
- """Hash a file like OpenSubtitles"""
- longlongformat = 'q' # long long
- bytesize = struct.calcsize(longlongformat)
- f = open(self.path, 'rb')
- filesize = os.path.getsize(self.path)
+ def scan(self):
+ """Scan and return associated subtitles
+
+ :return: associated subtitles
+ :rtype: list of :class:`~subliminal.subtitles.Subtitle`
+
+ """
+ if not self.exists:
+ return []
+ basepath = os.path.splitext(self.path)[0]
+ results = []
+ video_infos = None
+ try:
+ video_infos = enzyme.parse(self.path)
+ logger.debug(u'Succeeded parsing %s with enzyme: %r' % (self.path, video_infos))
+ except:
+ logger.debug(u'Failed parsing %s with enzyme' % self.path)
+ if isinstance(video_infos, enzyme.core.AVContainer):
+ results.extend([subtitles.EmbeddedSubtitle.from_enzyme(self.path, s) for s in video_infos.subtitles])
+ for l in list_languages(1):
+ for e in subtitles.EXTENSIONS:
+ single_path = basepath + '%s' % e
+ if os.path.exists(single_path):
+ results.append(subtitles.ExternalSubtitle(single_path, None))
+ multi_path = basepath + '.%s%s' % (l, e)
+ if os.path.exists(multi_path):
+ results.append(subtitles.ExternalSubtitle(multi_path, l))
+ return results
+
+ def __repr__(self):
+ return '%s(%r)' % (self.__class__.__name__, self.release)
+
+
+class Episode(Video):
+ """Episode :class:`Video`
+
+ :param string path: path
+ :param string series: series
+ :param int season: season number
+ :param int episode: episode number
+ :param string title: title
+ :param guess: guessed informations
+ :type guess: :class:`~guessit.guess.Guess`
+ :param string tvdbid: tvdbid
+ :param string imdbid: imdbid
+
+ """
+ def __init__(self, path, series, season, episode, title=None, guess=None, tvdbid=None, imdbid=None):
+ super(Episode, self).__init__(path, guess, imdbid)
+ self.series = series
+ self.title = title
+ self.season = season
+ self.episode = episode
+ self.tvdbid = tvdbid
+
+
+class Movie(Video):
+ """Movie :class:`Video`
+
+ :param string path: path
+ :param string title: title
+ :param int year: year
+ :param guess: guessed informations
+ :type guess: :class:`~guessit.guess.Guess`
+ :param string imdbid: imdbid
+
+ """
+ def __init__(self, path, title, year=None, guess=None, imdbid=None):
+ super(Movie, self).__init__(path, guess, imdbid)
+ self.title = title
+ self.year = year
+
+
+class UnknownVideo(Video):
+ """Unknown video"""
+ pass
+
+
+def scan(entry, max_depth=3, depth=0):
+ """Scan a path for videos and subtitles
+
+ :param string entry: path
+ :param int max_depth: maximum folder depth
+ :param int depth: starting depth
+ :return: found videos and subtitles
+ :rtype: list of (:class:`Video`, [:class:`~subliminal.subtitles.Subtitle`])
+
+ """
+ if depth > max_depth and max_depth != 0: # we do not want to search the whole file system except if max_depth = 0
+ return []
+ if depth == 0:
+ entry = os.path.abspath(entry)
+ if os.path.isdir(entry): # a dir? recurse
+ logger.debug(u'Scanning directory %s with depth %d/%d' % (entry, depth, max_depth))
+ result = []
+ for e in os.listdir(entry):
+ result.extend(scan(os.path.join(entry, e), max_depth, depth + 1))
+ return result
+ if os.path.isfile(entry) or depth == 0:
+ logger.debug(u'Scanning file %s with depth %d/%d' % (entry, depth, max_depth))
+ if depth != 0: # trust the user: only check for valid format if recursing
+ if mimetypes.guess_type(entry)[0] not in MIMETYPES and os.path.splitext(entry)[1] not in EXTENSIONS:
+ return []
+ video = Video.from_path(entry)
+ return [(video, video.scan())]
+ logger.warning(u'Scanning entry %s failed with depth %d/%d' % (entry, depth, max_depth))
+ return [] # anything else
+
+
+def hash_opensubtitles(path):
+ """Compute a hash using OpenSubtitles' algorithm
+
+ :param string path: path
+ :return: hash
+ :rtype: string
+
+ """
+ longlongformat = 'q' # long long
+ bytesize = struct.calcsize(longlongformat)
+ with open(path, 'rb') as f:
+ filesize = os.path.getsize(path)
filehash = filesize
if filesize < 65536 * 2:
- return []
+ return None
for _ in range(65536 / bytesize):
filebuffer = f.read(bytesize)
(l_value,) = struct.unpack(longlongformat, filebuffer)
@@ -112,103 +246,24 @@ class Video(object):
(l_value,) = struct.unpack(longlongformat, filebuffer)
filehash += l_value
filehash = filehash & 0xFFFFFFFFFFFFFFFF
- f.close()
- returnedhash = '%016x' % filehash
- return returnedhash
-
- def _computeHashTheSubDB(self):
- """Hash a file like TheSubDB"""
- readsize = 64 * 1024
- with open(self.path, 'rb') as f:
- data = f.read(readsize)
- f.seek(-readsize, os.SEEK_END)
- data += f.read(readsize)
- return hashlib.md5(data).hexdigest()
-
- def mkvmerge(self, subs, out=None, mkvmerge_bin='mkvmerge', title=None):
- """Merge the video with subs"""
- if not out:
- out = self.path + '.merged.mkv'
- args = [mkvmerge_bin, '-o', out, self.path]
- if title:
- args += ['--title', title]
- for sub in subs:
- if sub.language:
- track_id = 0
- if isinstance(sub, subtitles.EmbeddedSubtitle):
- track_id = sub.track_id
- args += ['--language', str(track_id) + ':' + sub.language, sub.path]
- continue
- args += [sub.path]
- with open(os.devnull, 'w') as devnull:
- p = subprocess.Popen(args, stdout=devnull, stderr=devnull)
- p.wait()
-
- def scan(self):
- """Scan and return associated Subtitles"""
- if not self.exists:
- return []
- basepath = os.path.splitext(self.path)[0]
- results = []
- video_infos = None
- try:
- video_infos = enzyme.parse(self.path)
- except enzyme.ParseError:
- pass
- if isinstance(video_infos, enzyme.core.AVContainer):
- results.extend([subtitles.EmbeddedSubtitle.fromEnzyme(self.path, s) for s in video_infos.subtitles])
- for l in list_languages(1):
- for e in subtitles.EXTENSIONS:
- single_path = basepath + '%s' % e
- if os.path.exists(single_path):
- results.append(subtitles.ExternalSubtitle(single_path, None))
- multi_path = basepath + '.%s%s' % (l, e)
- if os.path.exists(multi_path):
- results.append(subtitles.ExternalSubtitle(multi_path, l))
- return results
+ returnedhash = '%016x' % filehash
+ logger.debug(u'Computed OpenSubtitle hash %s for %s' % (returnedhash, path))
+ return returnedhash
-class Episode(Video):
- """Episode class"""
- def __init__(self, release, series, season, episode, title=None, guess=None, tvdbid=None, imdbid=None):
- super(Episode, self).__init__(release, guess, imdbid)
- self.series = series
- self.title = title
- self.season = season
- self.episode = episode
- self.tvdbid = tvdbid
+def hash_thesubdb(path):
+ """Compute a hash using TheSubDB's algorithm
+ :param string path: path
+ :return: hash
+ :rtype: string
-class Movie(Video):
- """Movie class"""
- def __init__(self, release, title, year=None, guess=None, imdbid=None):
- super(Movie, self).__init__(release, guess, imdbid)
- self.title = title
- self.year = year
-
-
-class UnknownVideo(Video):
- """Unknown video"""
- def __init__(self, release, guess, imdbid=None):
- super(UnknownVideo, self).__init__(release, guess, imdbid)
- self.guess = guess
-
-
-def scan(entry, max_depth=3, depth=0):
- """Scan a path and return a list of tuples (video, [subtitle])"""
- if depth > max_depth and max_depth != 0: # we do not want to search the whole file system except if max_depth = 0
- return []
- if depth == 0:
- entry = os.path.abspath(entry)
- if os.path.isfile(entry): # a file? scan it
- if depth != 0: # trust the user: only check for valid format if recursing
- if mimetypes.guess_type(entry)[0] not in MIMETYPES and os.path.splitext(entry)[1] not in EXTENSIONS:
- return []
- video = Video.fromPath(entry)
- return [(video, video.scan())]
- if os.path.isdir(entry): # a dir? recurse
- result = []
- for e in os.listdir(entry):
- result.extend(scan(os.path.join(entry, e), max_depth, depth + 1))
- return result
- return [] # anything else
+ """
+ readsize = 64 * 1024
+ with open(path, 'rb') as f:
+ data = f.read(readsize)
+ f.seek(-readsize, os.SEEK_END)
+ data += f.read(readsize)
+ returnedhash = hashlib.md5(data).hexdigest()
+ logger.debug(u'Computed TheSubDB hash %s for %s' % (returnedhash, path))
+ return returnedhash