diff --git a/couchpotato/core/_base/_core.py b/couchpotato/core/_base/_core.py index 1ca7d878..ca45ceb0 100644 --- a/couchpotato/core/_base/_core.py +++ b/couchpotato/core/_base/_core.py @@ -90,7 +90,11 @@ class Core(Plugin): def shutdown(): self.initShutdown() - IOLoop.current().add_callback(shutdown) + + if IOLoop.current()._closing: + shutdown() + else: + IOLoop.current().add_callback(shutdown) return 'shutdown' @@ -139,7 +143,8 @@ class Core(Plugin): log.debug('Safe to shutdown/restart') try: - IOLoop.current().stop() + if not IOLoop.current()._closing: + IOLoop.current().stop() except RuntimeError: pass except: diff --git a/couchpotato/core/database.py b/couchpotato/core/database.py index 532befae..f9607137 100644 --- a/couchpotato/core/database.py +++ b/couchpotato/core/database.py @@ -28,6 +28,7 @@ class Database(object): addEvent('database.setup_index', self.setupIndex) addEvent('app.migrate', self.migrate) + addEvent('app.after_shutdown', self.close) def getDB(self): @@ -37,6 +38,9 @@ class Database(object): return self.db + def close(self, **kwargs): + self.getDB().close() + def setupIndex(self, index_name, klass): self.indexes.append(index_name) @@ -412,7 +416,10 @@ class Database(object): empty_info = True rel['info'] = {} - quality = quality_link[rel.get('quality_id')] + quality = quality_link.get(rel.get('quality_id')) + if not quality: + continue + release_status = statuses.get(rel.get('status_id')).get('identifier') if rel['info'].get('download_id'): diff --git a/couchpotato/core/media/__init__.py b/couchpotato/core/media/__init__.py index c491c016..f0abd28c 100644 --- a/couchpotato/core/media/__init__.py +++ b/couchpotato/core/media/__init__.py @@ -25,7 +25,6 @@ class MediaBase(Plugin): def onComplete(): try: - db = get_db() media = fireEvent('media.get', media_id, single = True) event_name = '%s.searcher.single' % media.get('type') diff --git a/couchpotato/core/media/_base/media/main.py b/couchpotato/core/media/_base/media/main.py index 29a98fd1..beb9dd02 100644 --- a/couchpotato/core/media/_base/media/main.py +++ b/couchpotato/core/media/_base/media/main.py @@ -107,8 +107,7 @@ class MediaPlugin(MediaBase): def handler(): fireEvent(event, media_id = media_id, on_complete = self.createOnComplete(media_id)) - if handler: - return handler + return handler except: log.error('Refresh handler for non existing media: %s', traceback.format_exc()) @@ -361,13 +360,18 @@ class MediaPlugin(MediaBase): media = db.get('id', media_id) if media: deleted = False + + media_releases = fireEvent('release.for_media', media['_id'], single = True) + if delete_from == 'all': + # Delete connected releases + for release in media_releases: + db.delete(release) + db.delete(media) deleted = True else: - media_releases = fireEvent('release.for_media', media['_id'], single = True) - total_releases = len(media_releases) total_deleted = 0 new_media_status = None diff --git a/couchpotato/core/media/_base/providers/base.py b/couchpotato/core/media/_base/providers/base.py index 2b668931..b13b279d 100644 --- a/couchpotato/core/media/_base/providers/base.py +++ b/couchpotato/core/media/_base/providers/base.py @@ -88,10 +88,14 @@ class Provider(Plugin): if data and len(data) > 0: try: - data = XMLTree.fromstring(ss(data)) + data = XMLTree.fromstring(data) return self.getElements(data, item_path) except: - log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) + try: + data = XMLTree.fromstring(ss(data)) + return self.getElements(data, item_path) + except: + log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc())) return [] @@ -298,7 +302,7 @@ class ResultList(list): old_score = new_result['score'] new_result['score'] = int(old_score * is_correct_weight) - log.info('Found correct release with weight %.02f, old_score(%d) now scaled to score(%d)', ( + log.info2('Found correct release with weight %.02f, old_score(%d) now scaled to score(%d)', ( is_correct_weight, old_score, new_result['score'] diff --git a/couchpotato/core/media/_base/providers/nzb/binsearch.py b/couchpotato/core/media/_base/providers/nzb/binsearch.py index fbb67bfc..84c7b314 100644 --- a/couchpotato/core/media/_base/providers/nzb/binsearch.py +++ b/couchpotato/core/media/_base/providers/nzb/binsearch.py @@ -50,8 +50,8 @@ class Base(NZBProvider): def extra_check(item): parts = re.search('available:.(?P\d+)./.(?P\d+)', info.text) - total = tryInt(parts.group('total')) - parts = tryInt(parts.group('parts')) + total = float(tryInt(parts.group('total'))) + parts = float(tryInt(parts.group('parts'))) if (total / parts) < 1 and ((total / parts) < 0.95 or ((total / parts) >= 0.95 and not ('par2' in info.text.lower() or 'pa3' in info.text.lower()))): log.info2('Wrong: \'%s\', not complete: %s out of %s', (item['name'], parts, total)) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 31223103..04915776 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -45,7 +45,7 @@ class Base(NZBProvider, RSS): def _searchOnHost(self, host, media, quality, results): - query = self.buildUrl(media, host['api_key']) + query = self.buildUrl(media, host) url = '%s&%s' % (self.getUrl(host['host']), query) nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()}) @@ -244,7 +244,7 @@ config = [{ }, { 'name': 'host', - 'default': 'api.nzb.su,dognzb.cr,nzbs.org,https://index.nzbgeek.info, https://smackdownonyou.com, https://www.nzbfinder.ws', + 'default': 'api.nzb.su,api.dognzb.cr,nzbs.org,https://index.nzbgeek.info, https://smackdownonyou.com, https://www.nzbfinder.ws', 'description': 'The hostname of your newznab provider', }, { diff --git a/couchpotato/core/media/_base/providers/torrent/base.py b/couchpotato/core/media/_base/providers/torrent/base.py index 2ed496c5..9f5f2890 100644 --- a/couchpotato/core/media/_base/providers/torrent/base.py +++ b/couchpotato/core/media/_base/providers/torrent/base.py @@ -44,7 +44,8 @@ class TorrentProvider(YarrProvider): prop_name = 'proxy.%s' % proxy last_check = float(Env.prop(prop_name, default = 0)) - if last_check > time.time() - 1209600: + + if last_check > time.time() - 86400: continue data = '' diff --git a/couchpotato/core/media/_base/providers/torrent/bithdtv.py b/couchpotato/core/media/_base/providers/torrent/bithdtv.py index 94aafcd1..bb12c66a 100644 --- a/couchpotato/core/media/_base/providers/torrent/bithdtv.py +++ b/couchpotato/core/media/_base/providers/torrent/bithdtv.py @@ -25,7 +25,7 @@ class Base(TorrentProvider): def _search(self, media, quality, results): - query = self.buildUrl(media) + query = self.buildUrl(media, quality) url = "%s&%s" % (self.urls['search'], query) diff --git a/couchpotato/core/media/_base/providers/torrent/bitsoup.py b/couchpotato/core/media/_base/providers/torrent/bitsoup.py index 9aeb6243..f32e79e7 100644 --- a/couchpotato/core/media/_base/providers/torrent/bitsoup.py +++ b/couchpotato/core/media/_base/providers/torrent/bitsoup.py @@ -1,7 +1,6 @@ import traceback from bs4 import BeautifulSoup -from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.media._base.providers.torrent.base import TorrentProvider @@ -16,7 +15,7 @@ class Base(TorrentProvider): 'test': 'https://www.bitsoup.me/', 'login': 'https://www.bitsoup.me/takelogin.php', 'login_check': 'https://www.bitsoup.me/my.php', - 'search': 'https://www.bitsoup.me/browse.php?', + 'search': 'https://www.bitsoup.me/browse.php?%s', 'baseurl': 'https://www.bitsoup.me/%s', } @@ -24,13 +23,7 @@ class Base(TorrentProvider): def _searchOnTitle(self, title, movie, quality, results): - q = '"%s" %s' % (simplifyString(title), movie['info']['year']) - arguments = tryUrlencode({ - 'search': q, - }) - url = "%s&%s" % (self.urls['search'], arguments) - - url = self.urls['search'] % self.buildUrl(movie, quality) + url = self.urls['search'] % self.buildUrl(title, movie, quality) data = self.getHTMLData(url) if data: diff --git a/couchpotato/core/media/_base/providers/torrent/sceneaccess.py b/couchpotato/core/media/_base/providers/torrent/sceneaccess.py index 2622e318..e488b63b 100644 --- a/couchpotato/core/media/_base/providers/torrent/sceneaccess.py +++ b/couchpotato/core/media/_base/providers/torrent/sceneaccess.py @@ -24,9 +24,9 @@ class Base(TorrentProvider): http_time_between_calls = 1 # Seconds - def _search(self, media, quality, results): + def _searchOnTitle(self, title, media, quality, results): - url = self.buildUrl(media, quality) + url = self.buildUrl(title, media, quality) data = self.getHTMLData(url) if data: diff --git a/couchpotato/core/media/_base/providers/torrent/torrentday.py b/couchpotato/core/media/_base/providers/torrent/torrentday.py index bcc0593b..9304e552 100644 --- a/couchpotato/core/media/_base/providers/torrent/torrentday.py +++ b/couchpotato/core/media/_base/providers/torrent/torrentday.py @@ -1,3 +1,4 @@ +from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog from couchpotato.core.media._base.providers.torrent.base import TorrentProvider @@ -18,16 +19,16 @@ class Base(TorrentProvider): http_time_between_calls = 1 # Seconds - def _search(self, media, quality, results): + def _searchOnTitle(self, title, media, quality, results): - query = self.buildUrl(media) + query = '"%s" %s' % (title, media['info']['year']) data = { '/browse.php?': None, 'cata': 'yes', 'jxt': 8, 'jxw': 'b', - 'search': query, + 'search': tryUrlencode(query), } data = self.getJsonData(self.urls['search'], data = data) diff --git a/couchpotato/core/media/_base/providers/torrent/yify.py b/couchpotato/core/media/_base/providers/torrent/yify.py index a6941004..82413fc5 100644 --- a/couchpotato/core/media/_base/providers/torrent/yify.py +++ b/couchpotato/core/media/_base/providers/torrent/yify.py @@ -35,7 +35,11 @@ class Base(TorrentProvider): def _search(self, movie, quality, results): - search_url = self.urls['search'] % (self.getDomain(), getIdentifier(movie), quality['identifier']) + domain = self.getDomain() + if not domain: + return + + search_url = self.urls['search'] % (domain, getIdentifier(movie), quality['identifier']) data = self.getJsonData(search_url) @@ -43,21 +47,19 @@ class Base(TorrentProvider): try: for result in data.get('MovieList'): - try: - title = result['TorrentUrl'].split('/')[-1][:-8].replace('_', '.').strip('._') - title = title.replace('.-.', '-') - title = title.replace('..', '.') - except: - continue + if result['Quality'] and result['Quality'] not in result['MovieTitle']: + title = result['MovieTitle'] + ' BrRip ' + result['Quality'] + else: + title = result['MovieTitle'] + ' BrRip' results.append({ 'id': result['MovieID'], 'name': title, 'url': result['TorrentMagnetUrl'], - 'detail_url': self.urls['detail'] % (self.getDomain(), result['MovieID']), + 'detail_url': self.urls['detail'] % (domain, result['MovieID']), 'size': self.parseSize(result['Size']), 'seeders': tryInt(result['TorrentSeeds']), - 'leechers': tryInt(result['TorrentPeers']) + 'leechers': tryInt(result['TorrentPeers']), }) except: diff --git a/couchpotato/core/media/_base/searcher/main.py b/couchpotato/core/media/_base/searcher/main.py index fc07ded2..4e8dae2e 100644 --- a/couchpotato/core/media/_base/searcher/main.py +++ b/couchpotato/core/media/_base/searcher/main.py @@ -87,31 +87,23 @@ class Searcher(SearcherBase): def containsOtherQuality(self, nzb, movie_year = None, preferred_quality = None): if not preferred_quality: preferred_quality = {} - name = nzb['name'] - size = nzb.get('size', 0) - nzb_words = re.split('\W+', simplifyString(name)) - - qualities = fireEvent('quality.all', single = True) - found = {} - for quality in qualities: - # Main in words - if quality['identifier'] in nzb_words: - found[quality['identifier']] = True - - # Alt in words - if list(set(nzb_words) & set(quality['alternative'])): - found[quality['identifier']] = True # Try guessing via quality tags - guess = fireEvent('quality.guess', [nzb.get('name')], single = True) + guess = fireEvent('quality.guess', files = [nzb.get('name')], size = nzb.get('size', None), single = True) if guess: found[guess['identifier']] = True # Hack for older movies that don't contain quality tag + name = nzb['name'] + size = nzb.get('size', 0) + year_name = fireEvent('scanner.name_year', name, single = True) if len(found) == 0 and movie_year < datetime.datetime.now().year - 3 and not year_name.get('year', None): - if size > 3000: # Assume dvdr + if size > 20000: # Assume bd50 + log.info('Quality was missing in name, assuming it\'s a BR-Disk based on the size: %s', size) + found['bd50'] = True + elif size > 3000: # Assume dvdr log.info('Quality was missing in name, assuming it\'s a DVD-R based on the size: %s', size) found['dvdr'] = True else: # Assume dvdrip @@ -123,7 +115,10 @@ class Searcher(SearcherBase): if found.get(allowed): del found[allowed] - return not (found.get(preferred_quality['identifier']) and len(found) == 1) + if found.get(preferred_quality['identifier']) and len(found) == 1: + return False + + return found def correct3D(self, nzb, preferred_quality = None): if not preferred_quality: preferred_quality = {} diff --git a/couchpotato/core/media/movie/_base/main.py b/couchpotato/core/media/movie/_base/main.py index a783cd1e..860ec965 100644 --- a/couchpotato/core/media/movie/_base/main.py +++ b/couchpotato/core/media/movie/_base/main.py @@ -139,7 +139,7 @@ class MovieBase(MovieTypeBase): # Clean snatched history for release in fireEvent('release.for_media', m['_id'], single = True): - if release.get('status') in ['downloaded', 'snatched', 'done']: + if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']: if params.get('ignore_previous', False): release['status'] = 'ignored' db.update(release) diff --git a/couchpotato/core/media/movie/_base/static/2_manage.js b/couchpotato/core/media/movie/_base/static/manage.js similarity index 99% rename from couchpotato/core/media/movie/_base/static/2_manage.js rename to couchpotato/core/media/movie/_base/static/manage.js index be99a845..e8618999 100644 --- a/couchpotato/core/media/movie/_base/static/2_manage.js +++ b/couchpotato/core/media/movie/_base/static/manage.js @@ -2,6 +2,7 @@ Page.Manage = new Class({ Extends: PageBase, + order: 20, name: 'manage', title: 'Do stuff to your existing movies!', diff --git a/couchpotato/core/media/movie/_base/static/movie.actions.js b/couchpotato/core/media/movie/_base/static/movie.actions.js index 4d9dcccb..26ee2421 100644 --- a/couchpotato/core/media/movie/_base/static/movie.actions.js +++ b/couchpotato/core/media/movie/_base/static/movie.actions.js @@ -78,7 +78,7 @@ MA.IMDB = new Class({ create: function(){ var self = this; - self.id = self.movie.get('imdb') || self.movie.get('identifier'); + self.id = self.movie.getIdentifier ? self.movie.getIdentifier() : self.get('imdb'); self.el = new Element('a.imdb', { 'title': 'Go to the IMDB page of ' + self.getTitle(), @@ -684,7 +684,7 @@ MA.Readd = new Class({ var movie_done = self.movie.data.status == 'done'; if(self.movie.data.releases && !movie_done) var snatched = self.movie.data.releases.filter(function(release){ - return release.status && (release.status == 'snatched' || release.status == 'downloaded' || release.status == 'done'); + return release.status && (release.status == 'snatched' || release.status == 'seeding' || release.status == 'downloaded' || release.status == 'done'); }).length; if(movie_done || snatched && snatched > 0) @@ -703,7 +703,7 @@ MA.Readd = new Class({ Api.request('movie.add', { 'data': { - 'identifier': self.movie.get('identifier'), + 'identifier': self.movie.getIdentifier(), 'ignore_previous': 1 } }); diff --git a/couchpotato/core/media/movie/_base/static/movie.css b/couchpotato/core/media/movie/_base/static/movie.css index 6718dc3a..05fcddf4 100644 --- a/couchpotato/core/media/movie/_base/static/movie.css +++ b/couchpotato/core/media/movie/_base/static/movie.css @@ -123,6 +123,7 @@ .movies.thumbs_list .movie { width: 16.66667%; height: auto; + min-height: 200px; display: inline-block; margin: 0; padding: 0; @@ -133,6 +134,7 @@ @media all and (max-width: 800px) { .movies.thumbs_list .movie { width: 25%; + min-height: 100px; } } diff --git a/couchpotato/core/media/movie/_base/static/movie.js b/couchpotato/core/media/movie/_base/static/movie.js index 86205efc..1a547f2f 100644 --- a/couchpotato/core/media/movie/_base/static/movie.js +++ b/couchpotato/core/media/movie/_base/static/movie.js @@ -300,6 +300,17 @@ var Movie = new Class({ self.el.removeClass(self.view+'_view') }, + getIdentifier: function(){ + var self = this; + + try { + return self.get('identifiers').imdb; + } + catch (e){ } + + return self.get('imdb'); + }, + get: function(attr){ return this.data[attr] || this.data.info[attr] }, diff --git a/couchpotato/core/media/movie/_base/static/1_wanted.js b/couchpotato/core/media/movie/_base/static/wanted.js similarity index 99% rename from couchpotato/core/media/movie/_base/static/1_wanted.js rename to couchpotato/core/media/movie/_base/static/wanted.js index ca57f957..461a64e1 100644 --- a/couchpotato/core/media/movie/_base/static/1_wanted.js +++ b/couchpotato/core/media/movie/_base/static/wanted.js @@ -2,6 +2,7 @@ Page.Wanted = new Class({ Extends: PageBase, + order: 10, name: 'wanted', title: 'Gimmy gimmy gimmy!', folder_browser: null, diff --git a/couchpotato/core/media/movie/charts/__init__.py b/couchpotato/core/media/movie/charts/__init__.py index 9c673751..361da51a 100644 --- a/couchpotato/core/media/movie/charts/__init__.py +++ b/couchpotato/core/media/movie/charts/__init__.py @@ -28,6 +28,20 @@ config = [{ 'advanced': True, 'description': '(hours)', }, + { + 'name': 'hide_wanted', + 'default': False, + 'type': 'bool', + 'advanced': True, + 'description': 'Hide the chart movies that are already in your wanted list.', + }, + { + 'name': 'hide_library', + 'default': False, + 'type': 'bool', + 'advanced': True, + 'description': 'Hide the chart movies that are already in your library.', + }, ], }, ], diff --git a/couchpotato/core/media/movie/charts/main.py b/couchpotato/core/media/movie/charts/main.py index 71002752..58133d83 100644 --- a/couchpotato/core/media/movie/charts/main.py +++ b/couchpotato/core/media/movie/charts/main.py @@ -36,7 +36,6 @@ class Charts(Plugin): 'charts': charts } - def updateViewCache(self): if self.update_in_progress: @@ -49,6 +48,9 @@ class Charts(Plugin): try: self.update_in_progress = True charts = fireEvent('automation.get_chart_list', merge = True) + for chart in charts: + chart['hide_wanted'] = self.conf('hide_wanted') + chart['hide_library'] = self.conf('hide_library') self.setCache('charts_cached', charts, timeout = 7200 * tryInt(self.conf('update_interval', default = 12))) except: log.error('Failed refreshing charts') diff --git a/couchpotato/core/media/movie/charts/static/charts.css b/couchpotato/core/media/movie/charts/static/charts.css index 610ac153..ce8ff0ab 100644 --- a/couchpotato/core/media/movie/charts/static/charts.css +++ b/couchpotato/core/media/movie/charts/static/charts.css @@ -11,6 +11,17 @@ display: inline-block; width: 50%; vertical-align: top; + max-height: 510px; + overflow: hidden; + scrollbar-base-color: #4e5969; +} + +.charts .chart:hover { + overflow-y: auto; +} + +.charts .chart .media_result.hidden { + display: none; } .charts .refresh { diff --git a/couchpotato/core/media/movie/charts/static/charts.js b/couchpotato/core/media/movie/charts/static/charts.js index 00033f4a..d04f1c2c 100644 --- a/couchpotato/core/media/movie/charts/static/charts.js +++ b/couchpotato/core/media/movie/charts/static/charts.js @@ -89,7 +89,7 @@ var Charts = new Class({ } }); - var in_database_class = movie.in_wanted ? 'chart_in_wanted' : (movie.in_library ? 'chart_in_library' : ''), + var in_database_class = (chart.hide_wanted && movie.in_wanted) ? 'hidden' : (movie.in_wanted ? 'chart_in_wanted' : ((chart.hide_library && movie.in_library) ? 'hidden': (movie.in_library ? 'chart_in_library' : ''))), in_database_title = movie.in_wanted ? 'Movie in wanted list' : (movie.in_library ? 'Movie in library' : ''); m.el diff --git a/couchpotato/core/media/movie/providers/automation/bluray.py b/couchpotato/core/media/movie/providers/automation/bluray.py index 630d0cbb..0501c601 100644 --- a/couchpotato/core/media/movie/providers/automation/bluray.py +++ b/couchpotato/core/media/movie/providers/automation/bluray.py @@ -1,4 +1,5 @@ from bs4 import BeautifulSoup +from couchpotato import fireEvent from couchpotato.core.helpers.rss import RSS from couchpotato.core.helpers.variable import tryInt from couchpotato.core.logger import CPLog @@ -96,8 +97,14 @@ class Bluray(Automation, RSS): movie = self.search(name, year) if movie: + if movie.get('imdb') in movie_ids: continue + + is_movie = fireEvent('movie.is_movie', identifier = movie.get('imdb'), single = True) + if not is_movie: + continue + movie_ids.append(movie.get('imdb')) movie_list['list'].append( movie ) if len(movie_list['list']) >= max_items: diff --git a/couchpotato/core/media/movie/providers/automation/imdb.py b/couchpotato/core/media/movie/providers/automation/imdb.py index 0f2e2940..b52816a8 100644 --- a/couchpotato/core/media/movie/providers/automation/imdb.py +++ b/couchpotato/core/media/movie/providers/automation/imdb.py @@ -100,20 +100,28 @@ class IMDBAutomation(IMDBBase): enabled_option = 'automation_providers_enabled' - chart_urls = { - 'theater': 'http://www.imdb.com/movies-in-theaters/', - 'top250': 'http://www.imdb.com/chart/top', - 'boxoffice': 'http://www.imdb.com/chart/', - } - chart_names = { - 'theater': 'IMDB - Movies in Theaters', - 'top250': 'IMDB - Top 250 Movies', - 'boxoffice': 'IMDB - Box Office', - } - chart_order = { - 'theater': 2, - 'top250': 4, - 'boxoffice': 3, + charts = { + 'theater': { + 'order': 1, + 'name': 'IMDB - Movies in Theaters', + 'url': 'http://www.imdb.com/movies-in-theaters/', + }, + 'boxoffice': { + 'order': 2, + 'name': 'IMDB - Box Office', + 'url': 'http://www.imdb.com/chart/', + }, + 'rentals': { + 'order': 3, + 'name': 'IMDB - Top DVD rentals', + 'url': 'http://m.imdb.com/boxoffice_json', + 'type': 'json', + }, + 'top250': { + 'order': 4, + 'name': 'IMDB - Top 250 Movies', + 'url': 'http://www.imdb.com/chart/top', + }, } first_table = ['boxoffice'] @@ -122,23 +130,30 @@ class IMDBAutomation(IMDBBase): movies = [] - for url in self.chart_urls: - if self.conf('automation_charts_%s' % url): - data = self.getHTMLData(self.chart_urls[url]) + for name in self.charts: + chart = self.charts[name] + url = chart.get('url') + + if self.conf('automation_charts_%s' % name): + data = self.getHTMLData(url) + if data: - html = BeautifulSoup(data) - try: - result_div = html.find('div', attrs = {'id': 'main'}) + html = BeautifulSoup(data) - try: - if url in self.first_table: - table = result_div.find('table') - result_div = table if table else result_div - except: - pass + if chart.get('type', 'html') == 'html': + result_div = html.find('div', attrs = {'id': 'main'}) - imdb_ids = getImdb(str(result_div), multiple = True) + try: + if url in self.first_table: + table = result_div.find('table') + result_div = table if table else result_div + except: + pass + + imdb_ids = getImdb(str(result_div), multiple = True) + else: + imdb_ids = getImdb(str(data), multiple = True) for imdb_id in imdb_ids: info = self.getInfo(imdb_id) @@ -153,42 +168,56 @@ class IMDBAutomation(IMDBBase): return movies - def getChartList(self): + # Nearly identical to 'getIMDBids', but we don't care about minimalMovie and return all movie data (not just id) movie_lists = [] - max_items = int(self.conf('max_items', section='charts', default=5)) + max_items = int(self.conf('max_items', section = 'charts', default=5)) - for url in self.chart_urls: - if self.conf('chart_display_%s' % url): - movie_list = {'name': self.chart_names[url], 'url': self.chart_urls[url], 'order': self.chart_order[url], 'list': []} - data = self.getHTMLData(self.chart_urls[url]) + for name in self.charts: + chart = self.charts[name].copy() + url = chart.get('url') + + if self.conf('chart_display_%s' % name): + + chart['list'] = [] + + data = self.getHTMLData(url) if data: html = BeautifulSoup(data) try: - result_div = html.find('div', attrs = {'id': 'main'}) - try: - if url in self.first_table: - table = result_div.find('table') - result_div = table if table else result_div - except: - pass + if chart.get('type', 'html') == 'html': + result_div = html.find('div', attrs = {'id': 'main'}) - imdb_ids = getImdb(str(result_div), multiple = True) + try: + if url in self.first_table: + table = result_div.find('table') + result_div = table if table else result_div + except: + pass + + imdb_ids = getImdb(str(result_div), multiple = True) + else: + imdb_ids = getImdb(str(data), multiple = True) for imdb_id in imdb_ids[0:max_items]: + + is_movie = fireEvent('movie.is_movie', identifier = imdb_id, single = True) + if not is_movie: + continue + info = self.getInfo(imdb_id) - movie_list['list'].append(info) + chart['list'].append(info) if self.shuttingDown(): break except: log.error('Failed loading IMDB chart results from %s: %s', (url, traceback.format_exc())) - if movie_list['list']: - movie_lists.append(movie_list) + if chart['list']: + movie_lists.append(chart) return movie_lists @@ -240,12 +269,19 @@ config = [{ 'description': 'New Movies In-Theaters chart', 'default': True, }, + { + 'name': 'automation_charts_rentals', + 'type': 'bool', + 'label': 'DVD Rentals', + 'description': 'Top DVD rentals chart', + 'default': True, + }, { 'name': 'automation_charts_top250', 'type': 'bool', 'label': 'TOP 250', 'description': 'IMDB TOP 250 chart', - 'default': True, + 'default': False, }, { 'name': 'automation_charts_boxoffice', @@ -282,6 +318,13 @@ config = [{ 'description': 'IMDB TOP 250 chart', 'default': False, }, + { + 'name': 'chart_display_rentals', + 'type': 'bool', + 'label': 'DVD Rentals', + 'description': 'Top DVD rentals chart', + 'default': True, + }, { 'name': 'chart_display_boxoffice', 'type': 'bool', diff --git a/couchpotato/core/media/movie/providers/nzb/newznab.py b/couchpotato/core/media/movie/providers/nzb/newznab.py index 9783f8d5..fc94acbf 100644 --- a/couchpotato/core/media/movie/providers/nzb/newznab.py +++ b/couchpotato/core/media/movie/providers/nzb/newznab.py @@ -11,11 +11,16 @@ autoload = 'Newznab' class Newznab(MovieProvider, Base): - def buildUrl(self, media, api_key): + def buildUrl(self, media, host): + query = tryUrlencode({ 't': 'movie', 'imdbid': getIdentifier(media).replace('tt', ''), - 'apikey': api_key, + 'apikey': host['api_key'], 'extended': 1 }) + + if len(host.get('custom_tag', '')) > 0: + query = '%s&%s' % (query, host.get('custom_tag')) + return query diff --git a/couchpotato/core/media/movie/providers/torrent/bithdtv.py b/couchpotato/core/media/movie/providers/torrent/bithdtv.py index e7b16af9..da6954c8 100644 --- a/couchpotato/core/media/movie/providers/torrent/bithdtv.py +++ b/couchpotato/core/media/movie/providers/torrent/bithdtv.py @@ -10,10 +10,14 @@ autoload = 'BiTHDTV' class BiTHDTV(MovieProvider, Base): + cat_ids = [ + ([2], ['bd50']), + ] + cat_backup_id = 7 # Movies - def buildUrl(self, media): + def buildUrl(self, media, quality): query = tryUrlencode({ 'search': fireEvent('library.query', media, single = True), - 'cat': 7 # Movie cat + 'cat': self.getCatId(quality)[0] }) return query diff --git a/couchpotato/core/media/movie/providers/torrent/bitsoup.py b/couchpotato/core/media/movie/providers/torrent/bitsoup.py index c351ab9f..e9d69fe5 100644 --- a/couchpotato/core/media/movie/providers/torrent/bitsoup.py +++ b/couchpotato/core/media/movie/providers/torrent/bitsoup.py @@ -1,6 +1,5 @@ from couchpotato.core.helpers.encoding import tryUrlencode from couchpotato.core.logger import CPLog -from couchpotato.core.event import fireEvent from couchpotato.core.media._base.providers.torrent.bitsoup import Base from couchpotato.core.media.movie.providers.base import MovieProvider @@ -18,12 +17,9 @@ class Bitsoup(MovieProvider, Base): ] cat_backup_id = 0 - def buildUrl(self, media, quality): + def buildUrl(self, title, media, quality): query = tryUrlencode({ - 'search': '"%s" %s' % ( - fireEvent('library.query', media, include_year = False, single = True), - media['info']['year'] - ), + 'search': '"%s" %s' % (title, media['info']['year']), 'cat': self.getCatId(quality)[0], }) return query diff --git a/couchpotato/core/media/movie/providers/torrent/iptorrents.py b/couchpotato/core/media/movie/providers/torrent/iptorrents.py index f06aadfc..89aeee80 100644 --- a/couchpotato/core/media/movie/providers/torrent/iptorrents.py +++ b/couchpotato/core/media/movie/providers/torrent/iptorrents.py @@ -18,6 +18,6 @@ class IPTorrents(MovieProvider, Base): ] def buildUrl(self, title, media, quality): - query = '%s %s' % (title.replace(':', ''), media['info']['year']) + query = '"%s" %s' % (title.replace(':', ''), media['info']['year']) return self._buildUrl(query, quality) diff --git a/couchpotato/core/media/movie/providers/torrent/sceneaccess.py b/couchpotato/core/media/movie/providers/torrent/sceneaccess.py index dee4b32c..3eed04dd 100644 --- a/couchpotato/core/media/movie/providers/torrent/sceneaccess.py +++ b/couchpotato/core/media/movie/providers/torrent/sceneaccess.py @@ -17,13 +17,13 @@ class SceneAccess(MovieProvider, Base): ([8], ['dvdr']), ] - def buildUrl(self, media, quality): + def buildUrl(self, title, media, quality): cat_id = self.getCatId(quality)[0] url = self.urls['search'] % (cat_id, cat_id) arguments = tryUrlencode({ - 'search': fireEvent('library.query', media, single = True), - 'method': 3, + 'search': '"%s" %s' % (title, media['info']['year']), + 'method': 2, }) query = "%s&%s" % (url, arguments) diff --git a/couchpotato/core/media/movie/providers/torrent/torrentday.py b/couchpotato/core/media/movie/providers/torrent/torrentday.py index 872a7d17..768d3043 100644 --- a/couchpotato/core/media/movie/providers/torrent/torrentday.py +++ b/couchpotato/core/media/movie/providers/torrent/torrentday.py @@ -1,5 +1,4 @@ from couchpotato.core.logger import CPLog -from couchpotato.core.event import fireEvent from couchpotato.core.media._base.providers.torrent.torrentday import Base from couchpotato.core.media.movie.providers.base import MovieProvider @@ -16,6 +15,3 @@ class TorrentDay(MovieProvider, Base): ([3], ['dvdr']), ([5], ['bd50']), ] - - def buildUrl(self, media): - return fireEvent('library.query', media, single = True) diff --git a/couchpotato/core/media/movie/providers/userscript/filmstarts.py b/couchpotato/core/media/movie/providers/userscript/filmstarts.py index 1e35be27..59027e03 100644 --- a/couchpotato/core/media/movie/providers/userscript/filmstarts.py +++ b/couchpotato/core/media/movie/providers/userscript/filmstarts.py @@ -1,8 +1,9 @@ -from BeautifulSoup import BeautifulSoup +from bs4 import BeautifulSoup from couchpotato.core.media._base.providers.userscript.base import UserscriptBase autoload = 'Filmstarts' + class Filmstarts(UserscriptBase): includes = ['*://www.filmstarts.de/kritiken/*'] diff --git a/couchpotato/core/media/movie/searcher.py b/couchpotato/core/media/movie/searcher.py index 68a76ade..5a8d6155 100644 --- a/couchpotato/core/media/movie/searcher.py +++ b/couchpotato/core/media/movie/searcher.py @@ -54,7 +54,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase): }) if self.conf('run_on_launch'): - addEvent('app.load', self.searchAll) + def on_load(): + time.sleep(.1) + self.searchAll() + addEvent('app.load', on_load, priority = 1000) def searchAllView(self, **kwargs): @@ -126,6 +129,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): release_dates = fireEvent('movie.update_release_dates', movie['_id'], merge = True) found_releases = [] + previous_releases = movie.get('releases', []) too_early_to_search = [] default_title = getTitle(movie) @@ -139,16 +143,15 @@ class MovieSearcher(SearcherBase, MovieTypeBase): db = get_db() profile = db.get('id', movie['profile_id']) - quality_order = fireEvent('quality.order', single = True) - ret = False index = 0 for q_identifier in profile.get('qualities'): quality_custom = { + 'index': index, 'quality': q_identifier, 'finish': profile['finish'][index], - 'wait_for': profile['wait_for'][index], + 'wait_for': tryInt(profile['wait_for'][index]), '3d': profile['3d'][index] if profile.get('3d') else False } @@ -162,43 +165,51 @@ class MovieSearcher(SearcherBase, MovieTypeBase): # See if better quality is available for release in movie.get('releases', []): - if quality_order.index(release['quality']) <= quality_order.index(q_identifier) and release['status'] not in ['available', 'ignored', 'failed']: - has_better_quality += 1 + if release['status'] not in ['available', 'ignored', 'failed']: + is_higher = fireEvent('quality.ishigher', \ + {'identifier': q_identifier, 'is_3d': quality_custom.get('3d', 0)}, \ + {'identifier': release['quality'], 'is_3d': release.get('is_3d', 0)}, \ + profile, single = True) + if is_higher != 'higher': + has_better_quality += 1 # Don't search for quality lower then already available. - if has_better_quality is 0: - - quality = fireEvent('quality.single', identifier = q_identifier, single = True) - log.info('Search for %s in %s', (default_title, quality['label'])) - - # Extend quality with profile customs - quality['custom'] = quality_custom - - results = fireEvent('searcher.search', search_protocols, movie, quality, single = True) or [] - if len(results) == 0: - log.debug('Nothing found for %s in %s', (default_title, quality['label'])) - - # Check if movie isn't deleted while searching - if not fireEvent('media.get', movie.get('_id'), single = True): - break - - # Add them to this movie releases list - found_releases += fireEvent('release.create_from_search', results, movie, quality, single = True) - - # Try find a valid result and download it - if fireEvent('release.try_download_result', results, movie, quality_custom, manual, single = True): - ret = True - - # Remove releases that aren't found anymore - for release in movie.get('releases', []): - if release.get('status') == 'available' and release.get('identifier') not in found_releases: - fireEvent('release.delete', release.get('_id'), single = True) - - else: + if has_better_quality > 0: log.info('Better quality (%s) already available or snatched for %s', (q_identifier, default_title)) fireEvent('media.restatus', movie['_id']) break + quality = fireEvent('quality.single', identifier = q_identifier, single = True) + log.info('Search for %s in %s', (default_title, quality['label'])) + + # Extend quality with profile customs + quality['custom'] = quality_custom + + results = fireEvent('searcher.search', search_protocols, movie, quality, single = True) or [] + if len(results) == 0: + log.debug('Nothing found for %s in %s', (default_title, quality['label'])) + + # Check if movie isn't deleted while searching + if not fireEvent('media.get', movie.get('_id'), single = True): + break + + # Add them to this movie releases list + found_releases += fireEvent('release.create_from_search', results, movie, quality, single = True) + + # Try find a valid result and download it + if fireEvent('release.try_download_result', results, movie, quality_custom, manual, single = True): + ret = True + + # Remove releases that aren't found anymore + temp_previous_releases = [] + for release in previous_releases: + if release.get('status') == 'available' and release.get('identifier') not in found_releases: + fireEvent('release.delete', release.get('_id'), single = True) + else: + temp_previous_releases.append(release) + previous_releases = temp_previous_releases + del temp_previous_releases + # Break if CP wants to shut down if self.shuttingDown() or ret: break @@ -230,8 +241,9 @@ class MovieSearcher(SearcherBase, MovieTypeBase): preferred_quality = quality if quality else fireEvent('quality.single', identifier = quality['identifier'], single = True) # Contains lower quality string - if fireEvent('searcher.contains_other_quality', nzb, movie_year = media['info']['year'], preferred_quality = preferred_quality, single = True): - log.info2('Wrong: %s, looking for %s', (nzb['name'], quality['label'])) + contains_other = fireEvent('searcher.contains_other_quality', nzb, movie_year = media['info']['year'], preferred_quality = preferred_quality, single = True) + if contains_other != False: + log.info2('Wrong: %s, looking for %s, found %s', (nzb['name'], quality['label'], [x for x in contains_other] if contains_other else 'no quality')) return False # Contains lower quality string @@ -288,7 +300,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase): now_year = date.today().year now_month = date.today().month - if (year is None or year < now_year - 1) and (not dates or (dates.get('theater', 0) == 0 and dates.get('dvd', 0) == 0)): + if (year is None or year < now_year - 1 or (year <= now_year - 1 and now_month > 4)) and (not dates or (dates.get('theater', 0) == 0 and dates.get('dvd', 0) == 0)): return True else: diff --git a/couchpotato/core/plugins/log/main.py b/couchpotato/core/plugins/log/main.py index 2565398e..5206acd6 100644 --- a/couchpotato/core/plugins/log/main.py +++ b/couchpotato/core/plugins/log/main.py @@ -1,9 +1,10 @@ import os +import re import traceback from couchpotato.api import addApiView from couchpotato.core.helpers.encoding import toUnicode -from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.helpers.variable import tryInt, splitString from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env @@ -22,7 +23,11 @@ class Logging(Plugin): }, 'return': {'type': 'object', 'example': """{ 'success': True, - 'log': string, //Log file + 'log': [{ + 'time': '03-12 09:12:59', + 'type': 'INFO', + 'message': 'Log message' + }, ..], //Log file 'total': int, //Total log files available }"""} }) @@ -34,7 +39,11 @@ class Logging(Plugin): }, 'return': {'type': 'object', 'example': """{ 'success': True, - 'log': string, //Log file + 'log': [{ + 'time': '03-12 09:12:59', + 'type': 'INFO', + 'message': 'Log message' + }, ..] }"""} }) addApiView('logging.clear', self.clear, docs = { @@ -71,16 +80,18 @@ class Logging(Plugin): if current_path: f = open(current_path, 'r') log_content = f.read() + logs = self.toList(log_content) return { 'success': True, - 'log': toUnicode(log_content), + 'log': logs, 'total': total, } - def partial(self, type = 'all', lines = 30, **kwargs): + def partial(self, type = 'all', lines = 30, offset = 0, **kwargs): total_lines = tryInt(lines) + offset = tryInt(offset) log_lines = [] @@ -93,28 +104,57 @@ class Logging(Plugin): break f = open(path, 'r') - reversed_lines = toUnicode(f.read()).split('[0m\n') - reversed_lines.reverse() + log_content = toUnicode(f.read()) + raw_lines = self.toList(log_content) + raw_lines.reverse() brk = False - for line in reversed_lines: + for line in raw_lines: - if type == 'all' or '%s ' % type.upper() in line: + if type == 'all' or line.get('type') == type.upper(): log_lines.append(line) - if len(log_lines) >= total_lines: + if len(log_lines) >= (total_lines + offset): brk = True break if brk: break + log_lines = log_lines[offset:] log_lines.reverse() + return { 'success': True, - 'log': '[0m\n'.join(log_lines), + 'log': log_lines, } + def toList(self, log_content = ''): + + logs_raw = toUnicode(log_content).split('[0m\n') + + logs = [] + for log in logs_raw: + split = splitString(log, '\x1b') + if split: + try: + date, time, log_type = splitString(split[0], ' ') + timestamp = '%s %s' % (date, time) + except: + timestamp = 'UNKNOWN' + log_type = 'UNKNOWN' + + message = ''.join(split[1]) if len(split) > 1 else split[0] + message = re.sub('\[\d+m\[', '[', message) + + logs.append({ + 'time': timestamp, + 'type': log_type, + 'message': message + }) + + return logs + def clear(self, **kwargs): for x in range(0, 50): diff --git a/couchpotato/core/plugins/log/static/log.css b/couchpotato/core/plugins/log/static/log.css index b706835a..e8797c69 100644 --- a/couchpotato/core/plugins/log/static/log.css +++ b/couchpotato/core/plugins/log/static/log.css @@ -16,10 +16,14 @@ display: inline-block; padding: 5px 10px; margin: 0; + } + + .page.log .nav li.select, + .page.log .nav li.clear { cursor: pointer; } - .page.log .nav li:hover:not(.active) { + .page.log .nav li:hover:not(.active, .filter) { background: rgba(255, 255, 255, 0.1); } @@ -51,13 +55,12 @@ line-height: 150%; font-size: 11px; font-family: Lucida Console, Monaco, Nimbus Mono L, monospace, serif; + color: #FFF; } - .page.log .container .error { - color: #FFA4A4; - white-space: pre-wrap; + .page.log .container select { + vertical-align: top; } - .page.log .container .debug { color: lightgrey; } .page.log .container .time { clear: both; @@ -68,10 +71,25 @@ position: relative; } - .page.log .container .time:last-child { display: none; } - - .page.log .container .time span { - float: right; - width: 86%; + .page.log [data-filter=INFO] .error, + .page.log [data-filter=INFO] .debug, + .page.log [data-filter=ERROR] .debug, + .page.log [data-filter=ERROR] .info, + .page.log [data-filter=DEBUG] .info, + .page.log [data-filter=DEBUG] .error { + display: none; } + .page.log .container .type { + margin-left: 10px; + } + + .page.log .container .message { + float: right; + width: 86%; + white-space: pre-wrap; + } + + .page.log .container .error { color: #FFA4A4; } + .page.log .container .debug { opacity: .4; } + diff --git a/couchpotato/core/plugins/log/static/log.js b/couchpotato/core/plugins/log/static/log.js index c5effeba..670f5b75 100644 --- a/couchpotato/core/plugins/log/static/log.js +++ b/couchpotato/core/plugins/log/static/log.js @@ -2,6 +2,7 @@ Page.Log = new Class({ Extends: PageBase, + order: 60, name: 'log', title: 'Show recent logs.', has_tab: false, @@ -26,25 +27,46 @@ Page.Log = new Class({ 'nr': nr }, 'onComplete': function(json){ - self.log.set('html', self.addColors(json.log)); + self.log.set('text', ''); + self.log.adopt(self.createLogElements(json.log)); self.log.removeClass('loading'); - new Fx.Scroll(window, {'duration': 0}).toBottom(); + var nav = new Element('ul.nav', { + 'events': { + 'click:relay(li.select)': function(e, el){ + self.getLogs(parseInt(el.get('text'))-1); + } + } + }); - var nav = new Element('ul.nav').inject(self.log, 'top'); + // Type selection + new Element('li.filter').grab( + new Element('select', { + 'events': { + 'change': function(){ + var type_filter = this.getSelected()[0].get('value'); + self.log.set('data-filter', type_filter); + self.scrollToBottom(); + } + } + }).adopt( + new Element('option', {'value': 'ALL', 'text': 'Show all logs'}), + new Element('option', {'value': 'INFO', 'text': 'Show only INFO'}), + new Element('option', {'value': 'DEBUG', 'text': 'Show only DEBUG'}), + new Element('option', {'value': 'ERROR', 'text': 'Show only ERROR'}) + ) + ).inject(nav); + + // Selections for (var i = 0; i <= json.total; i++) { new Element('li', { 'text': i+1, - 'class': nr == i ? 'active': '', - 'events': { - 'click': function(e){ - self.getLogs(e.target.get('text')-1); - } - } + 'class': 'select ' + (nr == i ? 'active': '') }).inject(nav); } - new Element('li', { + // Clear button + new Element('li.clear', { 'text': 'clear', 'events': { 'click': function(){ @@ -56,26 +78,40 @@ Page.Log = new Class({ } } - }).inject(nav) + }).inject(nav); + + // Add to page + nav.inject(self.log, 'top'); + + self.scrollToBottom(); } }); }, - addColors: function(text){ + createLogElements: function(logs){ - text = text - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"') - .replace(/\u001b\[31m/gi, '') - .replace(/\u001b\[36m/gi, '') - .replace(/\u001b\[33m/gi, '') - .replace(/\u001b\[0m\n/gi, '
') - .replace(/\u001b\[0m/gi, ''); + var elements = []; - return '
' + text + '
'; + logs.each(function(log){ + elements.include(new Element('div', { + 'class': 'time ' + log.type.toLowerCase(), + 'text': log.time + }).adopt( + new Element('span.type', { + 'text': log.type + }), + new Element('span.message', { + 'text': log.message + }) + )) + }); + + return elements; + }, + + scrollToBottom: function(){ + new Fx.Scroll(window, {'duration': 0}).toBottom(); } }); diff --git a/couchpotato/core/plugins/profile/main.py b/couchpotato/core/plugins/profile/main.py index a7fdcd41..8dc59336 100644 --- a/couchpotato/core/plugins/profile/main.py +++ b/couchpotato/core/plugins/profile/main.py @@ -34,7 +34,7 @@ class ProfilePlugin(Plugin): }) addEvent('app.initialize', self.fill, priority = 90) - addEvent('app.load', self.forceDefaults) + addEvent('app.load', self.forceDefaults, priority = 110) def forceDefaults(self): @@ -87,7 +87,7 @@ class ProfilePlugin(Plugin): order = 0 for type in kwargs.get('types', []): profile['qualities'].append(type.get('quality')) - profile['wait_for'].append(tryInt(type.get('wait_for'))) + profile['wait_for'].append(tryInt(kwargs.get('wait_for', 0))) profile['finish'].append((tryInt(type.get('finish')) == 1) if order > 0 else True) profile['3d'].append(tryInt(type.get('3d'))) order += 1 diff --git a/couchpotato/core/plugins/profile/static/profile.js b/couchpotato/core/plugins/profile/static/profile.js index 8441b6a8..c62b137c 100644 --- a/couchpotato/core/plugins/profile/static/profile.js +++ b/couchpotato/core/plugins/profile/static/profile.js @@ -41,7 +41,7 @@ var Profile = new Class({ new Element('span', {'text':'Wait'}), new Element('input.inlay.xsmall', { 'type':'text', - 'value': data.types && data.types.length > 0 ? data.types[0].wait_for : 0 + 'value': data.wait_for && data.wait_for.length > 0 ? data.wait_for[0] : 0 }), new Element('span', {'text':'day(s) for a better quality.'}) ), @@ -63,8 +63,7 @@ var Profile = new Class({ data.types.include({ 'quality': quality, 'finish': data.finish[nr] || false, - '3d': data['3d'] ? data['3d'][nr] || false : false, - 'wait_for': data.wait_for[nr] || 0 + '3d': data['3d'] ? data['3d'][nr] || false : false }) }); } @@ -126,8 +125,7 @@ var Profile = new Class({ data.types.include({ 'quality': type.getElement('select').get('value'), 'finish': +type.getElement('input.finish[type=checkbox]').checked, - '3d': +type.getElement('input.3d[type=checkbox]').checked, - 'wait_for': 0 + '3d': +type.getElement('input.3d[type=checkbox]').checked }); }); @@ -340,8 +338,7 @@ Profile.Type = new Class({ return { 'quality': self.qualities.get('value'), 'finish': +self.finish.checked, - '3d': +self['3d'].checked, - 'wait_for': 0 + '3d': +self['3d'].checked } }, diff --git a/couchpotato/core/plugins/quality/main.py b/couchpotato/core/plugins/quality/main.py index c27815ed..1ca44424 100644 --- a/couchpotato/core/plugins/quality/main.py +++ b/couchpotato/core/plugins/quality/main.py @@ -21,11 +21,11 @@ class QualityPlugin(Plugin): } qualities = [ - {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]}, + {'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']}, {'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts', 'x264', 'h264']}, {'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']}, - {'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, - {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd'], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r')]}, + {'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]}, + {'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd'], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']}, {'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':[], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]}, {'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]}, {'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':[]}, @@ -35,9 +35,9 @@ class QualityPlugin(Plugin): ] pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr'] threed_tags = { - 'hsbs': [('half', 'sbs')], - 'fsbs': [('full', 'sbs')], - '3d': [], + 'sbs': [('half', 'sbs'), 'hsbs', ('full', 'sbs'), 'fsbs'], + 'ou': [('half', 'ou'), 'hou', ('full', 'ou'), 'fou'], + '3d': ['2d3d', '3d2d'], } cached_qualities = None @@ -49,6 +49,8 @@ class QualityPlugin(Plugin): addEvent('quality.guess', self.guess) addEvent('quality.pre_releases', self.preReleases) addEvent('quality.order', self.getOrder) + addEvent('quality.ishigher', self.isHigher) + addEvent('quality.isfinish', self.isFinish) addApiView('quality.size.save', self.saveSize) addApiView('quality.list', self.allView, docs = { @@ -177,7 +179,7 @@ class QualityPlugin(Plugin): return False - def guess(self, files, extra = None): + def guess(self, files, extra = None, size = None): if not extra: extra = {} # Create hash for cache @@ -205,15 +207,20 @@ class QualityPlugin(Plugin): self.calcScore(score, quality, contains_score, threedscore) + # Evaluate score based on size + for quality in qualities: + size_score = self.guessSizeScore(quality, size = size) + self.calcScore(score, quality, size_score, penalty = False) + # Try again with loose testing for quality in qualities: loose_score = self.guessLooseScore(quality, extra = extra) - self.calcScore(score, quality, loose_score) + self.calcScore(score, quality, loose_score, penalty = False) - # Return nothing if all scores are 0 + # Return nothing if all scores are <= 0 has_non_zero = 0 for s in score: - if score[s] > 0: + if score[s]['score'] > 0: has_non_zero += 1 if not has_non_zero: @@ -281,7 +288,7 @@ class QualityPlugin(Plugin): return 1, key if list(set([key]) & set(words)): - log.debug('Found %s in %s', (tag, cur_file)) + log.debug('Found %s in %s', (key, cur_file)) return 1, key return 0, None @@ -308,7 +315,20 @@ class QualityPlugin(Plugin): return score - def calcScore(self, score, quality, add_score, threedscore = (0, None)): + + def guessSizeScore(self, quality, size = None): + + score = 0 + + if size: + + if tryInt(quality['size_min']) <= tryInt(size) <= tryInt(quality['size_max']): + log.info2('Found %s via release size: %s MB < %s MB < %s MB', (quality['identifier'], quality['size_min'], size, quality['size_max'])) + score += 5 + + return score + + def calcScore(self, score, quality, add_score, threedscore = (0, None), penalty = True): score[quality['identifier']]['score'] += add_score @@ -325,30 +345,79 @@ class QualityPlugin(Plugin): for q in self.qualities: self.cached_order[q.get('identifier')] = self.qualities.index(q) - if add_score != 0: + if penalty and add_score != 0: for allow in quality.get('allow', []): score[allow]['score'] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5 + # Give panelty for all lower qualities + for q in self.qualities[self.order.index(quality.get('identifier'))+1:]: + score[q.get('identifier')]['score'] -= 1 + + def isFinish(self, quality, profile): + if not isinstance(profile, dict) or not profile.get('qualities'): + return False + + try: + quality_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(quality.get('is_3d', 0))][0] + return profile['finish'][quality_order] + except: + return False + + def isHigher(self, quality, compare_with, profile = None): + if not isinstance(profile, dict) or not profile.get('qualities'): + profile = {'qualities': self.order} + + # Try to find quality in profile, if not found: a quality we do not want is lower than anything else + try: + quality_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(quality.get('is_3d', 0))][0] + except: + log.debug('Quality %s not found in profile identifiers %s', (quality['identifier'] + (' 3D' if quality.get('is_3d', 0) else ''), \ + [identifier + ('3D' if (profile['3d'][i] if profile.get('3d') else 0) else '') for i, identifier in enumerate(profile['qualities'])])) + return 'lower' + + # Try to find compare quality in profile, if not found: anything is higher than a not wanted quality + try: + compare_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == compare_with['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(compare_with.get('is_3d', 0))][0] + except: + log.debug('Compare quality %s not found in profile identifiers %s', (compare_with['identifier'] + (' 3D' if compare_with.get('is_3d', 0) else ''), \ + [identifier + (' 3D' if (profile['3d'][i] if profile.get('3d') else 0) else '') for i, identifier in enumerate(profile['qualities'])])) + return 'higher' + + # Note to self: a lower number means higher quality + if quality_order > compare_order: + return 'lower' + elif quality_order == compare_order: + return 'equal' + else: + return 'higher' + def doTest(self): tests = { - 'Movie Name (1999)-DVD-Rip.avi': 'dvdrip', - 'Movie Name 1999 720p Bluray.mkv': '720p', - 'Movie Name 1999 BR-Rip 720p.avi': 'brrip', - 'Movie Name 1999 720p Web Rip.avi': 'scr', - 'Movie Name 1999 Web DL.avi': 'brrip', - 'Movie.Name.1999.1080p.WEBRip.H264-Group': 'scr', - 'Movie.Name.1999.DVDRip-Group': 'dvdrip', - 'Movie.Name.1999.DVD-Rip-Group': 'dvdrip', - 'Movie.Name.1999.DVD-R-Group': 'dvdr', - 'Movie.Name.Camelie.1999.720p.BluRay.x264-Group': '720p', - 'Movie.Name.2008.German.DL.AC3.1080p.BluRay.x264-Group': '1080p', - 'Movie.Name.2004.GERMAN.AC3D.DL.1080p.BluRay.x264-Group': '1080p', + 'Movie Name (1999)-DVD-Rip.avi': {'size': 700, 'quality': 'dvdrip'}, + 'Movie Name 1999 720p Bluray.mkv': {'size': 4200, 'quality': '720p'}, + 'Movie Name 1999 BR-Rip 720p.avi': {'size': 1000, 'quality': 'brrip'}, + 'Movie Name 1999 720p Web Rip.avi': {'size': 1200, 'quality': 'scr'}, + 'Movie Name 1999 Web DL.avi': {'size': 800, 'quality': 'brrip'}, + 'Movie.Name.1999.1080p.WEBRip.H264-Group': {'size': 1500, 'quality': 'scr'}, + 'Movie.Name.1999.DVDRip-Group': {'size': 750, 'quality': 'dvdrip'}, + 'Movie.Name.1999.DVD-Rip-Group': {'size': 700, 'quality': 'dvdrip'}, + 'Movie.Name.1999.DVD-R-Group': {'size': 4500, 'quality': 'dvdr'}, + 'Movie.Name.Camelie.1999.720p.BluRay.x264-Group': {'size': 5500, 'quality': '720p'}, + 'Movie.Name.2008.German.DL.AC3.1080p.BluRay.x264-Group': {'size': 8500, 'extra': {'resolution_width': 1920, 'resolution_height': 1080} , 'quality': '1080p'}, + 'Movie.Name.2004.GERMAN.AC3D.DL.1080p.BluRay.x264-Group': {'size': 8000, 'quality': '1080p'}, + 'Movie.Name.2013.BR-Disk-Group.iso': {'size': 48000, 'quality': 'bd50'}, + 'Movie.Name.2013.2D+3D.BR-Disk-Group.iso': {'size': 52000, 'quality': 'bd50', 'is_3d': True}, + 'Movie.Rising.Name.Girl.2011.NTSC.DVD9-GroupDVD': {'size': 7200, 'quality': 'dvdr'}, + 'Movie Name (2013) 2D + 3D': {'size': 49000, 'quality': 'bd50', 'is_3d': True}, + 'Movie Monuments 2013 BrRip 1080p': {'size': 1800, 'quality': 'brrip'}, + 'Movie Monuments 2013 BrRip 720p': {'size': 1300, 'quality': 'brrip'}, } correct = 0 for name in tests: - success = self.guess([name]).get('identifier') == tests[name] + test_quality = self.guess(files = [name], extra = tests[name].get('extra', None), size = tests[name].get('size', None)) or {} + success = test_quality.get('identifier') == tests[name]['quality'] and test_quality.get('is_3d') == tests[name].get('is_3d', False) if not success: log.error('%s failed check, thinks it\'s %s', (name, self.guess([name]).get('identifier'))) diff --git a/couchpotato/core/plugins/release/main.py b/couchpotato/core/plugins/release/main.py index 72a0a502..7b227127 100644 --- a/couchpotato/core/plugins/release/main.py +++ b/couchpotato/core/plugins/release/main.py @@ -3,6 +3,7 @@ import os import time import traceback +from CodernityDB.database import RecordDeleted from couchpotato import md5, get_db from couchpotato.api import addApiView from couchpotato.core.event import fireEvent, addEvent @@ -58,7 +59,7 @@ class Release(Plugin): # Clean releases that didn't have activity in the last week addEvent('app.load', self.cleanDone) - fireEvent('schedule.interval', 'movie.clean_releases', self.cleanDone, hours = 4) + fireEvent('schedule.interval', 'movie.clean_releases', self.cleanDone, hours = 12) def cleanDone(self): log.debug('Removing releases from dashboard') @@ -68,6 +69,27 @@ class Release(Plugin): db = get_db() + # Get (and remove) parentless releases + releases = db.all('release', with_doc = True) + media_exist = [] + for release in releases: + if release.get('key') in media_exist: + continue + + try: + db.get('id', release.get('key')) + media_exist.append(release.get('key')) + except RecordDeleted: + db.delete(release['doc']) + log.debug('Deleted orphaned release: %s', release['doc']) + except: + log.debug('Failed cleaning up orphaned releases: %s', traceback.format_exc()) + + del media_exist + + # Reindex statuses + db.reindex_index('media_status') + # get movies last_edit more than a week ago medias = fireEvent('media.with_status', 'done', single = True) @@ -107,6 +129,7 @@ class Release(Plugin): 'media_id': media['_id'], 'identifier': release_identifier, 'quality': group['meta_data']['quality'].get('identifier'), + 'is_3d': group['meta_data']['quality'].get('is_3d', 0), 'last_edit': int(time.time()), 'status': 'done' } @@ -315,10 +338,12 @@ class Release(Plugin): def tryDownloadResult(self, results, media, quality_custom, manual = False): + wait_for = False + let_through = False + filtered_results = [] + + # If a single release comes through the "wait for", let through all for rel in results: - if not quality_custom.get('finish', False) and quality_custom.get('wait_for', 0) > 0 and rel.get('age') <= quality_custom.get('wait_for', 0): - log.info('Ignored, waiting %s days: %s', (quality_custom.get('wait_for'), rel['name'])) - continue if rel['status'] in ['ignored', 'failed']: log.info('Ignored: %s', rel['name']) @@ -328,13 +353,30 @@ class Release(Plugin): log.info('Ignored, score to low: %s', rel['name']) continue + rel['wait_for'] = False + if quality_custom.get('index') != 0 and quality_custom.get('wait_for', 0) > 0 and rel.get('age') <= quality_custom.get('wait_for', 0): + rel['wait_for'] = True + else: + let_through = True + + filtered_results.append(rel) + + # Loop through filtered results + for rel in filtered_results: + + # Only wait if not a single release is old enough + if rel.get('wait_for') and not let_through: + log.info('Ignored, waiting %s days: %s', (quality_custom.get('wait_for') - rel.get('age'), rel['name'])) + wait_for = True + continue + downloaded = fireEvent('release.download', data = rel, media = media, manual = manual, single = True) if downloaded is True: return True elif downloaded != 'try_next': break - return False + return wait_for def createFromSearch(self, search_results, media, quality): @@ -406,7 +448,7 @@ class Release(Plugin): rel = db.get('id', release_id) if rel and rel.get('status') != status: - release_name = rel.get('name') + release_name = rel['info'].get('name') if rel.get('files'): for file_type in rel.get('files', {}): if file_type == 'movie': diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 5424874b..3d56aa9c 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -112,7 +112,7 @@ class Renamer(Plugin): return if not base_folder: - base_folder = self.conf('from') + base_folder = sp(self.conf('from')) from_folder = sp(self.conf('from')) to_folder = sp(self.conf('to')) @@ -314,8 +314,14 @@ class Renamer(Plugin): 'cd': '', 'cd_nr': '', 'mpaa': media['info'].get('mpaa', ''), + 'mpaa_only': media['info'].get('mpaa', ''), 'category': category_label, + '3d': '3D' if group['meta_data']['quality'].get('is_3d', 0) else '', + '3d_type': group['meta_data'].get('3d_type'), } + + if replacements['mpaa_only'] not in ('G', 'PG', 'PG-13', 'R', 'NC-17'): + replacements['mpaa_only'] = 'Not Rated' for file_type in group['files']: @@ -412,8 +418,12 @@ class Renamer(Plugin): # Don't add language if multiple languages in 1 subtitle file if len(sub_langs) == 1: - sub_name = sub_name.replace(replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext'])) - rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name) + sub_suffix = '%s.%s' % (sub_langs[0], replacements['ext']) + + # Don't add language to subtitle file it it's already there + if not sub_name.endswith(sub_suffix): + sub_name = sub_name.replace(replacements['ext'], sub_suffix) + rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name) rename_files = mergeDicts(rename_files, rename_extras) @@ -440,17 +450,15 @@ class Renamer(Plugin): remove_leftovers = True # Mark movie "done" once it's found the quality with the finish check + profile = None try: if media.get('status') == 'active' and media.get('profile_id'): profile = db.get('id', media['profile_id']) - if group['meta_data']['quality']['identifier'] in profile.get('qualities', []): - nr = profile['qualities'].index(group['meta_data']['quality']['identifier']) - finish = profile['finish'][nr] - if finish: - mdia = db.get('id', media['_id']) - mdia['status'] = 'done' - mdia['last_edit'] = int(time.time()) - db.update(mdia) + if fireEvent('quality.isfinish', group['meta_data']['quality'], profile, single = True): + mdia = db.get('id', media['_id']) + mdia['status'] = 'done' + mdia['last_edit'] = int(time.time()) + db.update(mdia) except Exception as e: log.error('Failed marking movie finished: %s', (traceback.format_exc())) @@ -461,18 +469,19 @@ class Renamer(Plugin): # When a release already exists if release.get('status') == 'done': - release_order = quality_order.index(release['quality']) - group_quality_order = quality_order.index(group['meta_data']['quality']['identifier']) + # This is where CP removes older, lesser quality releases or releases that are not wanted anymore + is_higher = fireEvent('quality.ishigher', \ + group['meta_data']['quality'], {'identifier': release['quality'], 'is_3d': release.get('is_3d', 0)}, profile, single = True) - # This is where CP removes older, lesser quality releases - if release_order > group_quality_order: - log.info('Removing lesser quality %s for %s.', (media_title, release.get('quality'))) + if is_higher == 'higher': + log.info('Removing lesser or not wanted quality %s for %s.', (media_title, release.get('quality'))) for file_type in release.get('files', {}): for release_file in release['files'][file_type]: remove_files.append(release_file) remove_releases.append(release) + # Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc - elif release_order == group_quality_order: + elif is_higher == 'equal': log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (media_title, release.get('quality'))) for file_type in release.get('files', {}): for release_file in release['files'][file_type]: @@ -824,7 +833,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) def replaceDoubles(self, string): replaces = [ - ('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '), + ('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '), (' \\\\', '\\\\'), (' /', '/'), ('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'), ] @@ -1056,6 +1065,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) release_download.update({ 'imdb_id': getIdentifier(media), 'quality': rls['quality'], + 'is_3d': rls['is_3d'], 'protocol': rls.get('info', {}).get('protocol') or rls.get('info', {}).get('type'), 'release_id': rls['_id'], }) @@ -1195,6 +1205,8 @@ rename_options = { 'first': 'First letter (M)', 'quality': 'Quality (720p)', 'quality_type': '(HD) or (SD)', + '3d': '3D', + '3d_type': '3D Type (Full SBS)', 'video': 'Video (x264)', 'audio': 'Audio (DTS)', 'group': 'Releasegroup name', @@ -1207,7 +1219,8 @@ rename_options = { 'imdb_id': 'IMDB id (tt0123456)', 'cd': 'CD number (cd1)', 'cd_nr': 'Just the cd nr. (1)', - 'mpaa': 'MPAA Rating', + 'mpaa': 'MPAA or other certification', + 'mpaa_only': 'MPAA only certification (G|PG|PG-13|R|NC-17|Not Rated)', 'category': 'Category label', }, } diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index 1f636adf..167e76cd 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -6,7 +6,7 @@ import traceback from couchpotato import get_db from couchpotato.core.event import fireEvent, addEvent -from couchpotato.core.helpers.encoding import toUnicode, simplifyString, sp +from couchpotato.core.helpers.encoding import toUnicode, simplifyString, sp, ss from couchpotato.core.helpers.variable import getExt, getImdb, tryInt, \ splitString, getIdentifier from couchpotato.core.logger import CPLog @@ -40,6 +40,17 @@ class Scanner(Plugin): 'trailer': ['mov', 'mp4', 'flv'] } + threed_types = { + 'Half SBS': [('half', 'sbs'), ('h', 'sbs'), 'hsbs'], + 'Full SBS': [('full', 'sbs'), ('f', 'sbs'), 'fsbs'], + 'SBS': ['sbs'], + 'Half OU': [('half', 'ou'), ('h', 'ou'), 'hou'], + 'Full OU': [('full', 'ou'), ('h', 'ou'), 'fou'], + 'OU': ['ou'], + 'Frame Packed': ['mvc', ('complete', 'bluray')], + '3D': ['3d'] + } + file_types = { 'subtitle': ('subtitle', 'subtitle'), 'subtitle_extra': ('subtitle', 'subtitle_extra'), @@ -64,6 +75,16 @@ class Scanner(Plugin): 'video': ['x264', 'H264', 'DivX', 'Xvid'] } + resolutions = { + '1080p': {'resolution_width': 1920, 'resolution_height': 1080, 'aspect': 1.78}, + '1080i': {'resolution_width': 1920, 'resolution_height': 1080, 'aspect': 1.78}, + '720p': {'resolution_width': 1280, 'resolution_height': 720, 'aspect': 1.78}, + '720i': {'resolution_width': 1280, 'resolution_height': 720, 'aspect': 1.78}, + '480p': {'resolution_width': 640, 'resolution_height': 480, 'aspect': 1.33}, + '480i': {'resolution_width': 640, 'resolution_height': 480, 'aspect': 1.33}, + 'default': {'resolution_width': 0, 'resolution_height': 0, 'aspect': 1}, + } + audio_codec_map = { 0x2000: 'AC3', 0x2001: 'DTS', @@ -85,8 +106,8 @@ class Scanner(Plugin): 'HDTV': ['hdtv'] } - clean = '[ _\,\.\(\)\[\]\-]?(3d|hsbs|sbs|extended.cut|directors.cut|french|swedisch|danish|dutch|swesub|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip' \ - '|hdtvrip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|cd[1-9]|\[.*\])([ _\,\.\(\)\[\]\-]|$)' + clean = '([ _\,\.\(\)\[\]\-]|^)(3d|hsbs|sbs|ou|extended.cut|directors.cut|french|fr|swedisch|sw|danish|dutch|nl|swesub|subs|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip' \ + '|hdtvrip|webdl|web.dl|webrip|web.rip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|hc|\[.*\])(?=[ _\,\.\(\)\[\]\-]|$)' multipart_regex = [ '[ _\.-]+cd[ _\.-]*([0-9a-d]+)', #*cd1 '[ _\.-]+dvd[ _\.-]*([0-9a-d]+)', #*dvd1 @@ -164,7 +185,7 @@ class Scanner(Plugin): identifiers = [identifier] # Identifier with quality - quality = fireEvent('quality.guess', [file_path], single = True) if not is_dvd_file else {'identifier':'dvdr'} + quality = fireEvent('quality.guess', files = [file_path], size = self.getFileSize(file_path), single = True) if not is_dvd_file else {'identifier':'dvdr'} if quality: identifier_with_quality = '%s %s' % (identifier, quality.get('identifier', '')) identifiers = [identifier_with_quality, identifier] @@ -431,28 +452,39 @@ class Scanner(Plugin): for cur_file in files: if not self.filesizeBetween(cur_file, self.file_sizes['movie']): continue # Ignore smaller files - meta = self.getMeta(cur_file) + if not data.get('audio'): # Only get metadata from first media file + meta = self.getMeta(cur_file) - try: - data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video'])) - data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio'])) - data['resolution_width'] = meta.get('resolution_width', 720) - data['resolution_height'] = meta.get('resolution_height', 480) - data['audio_channels'] = meta.get('audio_channels', 2.0) - data['aspect'] = round(float(meta.get('resolution_width', 720)) / meta.get('resolution_height', 480), 2) - except: - log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc())) - pass + try: + data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video'])) + data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio'])) + data['audio_channels'] = meta.get('audio_channels', 2.0) + if meta.get('resolution_width'): + data['resolution_width'] = meta.get('resolution_width') + data['resolution_height'] = meta.get('resolution_height') + data['aspect'] = round(float(meta.get('resolution_width')) / meta.get('resolution_height', 1), 2) + else: + data.update(self.getResolution(cur_file)) + except: + log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc())) + pass - if data.get('audio'): break + data['size'] = data.get('size', 0) + self.getFileSize(cur_file) - # Use the quality guess first, if that failes use the quality we wanted to download data['quality'] = None + quality = fireEvent('quality.guess', size = data['size'], files = files, extra = data, single = True) + + # Use the quality that we snatched but check if it matches our guess if release_download and release_download.get('quality'): data['quality'] = fireEvent('quality.single', release_download.get('quality'), single = True) + data['quality']['is_3d'] = release_download.get('is_3d', 0) + if data['quality']['identifier'] != quality['identifier']: + log.info('Different quality snatched than detected for %s: %s vs. %s. Assuming snatched quality is correct.', (files[0], data['quality']['identifier'], quality['identifier'])) + if data['quality']['is_3d'] != quality['is_3d']: + log.info('Different 3d snatched than detected for %s: %s vs. %s. Assuming snatched 3d is correct.', (files[0], data['quality']['is_3d'], quality['is_3d'])) if not data['quality']: - data['quality'] = fireEvent('quality.guess', files = files, extra = data, single = True) + data['quality'] = quality if not data['quality']: data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True) @@ -462,9 +494,25 @@ class Scanner(Plugin): filename = re.sub('(.cp\(tt[0-9{7}]+\))', '', files[0]) data['group'] = self.getGroup(filename[len(folder):]) data['source'] = self.getSourceMedia(filename) - + if data['quality'].get('is_3d', 0): + data['3d_type'] = self.get3dType(filename) return data + def get3dType(self, filename): + filename = ss(filename) + + words = re.split('\W+', filename.lower()) + + for key in self.threed_types: + tags = self.threed_types.get(key, []) + + for tag in tags: + if (isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words)) or (isinstance(tag, (str, unicode)) and ss(tag.lower()) in words): + log.debug('Found %s in %s', (tag, filename)) + return key + + return '' + def getMeta(self, filename): try: @@ -708,19 +756,26 @@ class Scanner(Plugin): if not file_size: file_size = [] try: - return (file_size.get('min', 0) * 1048576) < os.path.getsize(file) < (file_size.get('max', 100000) * 1048576) + return file_size.get('min', 0) < self.getFileSize(file) < file_size.get('max', 100000) except: log.error('Couldn\'t get filesize of %s.', file) return False - def createStringIdentifier(self, file_path, folder = '', exclude_filename = False): + def getFileSize(self, file): + try: + return os.path.getsize(file) / 1024 / 1024 + except: + return None - year = self.findYear(file_path) + def createStringIdentifier(self, file_path, folder = '', exclude_filename = False): identifier = file_path.replace(folder, '').lstrip(os.path.sep) # root folder identifier = os.path.splitext(identifier)[0] # ext + # Make sure the identifier is lower case as all regex is with lower case tags + identifier = identifier.lower() + try: path_split = splitString(identifier, os.path.sep) identifier = path_split[-2] if len(path_split) > 1 and len(path_split[-2]) > len(path_split[-1]) else path_split[-1] # Only get filename @@ -735,8 +790,13 @@ class Scanner(Plugin): # remove cptag identifier = self.removeCPTag(identifier) - # groups, release tags, scenename cleaner, regex isn't correct - identifier = re.sub(self.clean, '::', simplifyString(identifier)).strip(':') + # simplify the string + identifier = simplifyString(identifier) + + year = self.findYear(file_path) + + # groups, release tags, scenename cleaner + identifier = re.sub(self.clean, '::', identifier).strip(':') # Year if year and identifier[:4] != year: @@ -785,6 +845,14 @@ class Scanner(Plugin): except: return '' + def getResolution(self, filename): + try: + for key in self.resolutions: + if key in filename.lower() and key != 'default': + return self.resolutions[key] + except: + return self.resolutions['default'] + def getGroup(self, file): try: match = re.findall('\-([A-Z0-9]+)[\.\/]', file, re.I) diff --git a/couchpotato/core/plugins/subtitle.py b/couchpotato/core/plugins/subtitle.py index 9fd7ef16..fdb640b1 100644 --- a/couchpotato/core/plugins/subtitle.py +++ b/couchpotato/core/plugins/subtitle.py @@ -32,7 +32,7 @@ class Subtitle(Plugin): for lang in self.getLanguages(): if lang not in available_languages: - download = subliminal.download_subtitles(files, multi = True, force = False, languages = [lang], services = self.services, cache_dir = Env.get('cache_dir')) + download = subliminal.download_subtitles(files, multi = True, force = self.conf('force'), languages = [lang], services = self.services, cache_dir = Env.get('cache_dir')) for subtitle in download: downloaded.extend(download[subtitle]) @@ -72,6 +72,14 @@ config = [{ 'name': 'languages', 'description': ('Comma separated, 2 letter country code.', 'Example: en, nl. See the codes at on Wikipedia'), }, + { + 'advanced': True, + 'name': 'force', + 'label': 'Force', + 'description': ('Force download all languages (including embedded).', 'This will also overwrite all existing subtitles.'), + 'default': False, + 'type': 'bool', + }, ], }, ], diff --git a/couchpotato/core/plugins/userscript/static/userscript.js b/couchpotato/core/plugins/userscript/static/userscript.js index 2ef1ea6a..d8caeb3f 100644 --- a/couchpotato/core/plugins/userscript/static/userscript.js +++ b/couchpotato/core/plugins/userscript/static/userscript.js @@ -2,6 +2,7 @@ Page.Userscript = new Class({ Extends: PageBase, + order: 80, name: 'userscript', has_tab: false, diff --git a/couchpotato/core/plugins/wizard/static/wizard.js b/couchpotato/core/plugins/wizard/static/wizard.js index b43e3985..fdd7b743 100644 --- a/couchpotato/core/plugins/wizard/static/wizard.js +++ b/couchpotato/core/plugins/wizard/static/wizard.js @@ -2,6 +2,7 @@ Page.Wizard = new Class({ Extends: Page.Settings, + order: 70, name: 'wizard', has_tab: false, wizard_only: true, diff --git a/couchpotato/runner.py b/couchpotato/runner.py index dcdac5c8..1684eddb 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -262,8 +262,14 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En # Go go go! from tornado.ioloop import IOLoop + from tornado.autoreload import add_reload_hook loop = IOLoop.current() + # Reload hook + def test(): + fireEvent('app.shutdown') + add_reload_hook(test) + # Some logging and fire load event try: log.info('Starting server on port %(port)s', config) except: pass diff --git a/couchpotato/static/scripts/couchpotato.js b/couchpotato/static/scripts/couchpotato.js index ffc0e9aa..e08e2c3d 100644 --- a/couchpotato/static/scripts/couchpotato.js +++ b/couchpotato/static/scripts/couchpotato.js @@ -137,19 +137,34 @@ createPages: function(){ var self = this; + var pages = []; Object.each(Page, function(page_class, class_name){ var pg = new Page[class_name](self, {}); self.pages[class_name] = pg; - self.fireEvent('load'+class_name); - - $(pg).inject(self.content); + pages.include({ + 'order': pg.order, + 'name': class_name, + 'class': pg + }); }); + pages.stableSort(self.sortPageByOrder).each(function(page){ + page['class'].load(); + self.fireEvent('load'+page.name); + $(page['class']).inject(self.content); + }); + + delete pages; + self.fireEvent('load'); }, + sortPageByOrder: function(a, b){ + return (a.order || 100) - (b.order || 100) + }, + openPage: function(url) { var self = this; diff --git a/couchpotato/static/scripts/page.js b/couchpotato/static/scripts/page.js index 58ba5acd..57b8b108 100644 --- a/couchpotato/static/scripts/page.js +++ b/couchpotato/static/scripts/page.js @@ -6,6 +6,7 @@ var PageBase = new Class({ }, + order: 1, has_tab: true, name: '', @@ -16,6 +17,10 @@ var PageBase = new Class({ // Create main page container self.el = new Element('div.page.'+self.name); + }, + + load: function(){ + var self = this; // Create tab for page if(self.has_tab){ @@ -26,6 +31,7 @@ var PageBase = new Class({ 'text': self.name.capitalize() }); } + }, open: function(action, params){ diff --git a/couchpotato/static/scripts/page/home.js b/couchpotato/static/scripts/page/home.js index bf435c14..7cde7d2a 100644 --- a/couchpotato/static/scripts/page/home.js +++ b/couchpotato/static/scripts/page/home.js @@ -104,7 +104,7 @@ Page.Home = new Class({ // Make all thumbnails the same size self.soon_list.addEvent('loaded', function(){ - var images = $(self.soon_list).getElements('.poster'), + var images = $(self.soon_list).getElements('.poster, .no_thumbnail'), timer, highest = 100; diff --git a/couchpotato/static/scripts/page/settings.js b/couchpotato/static/scripts/page/settings.js index 09155486..de5ffcbc 100644 --- a/couchpotato/static/scripts/page/settings.js +++ b/couchpotato/static/scripts/page/settings.js @@ -2,6 +2,7 @@ Page.Settings = new Class({ Extends: PageBase, + order: 50, name: 'settings', title: 'Change settings.', wizard_only: false, @@ -112,7 +113,7 @@ Page.Settings = new Class({ }, sortByOrder: function(a, b){ - return (a.order || 100) - (b.order || 100) + return (a.order || 100) - (b.order || 100) }, create: function(json){