Merge remote-tracking branch 'upstream/develop' into develop

This commit is contained in:
Dan Boehm
2014-05-09 13:04:36 -05:00
51 changed files with 737 additions and 299 deletions

View File

@@ -90,7 +90,11 @@ class Core(Plugin):
def shutdown():
self.initShutdown()
IOLoop.current().add_callback(shutdown)
if IOLoop.current()._closing:
shutdown()
else:
IOLoop.current().add_callback(shutdown)
return 'shutdown'
@@ -139,7 +143,8 @@ class Core(Plugin):
log.debug('Safe to shutdown/restart')
try:
IOLoop.current().stop()
if not IOLoop.current()._closing:
IOLoop.current().stop()
except RuntimeError:
pass
except:

View File

@@ -28,6 +28,7 @@ class Database(object):
addEvent('database.setup_index', self.setupIndex)
addEvent('app.migrate', self.migrate)
addEvent('app.after_shutdown', self.close)
def getDB(self):
@@ -37,6 +38,9 @@ class Database(object):
return self.db
def close(self, **kwargs):
self.getDB().close()
def setupIndex(self, index_name, klass):
self.indexes.append(index_name)
@@ -412,7 +416,10 @@ class Database(object):
empty_info = True
rel['info'] = {}
quality = quality_link[rel.get('quality_id')]
quality = quality_link.get(rel.get('quality_id'))
if not quality:
continue
release_status = statuses.get(rel.get('status_id')).get('identifier')
if rel['info'].get('download_id'):

View File

@@ -25,7 +25,6 @@ class MediaBase(Plugin):
def onComplete():
try:
db = get_db()
media = fireEvent('media.get', media_id, single = True)
event_name = '%s.searcher.single' % media.get('type')

View File

@@ -107,8 +107,7 @@ class MediaPlugin(MediaBase):
def handler():
fireEvent(event, media_id = media_id, on_complete = self.createOnComplete(media_id))
if handler:
return handler
return handler
except:
log.error('Refresh handler for non existing media: %s', traceback.format_exc())
@@ -361,13 +360,18 @@ class MediaPlugin(MediaBase):
media = db.get('id', media_id)
if media:
deleted = False
media_releases = fireEvent('release.for_media', media['_id'], single = True)
if delete_from == 'all':
# Delete connected releases
for release in media_releases:
db.delete(release)
db.delete(media)
deleted = True
else:
media_releases = fireEvent('release.for_media', media['_id'], single = True)
total_releases = len(media_releases)
total_deleted = 0
new_media_status = None

View File

@@ -88,10 +88,14 @@ class Provider(Plugin):
if data and len(data) > 0:
try:
data = XMLTree.fromstring(ss(data))
data = XMLTree.fromstring(data)
return self.getElements(data, item_path)
except:
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
try:
data = XMLTree.fromstring(ss(data))
return self.getElements(data, item_path)
except:
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
return []
@@ -298,7 +302,7 @@ class ResultList(list):
old_score = new_result['score']
new_result['score'] = int(old_score * is_correct_weight)
log.info('Found correct release with weight %.02f, old_score(%d) now scaled to score(%d)', (
log.info2('Found correct release with weight %.02f, old_score(%d) now scaled to score(%d)', (
is_correct_weight,
old_score,
new_result['score']

View File

@@ -50,8 +50,8 @@ class Base(NZBProvider):
def extra_check(item):
parts = re.search('available:.(?P<parts>\d+)./.(?P<total>\d+)', info.text)
total = tryInt(parts.group('total'))
parts = tryInt(parts.group('parts'))
total = float(tryInt(parts.group('total')))
parts = float(tryInt(parts.group('parts')))
if (total / parts) < 1 and ((total / parts) < 0.95 or ((total / parts) >= 0.95 and not ('par2' in info.text.lower() or 'pa3' in info.text.lower()))):
log.info2('Wrong: \'%s\', not complete: %s out of %s', (item['name'], parts, total))

View File

@@ -45,7 +45,7 @@ class Base(NZBProvider, RSS):
def _searchOnHost(self, host, media, quality, results):
query = self.buildUrl(media, host['api_key'])
query = self.buildUrl(media, host)
url = '%s&%s' % (self.getUrl(host['host']), query)
nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})
@@ -244,7 +244,7 @@ config = [{
},
{
'name': 'host',
'default': 'api.nzb.su,dognzb.cr,nzbs.org,https://index.nzbgeek.info, https://smackdownonyou.com, https://www.nzbfinder.ws',
'default': 'api.nzb.su,api.dognzb.cr,nzbs.org,https://index.nzbgeek.info, https://smackdownonyou.com, https://www.nzbfinder.ws',
'description': 'The hostname of your newznab provider',
},
{

View File

@@ -44,7 +44,8 @@ class TorrentProvider(YarrProvider):
prop_name = 'proxy.%s' % proxy
last_check = float(Env.prop(prop_name, default = 0))
if last_check > time.time() - 1209600:
if last_check > time.time() - 86400:
continue
data = ''

View File

@@ -25,7 +25,7 @@ class Base(TorrentProvider):
def _search(self, media, quality, results):
query = self.buildUrl(media)
query = self.buildUrl(media, quality)
url = "%s&%s" % (self.urls['search'], query)

View File

@@ -1,7 +1,6 @@
import traceback
from bs4 import BeautifulSoup
from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
@@ -16,7 +15,7 @@ class Base(TorrentProvider):
'test': 'https://www.bitsoup.me/',
'login': 'https://www.bitsoup.me/takelogin.php',
'login_check': 'https://www.bitsoup.me/my.php',
'search': 'https://www.bitsoup.me/browse.php?',
'search': 'https://www.bitsoup.me/browse.php?%s',
'baseurl': 'https://www.bitsoup.me/%s',
}
@@ -24,13 +23,7 @@ class Base(TorrentProvider):
def _searchOnTitle(self, title, movie, quality, results):
q = '"%s" %s' % (simplifyString(title), movie['info']['year'])
arguments = tryUrlencode({
'search': q,
})
url = "%s&%s" % (self.urls['search'], arguments)
url = self.urls['search'] % self.buildUrl(movie, quality)
url = self.urls['search'] % self.buildUrl(title, movie, quality)
data = self.getHTMLData(url)
if data:

View File

@@ -24,9 +24,9 @@ class Base(TorrentProvider):
http_time_between_calls = 1 # Seconds
def _search(self, media, quality, results):
def _searchOnTitle(self, title, media, quality, results):
url = self.buildUrl(media, quality)
url = self.buildUrl(title, media, quality)
data = self.getHTMLData(url)
if data:

View File

@@ -1,3 +1,4 @@
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.media._base.providers.torrent.base import TorrentProvider
@@ -18,16 +19,16 @@ class Base(TorrentProvider):
http_time_between_calls = 1 # Seconds
def _search(self, media, quality, results):
def _searchOnTitle(self, title, media, quality, results):
query = self.buildUrl(media)
query = '"%s" %s' % (title, media['info']['year'])
data = {
'/browse.php?': None,
'cata': 'yes',
'jxt': 8,
'jxw': 'b',
'search': query,
'search': tryUrlencode(query),
}
data = self.getJsonData(self.urls['search'], data = data)

View File

@@ -35,7 +35,11 @@ class Base(TorrentProvider):
def _search(self, movie, quality, results):
search_url = self.urls['search'] % (self.getDomain(), getIdentifier(movie), quality['identifier'])
domain = self.getDomain()
if not domain:
return
search_url = self.urls['search'] % (domain, getIdentifier(movie), quality['identifier'])
data = self.getJsonData(search_url)
@@ -43,21 +47,19 @@ class Base(TorrentProvider):
try:
for result in data.get('MovieList'):
try:
title = result['TorrentUrl'].split('/')[-1][:-8].replace('_', '.').strip('._')
title = title.replace('.-.', '-')
title = title.replace('..', '.')
except:
continue
if result['Quality'] and result['Quality'] not in result['MovieTitle']:
title = result['MovieTitle'] + ' BrRip ' + result['Quality']
else:
title = result['MovieTitle'] + ' BrRip'
results.append({
'id': result['MovieID'],
'name': title,
'url': result['TorrentMagnetUrl'],
'detail_url': self.urls['detail'] % (self.getDomain(), result['MovieID']),
'detail_url': self.urls['detail'] % (domain, result['MovieID']),
'size': self.parseSize(result['Size']),
'seeders': tryInt(result['TorrentSeeds']),
'leechers': tryInt(result['TorrentPeers'])
'leechers': tryInt(result['TorrentPeers']),
})
except:

View File

@@ -87,31 +87,23 @@ class Searcher(SearcherBase):
def containsOtherQuality(self, nzb, movie_year = None, preferred_quality = None):
if not preferred_quality: preferred_quality = {}
name = nzb['name']
size = nzb.get('size', 0)
nzb_words = re.split('\W+', simplifyString(name))
qualities = fireEvent('quality.all', single = True)
found = {}
for quality in qualities:
# Main in words
if quality['identifier'] in nzb_words:
found[quality['identifier']] = True
# Alt in words
if list(set(nzb_words) & set(quality['alternative'])):
found[quality['identifier']] = True
# Try guessing via quality tags
guess = fireEvent('quality.guess', [nzb.get('name')], single = True)
guess = fireEvent('quality.guess', files = [nzb.get('name')], size = nzb.get('size', None), single = True)
if guess:
found[guess['identifier']] = True
# Hack for older movies that don't contain quality tag
name = nzb['name']
size = nzb.get('size', 0)
year_name = fireEvent('scanner.name_year', name, single = True)
if len(found) == 0 and movie_year < datetime.datetime.now().year - 3 and not year_name.get('year', None):
if size > 3000: # Assume dvdr
if size > 20000: # Assume bd50
log.info('Quality was missing in name, assuming it\'s a BR-Disk based on the size: %s', size)
found['bd50'] = True
elif size > 3000: # Assume dvdr
log.info('Quality was missing in name, assuming it\'s a DVD-R based on the size: %s', size)
found['dvdr'] = True
else: # Assume dvdrip
@@ -123,7 +115,10 @@ class Searcher(SearcherBase):
if found.get(allowed):
del found[allowed]
return not (found.get(preferred_quality['identifier']) and len(found) == 1)
if found.get(preferred_quality['identifier']) and len(found) == 1:
return False
return found
def correct3D(self, nzb, preferred_quality = None):
if not preferred_quality: preferred_quality = {}

View File

@@ -139,7 +139,7 @@ class MovieBase(MovieTypeBase):
# Clean snatched history
for release in fireEvent('release.for_media', m['_id'], single = True):
if release.get('status') in ['downloaded', 'snatched', 'done']:
if release.get('status') in ['downloaded', 'snatched', 'seeding', 'done']:
if params.get('ignore_previous', False):
release['status'] = 'ignored'
db.update(release)

View File

@@ -2,6 +2,7 @@ Page.Manage = new Class({
Extends: PageBase,
order: 20,
name: 'manage',
title: 'Do stuff to your existing movies!',

View File

@@ -78,7 +78,7 @@ MA.IMDB = new Class({
create: function(){
var self = this;
self.id = self.movie.get('imdb') || self.movie.get('identifier');
self.id = self.movie.getIdentifier ? self.movie.getIdentifier() : self.get('imdb');
self.el = new Element('a.imdb', {
'title': 'Go to the IMDB page of ' + self.getTitle(),
@@ -684,7 +684,7 @@ MA.Readd = new Class({
var movie_done = self.movie.data.status == 'done';
if(self.movie.data.releases && !movie_done)
var snatched = self.movie.data.releases.filter(function(release){
return release.status && (release.status == 'snatched' || release.status == 'downloaded' || release.status == 'done');
return release.status && (release.status == 'snatched' || release.status == 'seeding' || release.status == 'downloaded' || release.status == 'done');
}).length;
if(movie_done || snatched && snatched > 0)
@@ -703,7 +703,7 @@ MA.Readd = new Class({
Api.request('movie.add', {
'data': {
'identifier': self.movie.get('identifier'),
'identifier': self.movie.getIdentifier(),
'ignore_previous': 1
}
});

View File

@@ -123,6 +123,7 @@
.movies.thumbs_list .movie {
width: 16.66667%;
height: auto;
min-height: 200px;
display: inline-block;
margin: 0;
padding: 0;
@@ -133,6 +134,7 @@
@media all and (max-width: 800px) {
.movies.thumbs_list .movie {
width: 25%;
min-height: 100px;
}
}

View File

@@ -300,6 +300,17 @@ var Movie = new Class({
self.el.removeClass(self.view+'_view')
},
getIdentifier: function(){
var self = this;
try {
return self.get('identifiers').imdb;
}
catch (e){ }
return self.get('imdb');
},
get: function(attr){
return this.data[attr] || this.data.info[attr]
},

View File

@@ -2,6 +2,7 @@ Page.Wanted = new Class({
Extends: PageBase,
order: 10,
name: 'wanted',
title: 'Gimmy gimmy gimmy!',
folder_browser: null,

View File

@@ -28,6 +28,20 @@ config = [{
'advanced': True,
'description': '(hours)',
},
{
'name': 'hide_wanted',
'default': False,
'type': 'bool',
'advanced': True,
'description': 'Hide the chart movies that are already in your wanted list.',
},
{
'name': 'hide_library',
'default': False,
'type': 'bool',
'advanced': True,
'description': 'Hide the chart movies that are already in your library.',
},
],
},
],

View File

@@ -36,7 +36,6 @@ class Charts(Plugin):
'charts': charts
}
def updateViewCache(self):
if self.update_in_progress:
@@ -49,6 +48,9 @@ class Charts(Plugin):
try:
self.update_in_progress = True
charts = fireEvent('automation.get_chart_list', merge = True)
for chart in charts:
chart['hide_wanted'] = self.conf('hide_wanted')
chart['hide_library'] = self.conf('hide_library')
self.setCache('charts_cached', charts, timeout = 7200 * tryInt(self.conf('update_interval', default = 12)))
except:
log.error('Failed refreshing charts')

View File

@@ -11,6 +11,17 @@
display: inline-block;
width: 50%;
vertical-align: top;
max-height: 510px;
overflow: hidden;
scrollbar-base-color: #4e5969;
}
.charts .chart:hover {
overflow-y: auto;
}
.charts .chart .media_result.hidden {
display: none;
}
.charts .refresh {

View File

@@ -89,7 +89,7 @@ var Charts = new Class({
}
});
var in_database_class = movie.in_wanted ? 'chart_in_wanted' : (movie.in_library ? 'chart_in_library' : ''),
var in_database_class = (chart.hide_wanted && movie.in_wanted) ? 'hidden' : (movie.in_wanted ? 'chart_in_wanted' : ((chart.hide_library && movie.in_library) ? 'hidden': (movie.in_library ? 'chart_in_library' : ''))),
in_database_title = movie.in_wanted ? 'Movie in wanted list' : (movie.in_library ? 'Movie in library' : '');
m.el

View File

@@ -1,4 +1,5 @@
from bs4 import BeautifulSoup
from couchpotato import fireEvent
from couchpotato.core.helpers.rss import RSS
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
@@ -96,8 +97,14 @@ class Bluray(Automation, RSS):
movie = self.search(name, year)
if movie:
if movie.get('imdb') in movie_ids:
continue
is_movie = fireEvent('movie.is_movie', identifier = movie.get('imdb'), single = True)
if not is_movie:
continue
movie_ids.append(movie.get('imdb'))
movie_list['list'].append( movie )
if len(movie_list['list']) >= max_items:

View File

@@ -100,20 +100,28 @@ class IMDBAutomation(IMDBBase):
enabled_option = 'automation_providers_enabled'
chart_urls = {
'theater': 'http://www.imdb.com/movies-in-theaters/',
'top250': 'http://www.imdb.com/chart/top',
'boxoffice': 'http://www.imdb.com/chart/',
}
chart_names = {
'theater': 'IMDB - Movies in Theaters',
'top250': 'IMDB - Top 250 Movies',
'boxoffice': 'IMDB - Box Office',
}
chart_order = {
'theater': 2,
'top250': 4,
'boxoffice': 3,
charts = {
'theater': {
'order': 1,
'name': 'IMDB - Movies in Theaters',
'url': 'http://www.imdb.com/movies-in-theaters/',
},
'boxoffice': {
'order': 2,
'name': 'IMDB - Box Office',
'url': 'http://www.imdb.com/chart/',
},
'rentals': {
'order': 3,
'name': 'IMDB - Top DVD rentals',
'url': 'http://m.imdb.com/boxoffice_json',
'type': 'json',
},
'top250': {
'order': 4,
'name': 'IMDB - Top 250 Movies',
'url': 'http://www.imdb.com/chart/top',
},
}
first_table = ['boxoffice']
@@ -122,23 +130,30 @@ class IMDBAutomation(IMDBBase):
movies = []
for url in self.chart_urls:
if self.conf('automation_charts_%s' % url):
data = self.getHTMLData(self.chart_urls[url])
for name in self.charts:
chart = self.charts[name]
url = chart.get('url')
if self.conf('automation_charts_%s' % name):
data = self.getHTMLData(url)
if data:
html = BeautifulSoup(data)
try:
result_div = html.find('div', attrs = {'id': 'main'})
html = BeautifulSoup(data)
try:
if url in self.first_table:
table = result_div.find('table')
result_div = table if table else result_div
except:
pass
if chart.get('type', 'html') == 'html':
result_div = html.find('div', attrs = {'id': 'main'})
imdb_ids = getImdb(str(result_div), multiple = True)
try:
if url in self.first_table:
table = result_div.find('table')
result_div = table if table else result_div
except:
pass
imdb_ids = getImdb(str(result_div), multiple = True)
else:
imdb_ids = getImdb(str(data), multiple = True)
for imdb_id in imdb_ids:
info = self.getInfo(imdb_id)
@@ -153,42 +168,56 @@ class IMDBAutomation(IMDBBase):
return movies
def getChartList(self):
# Nearly identical to 'getIMDBids', but we don't care about minimalMovie and return all movie data (not just id)
movie_lists = []
max_items = int(self.conf('max_items', section='charts', default=5))
max_items = int(self.conf('max_items', section = 'charts', default=5))
for url in self.chart_urls:
if self.conf('chart_display_%s' % url):
movie_list = {'name': self.chart_names[url], 'url': self.chart_urls[url], 'order': self.chart_order[url], 'list': []}
data = self.getHTMLData(self.chart_urls[url])
for name in self.charts:
chart = self.charts[name].copy()
url = chart.get('url')
if self.conf('chart_display_%s' % name):
chart['list'] = []
data = self.getHTMLData(url)
if data:
html = BeautifulSoup(data)
try:
result_div = html.find('div', attrs = {'id': 'main'})
try:
if url in self.first_table:
table = result_div.find('table')
result_div = table if table else result_div
except:
pass
if chart.get('type', 'html') == 'html':
result_div = html.find('div', attrs = {'id': 'main'})
imdb_ids = getImdb(str(result_div), multiple = True)
try:
if url in self.first_table:
table = result_div.find('table')
result_div = table if table else result_div
except:
pass
imdb_ids = getImdb(str(result_div), multiple = True)
else:
imdb_ids = getImdb(str(data), multiple = True)
for imdb_id in imdb_ids[0:max_items]:
is_movie = fireEvent('movie.is_movie', identifier = imdb_id, single = True)
if not is_movie:
continue
info = self.getInfo(imdb_id)
movie_list['list'].append(info)
chart['list'].append(info)
if self.shuttingDown():
break
except:
log.error('Failed loading IMDB chart results from %s: %s', (url, traceback.format_exc()))
if movie_list['list']:
movie_lists.append(movie_list)
if chart['list']:
movie_lists.append(chart)
return movie_lists
@@ -240,12 +269,19 @@ config = [{
'description': 'New Movies <a href="http://www.imdb.com/movies-in-theaters/">In-Theaters</a> chart',
'default': True,
},
{
'name': 'automation_charts_rentals',
'type': 'bool',
'label': 'DVD Rentals',
'description': 'Top DVD <a href="http://www.imdb.com/boxoffice/rentals/">rentals</a> chart',
'default': True,
},
{
'name': 'automation_charts_top250',
'type': 'bool',
'label': 'TOP 250',
'description': 'IMDB <a href="http://www.imdb.com/chart/top/">TOP 250</a> chart',
'default': True,
'default': False,
},
{
'name': 'automation_charts_boxoffice',
@@ -282,6 +318,13 @@ config = [{
'description': 'IMDB <a href="http://www.imdb.com/chart/top/">TOP 250</a> chart',
'default': False,
},
{
'name': 'chart_display_rentals',
'type': 'bool',
'label': 'DVD Rentals',
'description': 'Top DVD <a href="http://www.imdb.com/boxoffice/rentals/">rentals</a> chart',
'default': True,
},
{
'name': 'chart_display_boxoffice',
'type': 'bool',

View File

@@ -11,11 +11,16 @@ autoload = 'Newznab'
class Newznab(MovieProvider, Base):
def buildUrl(self, media, api_key):
def buildUrl(self, media, host):
query = tryUrlencode({
't': 'movie',
'imdbid': getIdentifier(media).replace('tt', ''),
'apikey': api_key,
'apikey': host['api_key'],
'extended': 1
})
if len(host.get('custom_tag', '')) > 0:
query = '%s&%s' % (query, host.get('custom_tag'))
return query

View File

@@ -10,10 +10,14 @@ autoload = 'BiTHDTV'
class BiTHDTV(MovieProvider, Base):
cat_ids = [
([2], ['bd50']),
]
cat_backup_id = 7 # Movies
def buildUrl(self, media):
def buildUrl(self, media, quality):
query = tryUrlencode({
'search': fireEvent('library.query', media, single = True),
'cat': 7 # Movie cat
'cat': self.getCatId(quality)[0]
})
return query

View File

@@ -1,6 +1,5 @@
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.logger import CPLog
from couchpotato.core.event import fireEvent
from couchpotato.core.media._base.providers.torrent.bitsoup import Base
from couchpotato.core.media.movie.providers.base import MovieProvider
@@ -18,12 +17,9 @@ class Bitsoup(MovieProvider, Base):
]
cat_backup_id = 0
def buildUrl(self, media, quality):
def buildUrl(self, title, media, quality):
query = tryUrlencode({
'search': '"%s" %s' % (
fireEvent('library.query', media, include_year = False, single = True),
media['info']['year']
),
'search': '"%s" %s' % (title, media['info']['year']),
'cat': self.getCatId(quality)[0],
})
return query

View File

@@ -18,6 +18,6 @@ class IPTorrents(MovieProvider, Base):
]
def buildUrl(self, title, media, quality):
query = '%s %s' % (title.replace(':', ''), media['info']['year'])
query = '"%s" %s' % (title.replace(':', ''), media['info']['year'])
return self._buildUrl(query, quality)

View File

@@ -17,13 +17,13 @@ class SceneAccess(MovieProvider, Base):
([8], ['dvdr']),
]
def buildUrl(self, media, quality):
def buildUrl(self, title, media, quality):
cat_id = self.getCatId(quality)[0]
url = self.urls['search'] % (cat_id, cat_id)
arguments = tryUrlencode({
'search': fireEvent('library.query', media, single = True),
'method': 3,
'search': '"%s" %s' % (title, media['info']['year']),
'method': 2,
})
query = "%s&%s" % (url, arguments)

View File

@@ -1,5 +1,4 @@
from couchpotato.core.logger import CPLog
from couchpotato.core.event import fireEvent
from couchpotato.core.media._base.providers.torrent.torrentday import Base
from couchpotato.core.media.movie.providers.base import MovieProvider
@@ -16,6 +15,3 @@ class TorrentDay(MovieProvider, Base):
([3], ['dvdr']),
([5], ['bd50']),
]
def buildUrl(self, media):
return fireEvent('library.query', media, single = True)

View File

@@ -1,8 +1,9 @@
from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup
from couchpotato.core.media._base.providers.userscript.base import UserscriptBase
autoload = 'Filmstarts'
class Filmstarts(UserscriptBase):
includes = ['*://www.filmstarts.de/kritiken/*']

View File

@@ -54,7 +54,10 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
})
if self.conf('run_on_launch'):
addEvent('app.load', self.searchAll)
def on_load():
time.sleep(.1)
self.searchAll()
addEvent('app.load', on_load, priority = 1000)
def searchAllView(self, **kwargs):
@@ -126,6 +129,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
release_dates = fireEvent('movie.update_release_dates', movie['_id'], merge = True)
found_releases = []
previous_releases = movie.get('releases', [])
too_early_to_search = []
default_title = getTitle(movie)
@@ -139,16 +143,15 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
db = get_db()
profile = db.get('id', movie['profile_id'])
quality_order = fireEvent('quality.order', single = True)
ret = False
index = 0
for q_identifier in profile.get('qualities'):
quality_custom = {
'index': index,
'quality': q_identifier,
'finish': profile['finish'][index],
'wait_for': profile['wait_for'][index],
'wait_for': tryInt(profile['wait_for'][index]),
'3d': profile['3d'][index] if profile.get('3d') else False
}
@@ -162,43 +165,51 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
# See if better quality is available
for release in movie.get('releases', []):
if quality_order.index(release['quality']) <= quality_order.index(q_identifier) and release['status'] not in ['available', 'ignored', 'failed']:
has_better_quality += 1
if release['status'] not in ['available', 'ignored', 'failed']:
is_higher = fireEvent('quality.ishigher', \
{'identifier': q_identifier, 'is_3d': quality_custom.get('3d', 0)}, \
{'identifier': release['quality'], 'is_3d': release.get('is_3d', 0)}, \
profile, single = True)
if is_higher != 'higher':
has_better_quality += 1
# Don't search for quality lower then already available.
if has_better_quality is 0:
quality = fireEvent('quality.single', identifier = q_identifier, single = True)
log.info('Search for %s in %s', (default_title, quality['label']))
# Extend quality with profile customs
quality['custom'] = quality_custom
results = fireEvent('searcher.search', search_protocols, movie, quality, single = True) or []
if len(results) == 0:
log.debug('Nothing found for %s in %s', (default_title, quality['label']))
# Check if movie isn't deleted while searching
if not fireEvent('media.get', movie.get('_id'), single = True):
break
# Add them to this movie releases list
found_releases += fireEvent('release.create_from_search', results, movie, quality, single = True)
# Try find a valid result and download it
if fireEvent('release.try_download_result', results, movie, quality_custom, manual, single = True):
ret = True
# Remove releases that aren't found anymore
for release in movie.get('releases', []):
if release.get('status') == 'available' and release.get('identifier') not in found_releases:
fireEvent('release.delete', release.get('_id'), single = True)
else:
if has_better_quality > 0:
log.info('Better quality (%s) already available or snatched for %s', (q_identifier, default_title))
fireEvent('media.restatus', movie['_id'])
break
quality = fireEvent('quality.single', identifier = q_identifier, single = True)
log.info('Search for %s in %s', (default_title, quality['label']))
# Extend quality with profile customs
quality['custom'] = quality_custom
results = fireEvent('searcher.search', search_protocols, movie, quality, single = True) or []
if len(results) == 0:
log.debug('Nothing found for %s in %s', (default_title, quality['label']))
# Check if movie isn't deleted while searching
if not fireEvent('media.get', movie.get('_id'), single = True):
break
# Add them to this movie releases list
found_releases += fireEvent('release.create_from_search', results, movie, quality, single = True)
# Try find a valid result and download it
if fireEvent('release.try_download_result', results, movie, quality_custom, manual, single = True):
ret = True
# Remove releases that aren't found anymore
temp_previous_releases = []
for release in previous_releases:
if release.get('status') == 'available' and release.get('identifier') not in found_releases:
fireEvent('release.delete', release.get('_id'), single = True)
else:
temp_previous_releases.append(release)
previous_releases = temp_previous_releases
del temp_previous_releases
# Break if CP wants to shut down
if self.shuttingDown() or ret:
break
@@ -230,8 +241,9 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
preferred_quality = quality if quality else fireEvent('quality.single', identifier = quality['identifier'], single = True)
# Contains lower quality string
if fireEvent('searcher.contains_other_quality', nzb, movie_year = media['info']['year'], preferred_quality = preferred_quality, single = True):
log.info2('Wrong: %s, looking for %s', (nzb['name'], quality['label']))
contains_other = fireEvent('searcher.contains_other_quality', nzb, movie_year = media['info']['year'], preferred_quality = preferred_quality, single = True)
if contains_other != False:
log.info2('Wrong: %s, looking for %s, found %s', (nzb['name'], quality['label'], [x for x in contains_other] if contains_other else 'no quality'))
return False
# Contains lower quality string
@@ -288,7 +300,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
now_year = date.today().year
now_month = date.today().month
if (year is None or year < now_year - 1) and (not dates or (dates.get('theater', 0) == 0 and dates.get('dvd', 0) == 0)):
if (year is None or year < now_year - 1 or (year <= now_year - 1 and now_month > 4)) and (not dates or (dates.get('theater', 0) == 0 and dates.get('dvd', 0) == 0)):
return True
else:

View File

@@ -1,9 +1,10 @@
import os
import re
import traceback
from couchpotato.api import addApiView
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.helpers.variable import tryInt, splitString
from couchpotato.core.logger import CPLog
from couchpotato.core.plugins.base import Plugin
from couchpotato.environment import Env
@@ -22,7 +23,11 @@ class Logging(Plugin):
},
'return': {'type': 'object', 'example': """{
'success': True,
'log': string, //Log file
'log': [{
'time': '03-12 09:12:59',
'type': 'INFO',
'message': 'Log message'
}, ..], //Log file
'total': int, //Total log files available
}"""}
})
@@ -34,7 +39,11 @@ class Logging(Plugin):
},
'return': {'type': 'object', 'example': """{
'success': True,
'log': string, //Log file
'log': [{
'time': '03-12 09:12:59',
'type': 'INFO',
'message': 'Log message'
}, ..]
}"""}
})
addApiView('logging.clear', self.clear, docs = {
@@ -71,16 +80,18 @@ class Logging(Plugin):
if current_path:
f = open(current_path, 'r')
log_content = f.read()
logs = self.toList(log_content)
return {
'success': True,
'log': toUnicode(log_content),
'log': logs,
'total': total,
}
def partial(self, type = 'all', lines = 30, **kwargs):
def partial(self, type = 'all', lines = 30, offset = 0, **kwargs):
total_lines = tryInt(lines)
offset = tryInt(offset)
log_lines = []
@@ -93,28 +104,57 @@ class Logging(Plugin):
break
f = open(path, 'r')
reversed_lines = toUnicode(f.read()).split('[0m\n')
reversed_lines.reverse()
log_content = toUnicode(f.read())
raw_lines = self.toList(log_content)
raw_lines.reverse()
brk = False
for line in reversed_lines:
for line in raw_lines:
if type == 'all' or '%s ' % type.upper() in line:
if type == 'all' or line.get('type') == type.upper():
log_lines.append(line)
if len(log_lines) >= total_lines:
if len(log_lines) >= (total_lines + offset):
brk = True
break
if brk:
break
log_lines = log_lines[offset:]
log_lines.reverse()
return {
'success': True,
'log': '[0m\n'.join(log_lines),
'log': log_lines,
}
def toList(self, log_content = ''):
logs_raw = toUnicode(log_content).split('[0m\n')
logs = []
for log in logs_raw:
split = splitString(log, '\x1b')
if split:
try:
date, time, log_type = splitString(split[0], ' ')
timestamp = '%s %s' % (date, time)
except:
timestamp = 'UNKNOWN'
log_type = 'UNKNOWN'
message = ''.join(split[1]) if len(split) > 1 else split[0]
message = re.sub('\[\d+m\[', '[', message)
logs.append({
'time': timestamp,
'type': log_type,
'message': message
})
return logs
def clear(self, **kwargs):
for x in range(0, 50):

View File

@@ -16,10 +16,14 @@
display: inline-block;
padding: 5px 10px;
margin: 0;
}
.page.log .nav li.select,
.page.log .nav li.clear {
cursor: pointer;
}
.page.log .nav li:hover:not(.active) {
.page.log .nav li:hover:not(.active, .filter) {
background: rgba(255, 255, 255, 0.1);
}
@@ -51,13 +55,12 @@
line-height: 150%;
font-size: 11px;
font-family: Lucida Console, Monaco, Nimbus Mono L, monospace, serif;
color: #FFF;
}
.page.log .container .error {
color: #FFA4A4;
white-space: pre-wrap;
.page.log .container select {
vertical-align: top;
}
.page.log .container .debug { color: lightgrey; }
.page.log .container .time {
clear: both;
@@ -68,10 +71,25 @@
position: relative;
}
.page.log .container .time:last-child { display: none; }
.page.log .container .time span {
float: right;
width: 86%;
.page.log [data-filter=INFO] .error,
.page.log [data-filter=INFO] .debug,
.page.log [data-filter=ERROR] .debug,
.page.log [data-filter=ERROR] .info,
.page.log [data-filter=DEBUG] .info,
.page.log [data-filter=DEBUG] .error {
display: none;
}
.page.log .container .type {
margin-left: 10px;
}
.page.log .container .message {
float: right;
width: 86%;
white-space: pre-wrap;
}
.page.log .container .error { color: #FFA4A4; }
.page.log .container .debug { opacity: .4; }

View File

@@ -2,6 +2,7 @@ Page.Log = new Class({
Extends: PageBase,
order: 60,
name: 'log',
title: 'Show recent logs.',
has_tab: false,
@@ -26,25 +27,46 @@ Page.Log = new Class({
'nr': nr
},
'onComplete': function(json){
self.log.set('html', self.addColors(json.log));
self.log.set('text', '');
self.log.adopt(self.createLogElements(json.log));
self.log.removeClass('loading');
new Fx.Scroll(window, {'duration': 0}).toBottom();
var nav = new Element('ul.nav', {
'events': {
'click:relay(li.select)': function(e, el){
self.getLogs(parseInt(el.get('text'))-1);
}
}
});
var nav = new Element('ul.nav').inject(self.log, 'top');
// Type selection
new Element('li.filter').grab(
new Element('select', {
'events': {
'change': function(){
var type_filter = this.getSelected()[0].get('value');
self.log.set('data-filter', type_filter);
self.scrollToBottom();
}
}
}).adopt(
new Element('option', {'value': 'ALL', 'text': 'Show all logs'}),
new Element('option', {'value': 'INFO', 'text': 'Show only INFO'}),
new Element('option', {'value': 'DEBUG', 'text': 'Show only DEBUG'}),
new Element('option', {'value': 'ERROR', 'text': 'Show only ERROR'})
)
).inject(nav);
// Selections
for (var i = 0; i <= json.total; i++) {
new Element('li', {
'text': i+1,
'class': nr == i ? 'active': '',
'events': {
'click': function(e){
self.getLogs(e.target.get('text')-1);
}
}
'class': 'select ' + (nr == i ? 'active': '')
}).inject(nav);
}
new Element('li', {
// Clear button
new Element('li.clear', {
'text': 'clear',
'events': {
'click': function(){
@@ -56,26 +78,40 @@ Page.Log = new Class({
}
}
}).inject(nav)
}).inject(nav);
// Add to page
nav.inject(self.log, 'top');
self.scrollToBottom();
}
});
},
addColors: function(text){
createLogElements: function(logs){
text = text
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/\u001b\[31m/gi, '</span><span class="error">')
.replace(/\u001b\[36m/gi, '</span><span class="debug">')
.replace(/\u001b\[33m/gi, '</span><span class="debug">')
.replace(/\u001b\[0m\n/gi, '</div><div class="time">')
.replace(/\u001b\[0m/gi, '</span><span>');
var elements = [];
return '<div class="time">' + text + '</div>';
logs.each(function(log){
elements.include(new Element('div', {
'class': 'time ' + log.type.toLowerCase(),
'text': log.time
}).adopt(
new Element('span.type', {
'text': log.type
}),
new Element('span.message', {
'text': log.message
})
))
});
return elements;
},
scrollToBottom: function(){
new Fx.Scroll(window, {'duration': 0}).toBottom();
}
});

View File

@@ -34,7 +34,7 @@ class ProfilePlugin(Plugin):
})
addEvent('app.initialize', self.fill, priority = 90)
addEvent('app.load', self.forceDefaults)
addEvent('app.load', self.forceDefaults, priority = 110)
def forceDefaults(self):
@@ -87,7 +87,7 @@ class ProfilePlugin(Plugin):
order = 0
for type in kwargs.get('types', []):
profile['qualities'].append(type.get('quality'))
profile['wait_for'].append(tryInt(type.get('wait_for')))
profile['wait_for'].append(tryInt(kwargs.get('wait_for', 0)))
profile['finish'].append((tryInt(type.get('finish')) == 1) if order > 0 else True)
profile['3d'].append(tryInt(type.get('3d')))
order += 1

View File

@@ -41,7 +41,7 @@ var Profile = new Class({
new Element('span', {'text':'Wait'}),
new Element('input.inlay.xsmall', {
'type':'text',
'value': data.types && data.types.length > 0 ? data.types[0].wait_for : 0
'value': data.wait_for && data.wait_for.length > 0 ? data.wait_for[0] : 0
}),
new Element('span', {'text':'day(s) for a better quality.'})
),
@@ -63,8 +63,7 @@ var Profile = new Class({
data.types.include({
'quality': quality,
'finish': data.finish[nr] || false,
'3d': data['3d'] ? data['3d'][nr] || false : false,
'wait_for': data.wait_for[nr] || 0
'3d': data['3d'] ? data['3d'][nr] || false : false
})
});
}
@@ -126,8 +125,7 @@ var Profile = new Class({
data.types.include({
'quality': type.getElement('select').get('value'),
'finish': +type.getElement('input.finish[type=checkbox]').checked,
'3d': +type.getElement('input.3d[type=checkbox]').checked,
'wait_for': 0
'3d': +type.getElement('input.3d[type=checkbox]').checked
});
});
@@ -340,8 +338,7 @@ Profile.Type = new Class({
return {
'quality': self.qualities.get('value'),
'finish': +self.finish.checked,
'3d': +self['3d'].checked,
'wait_for': 0
'3d': +self['3d'].checked
}
},

View File

@@ -21,11 +21,11 @@ class QualityPlugin(Plugin):
}
qualities = [
{'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (15000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':[], 'tags': ['bdmv', 'certificate', ('complete', 'bluray')]},
{'identifier': 'bd50', 'hd': True, 'allow_3d': True, 'size': (20000, 60000), 'label': 'BR-Disk', 'alternative': ['bd25'], 'allow': ['1080p'], 'ext':['iso', 'img'], 'tags': ['bdmv', 'certificate', ('complete', 'bluray'), 'avc', 'mvc']},
{'identifier': '1080p', 'hd': True, 'allow_3d': True, 'size': (4000, 20000), 'label': '1080p', 'width': 1920, 'height': 1080, 'alternative': [], 'allow': [], 'ext':['mkv', 'm2ts'], 'tags': ['m2ts', 'x264', 'h264']},
{'identifier': '720p', 'hd': True, 'allow_3d': True, 'size': (3000, 10000), 'label': '720p', 'width': 1280, 'height': 720, 'alternative': [], 'allow': [], 'ext':['mkv', 'ts'], 'tags': ['x264', 'h264']},
{'identifier': 'brrip', 'hd': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]},
{'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd'], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r')]},
{'identifier': 'brrip', 'hd': True, 'allow_3d': True, 'size': (700, 7000), 'label': 'BR-Rip', 'alternative': ['bdrip'], 'allow': ['720p', '1080p'], 'ext':[], 'tags': ['hdtv', 'hdrip', 'webdl', ('web', 'dl')]},
{'identifier': 'dvdr', 'size': (3000, 10000), 'label': 'DVD-R', 'alternative': ['br2dvd'], 'allow': [], 'ext':['iso', 'img', 'vob'], 'tags': ['pal', 'ntsc', 'video_ts', 'audio_ts', ('dvd', 'r'), 'dvd9']},
{'identifier': 'dvdrip', 'size': (600, 2400), 'label': 'DVD-Rip', 'width': 720, 'alternative': [], 'allow': [], 'ext':[], 'tags': [('dvd', 'rip'), ('dvd', 'xvid'), ('dvd', 'divx')]},
{'identifier': 'scr', 'size': (600, 1600), 'label': 'Screener', 'alternative': ['screener', 'dvdscr', 'ppvrip', 'dvdscreener', 'hdscr'], 'allow': ['dvdr', 'dvdrip', '720p', '1080p'], 'ext':[], 'tags': ['webrip', ('web', 'rip')]},
{'identifier': 'r5', 'size': (600, 1000), 'label': 'R5', 'alternative': ['r6'], 'allow': ['dvdr'], 'ext':[]},
@@ -35,9 +35,9 @@ class QualityPlugin(Plugin):
]
pre_releases = ['cam', 'ts', 'tc', 'r5', 'scr']
threed_tags = {
'hsbs': [('half', 'sbs')],
'fsbs': [('full', 'sbs')],
'3d': [],
'sbs': [('half', 'sbs'), 'hsbs', ('full', 'sbs'), 'fsbs'],
'ou': [('half', 'ou'), 'hou', ('full', 'ou'), 'fou'],
'3d': ['2d3d', '3d2d'],
}
cached_qualities = None
@@ -49,6 +49,8 @@ class QualityPlugin(Plugin):
addEvent('quality.guess', self.guess)
addEvent('quality.pre_releases', self.preReleases)
addEvent('quality.order', self.getOrder)
addEvent('quality.ishigher', self.isHigher)
addEvent('quality.isfinish', self.isFinish)
addApiView('quality.size.save', self.saveSize)
addApiView('quality.list', self.allView, docs = {
@@ -177,7 +179,7 @@ class QualityPlugin(Plugin):
return False
def guess(self, files, extra = None):
def guess(self, files, extra = None, size = None):
if not extra: extra = {}
# Create hash for cache
@@ -205,15 +207,20 @@ class QualityPlugin(Plugin):
self.calcScore(score, quality, contains_score, threedscore)
# Evaluate score based on size
for quality in qualities:
size_score = self.guessSizeScore(quality, size = size)
self.calcScore(score, quality, size_score, penalty = False)
# Try again with loose testing
for quality in qualities:
loose_score = self.guessLooseScore(quality, extra = extra)
self.calcScore(score, quality, loose_score)
self.calcScore(score, quality, loose_score, penalty = False)
# Return nothing if all scores are 0
# Return nothing if all scores are <= 0
has_non_zero = 0
for s in score:
if score[s] > 0:
if score[s]['score'] > 0:
has_non_zero += 1
if not has_non_zero:
@@ -281,7 +288,7 @@ class QualityPlugin(Plugin):
return 1, key
if list(set([key]) & set(words)):
log.debug('Found %s in %s', (tag, cur_file))
log.debug('Found %s in %s', (key, cur_file))
return 1, key
return 0, None
@@ -308,7 +315,20 @@ class QualityPlugin(Plugin):
return score
def calcScore(self, score, quality, add_score, threedscore = (0, None)):
def guessSizeScore(self, quality, size = None):
score = 0
if size:
if tryInt(quality['size_min']) <= tryInt(size) <= tryInt(quality['size_max']):
log.info2('Found %s via release size: %s MB < %s MB < %s MB', (quality['identifier'], quality['size_min'], size, quality['size_max']))
score += 5
return score
def calcScore(self, score, quality, add_score, threedscore = (0, None), penalty = True):
score[quality['identifier']]['score'] += add_score
@@ -325,30 +345,79 @@ class QualityPlugin(Plugin):
for q in self.qualities:
self.cached_order[q.get('identifier')] = self.qualities.index(q)
if add_score != 0:
if penalty and add_score != 0:
for allow in quality.get('allow', []):
score[allow]['score'] -= 40 if self.cached_order[allow] < self.cached_order[quality['identifier']] else 5
# Give panelty for all lower qualities
for q in self.qualities[self.order.index(quality.get('identifier'))+1:]:
score[q.get('identifier')]['score'] -= 1
def isFinish(self, quality, profile):
if not isinstance(profile, dict) or not profile.get('qualities'):
return False
try:
quality_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(quality.get('is_3d', 0))][0]
return profile['finish'][quality_order]
except:
return False
def isHigher(self, quality, compare_with, profile = None):
if not isinstance(profile, dict) or not profile.get('qualities'):
profile = {'qualities': self.order}
# Try to find quality in profile, if not found: a quality we do not want is lower than anything else
try:
quality_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == quality['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(quality.get('is_3d', 0))][0]
except:
log.debug('Quality %s not found in profile identifiers %s', (quality['identifier'] + (' 3D' if quality.get('is_3d', 0) else ''), \
[identifier + ('3D' if (profile['3d'][i] if profile.get('3d') else 0) else '') for i, identifier in enumerate(profile['qualities'])]))
return 'lower'
# Try to find compare quality in profile, if not found: anything is higher than a not wanted quality
try:
compare_order = [i for i, identifier in enumerate(profile['qualities']) if identifier == compare_with['identifier'] and bool(profile['3d'][i] if profile.get('3d') else 0) == bool(compare_with.get('is_3d', 0))][0]
except:
log.debug('Compare quality %s not found in profile identifiers %s', (compare_with['identifier'] + (' 3D' if compare_with.get('is_3d', 0) else ''), \
[identifier + (' 3D' if (profile['3d'][i] if profile.get('3d') else 0) else '') for i, identifier in enumerate(profile['qualities'])]))
return 'higher'
# Note to self: a lower number means higher quality
if quality_order > compare_order:
return 'lower'
elif quality_order == compare_order:
return 'equal'
else:
return 'higher'
def doTest(self):
tests = {
'Movie Name (1999)-DVD-Rip.avi': 'dvdrip',
'Movie Name 1999 720p Bluray.mkv': '720p',
'Movie Name 1999 BR-Rip 720p.avi': 'brrip',
'Movie Name 1999 720p Web Rip.avi': 'scr',
'Movie Name 1999 Web DL.avi': 'brrip',
'Movie.Name.1999.1080p.WEBRip.H264-Group': 'scr',
'Movie.Name.1999.DVDRip-Group': 'dvdrip',
'Movie.Name.1999.DVD-Rip-Group': 'dvdrip',
'Movie.Name.1999.DVD-R-Group': 'dvdr',
'Movie.Name.Camelie.1999.720p.BluRay.x264-Group': '720p',
'Movie.Name.2008.German.DL.AC3.1080p.BluRay.x264-Group': '1080p',
'Movie.Name.2004.GERMAN.AC3D.DL.1080p.BluRay.x264-Group': '1080p',
'Movie Name (1999)-DVD-Rip.avi': {'size': 700, 'quality': 'dvdrip'},
'Movie Name 1999 720p Bluray.mkv': {'size': 4200, 'quality': '720p'},
'Movie Name 1999 BR-Rip 720p.avi': {'size': 1000, 'quality': 'brrip'},
'Movie Name 1999 720p Web Rip.avi': {'size': 1200, 'quality': 'scr'},
'Movie Name 1999 Web DL.avi': {'size': 800, 'quality': 'brrip'},
'Movie.Name.1999.1080p.WEBRip.H264-Group': {'size': 1500, 'quality': 'scr'},
'Movie.Name.1999.DVDRip-Group': {'size': 750, 'quality': 'dvdrip'},
'Movie.Name.1999.DVD-Rip-Group': {'size': 700, 'quality': 'dvdrip'},
'Movie.Name.1999.DVD-R-Group': {'size': 4500, 'quality': 'dvdr'},
'Movie.Name.Camelie.1999.720p.BluRay.x264-Group': {'size': 5500, 'quality': '720p'},
'Movie.Name.2008.German.DL.AC3.1080p.BluRay.x264-Group': {'size': 8500, 'extra': {'resolution_width': 1920, 'resolution_height': 1080} , 'quality': '1080p'},
'Movie.Name.2004.GERMAN.AC3D.DL.1080p.BluRay.x264-Group': {'size': 8000, 'quality': '1080p'},
'Movie.Name.2013.BR-Disk-Group.iso': {'size': 48000, 'quality': 'bd50'},
'Movie.Name.2013.2D+3D.BR-Disk-Group.iso': {'size': 52000, 'quality': 'bd50', 'is_3d': True},
'Movie.Rising.Name.Girl.2011.NTSC.DVD9-GroupDVD': {'size': 7200, 'quality': 'dvdr'},
'Movie Name (2013) 2D + 3D': {'size': 49000, 'quality': 'bd50', 'is_3d': True},
'Movie Monuments 2013 BrRip 1080p': {'size': 1800, 'quality': 'brrip'},
'Movie Monuments 2013 BrRip 720p': {'size': 1300, 'quality': 'brrip'},
}
correct = 0
for name in tests:
success = self.guess([name]).get('identifier') == tests[name]
test_quality = self.guess(files = [name], extra = tests[name].get('extra', None), size = tests[name].get('size', None)) or {}
success = test_quality.get('identifier') == tests[name]['quality'] and test_quality.get('is_3d') == tests[name].get('is_3d', False)
if not success:
log.error('%s failed check, thinks it\'s %s', (name, self.guess([name]).get('identifier')))

View File

@@ -3,6 +3,7 @@ import os
import time
import traceback
from CodernityDB.database import RecordDeleted
from couchpotato import md5, get_db
from couchpotato.api import addApiView
from couchpotato.core.event import fireEvent, addEvent
@@ -58,7 +59,7 @@ class Release(Plugin):
# Clean releases that didn't have activity in the last week
addEvent('app.load', self.cleanDone)
fireEvent('schedule.interval', 'movie.clean_releases', self.cleanDone, hours = 4)
fireEvent('schedule.interval', 'movie.clean_releases', self.cleanDone, hours = 12)
def cleanDone(self):
log.debug('Removing releases from dashboard')
@@ -68,6 +69,27 @@ class Release(Plugin):
db = get_db()
# Get (and remove) parentless releases
releases = db.all('release', with_doc = True)
media_exist = []
for release in releases:
if release.get('key') in media_exist:
continue
try:
db.get('id', release.get('key'))
media_exist.append(release.get('key'))
except RecordDeleted:
db.delete(release['doc'])
log.debug('Deleted orphaned release: %s', release['doc'])
except:
log.debug('Failed cleaning up orphaned releases: %s', traceback.format_exc())
del media_exist
# Reindex statuses
db.reindex_index('media_status')
# get movies last_edit more than a week ago
medias = fireEvent('media.with_status', 'done', single = True)
@@ -107,6 +129,7 @@ class Release(Plugin):
'media_id': media['_id'],
'identifier': release_identifier,
'quality': group['meta_data']['quality'].get('identifier'),
'is_3d': group['meta_data']['quality'].get('is_3d', 0),
'last_edit': int(time.time()),
'status': 'done'
}
@@ -315,10 +338,12 @@ class Release(Plugin):
def tryDownloadResult(self, results, media, quality_custom, manual = False):
wait_for = False
let_through = False
filtered_results = []
# If a single release comes through the "wait for", let through all
for rel in results:
if not quality_custom.get('finish', False) and quality_custom.get('wait_for', 0) > 0 and rel.get('age') <= quality_custom.get('wait_for', 0):
log.info('Ignored, waiting %s days: %s', (quality_custom.get('wait_for'), rel['name']))
continue
if rel['status'] in ['ignored', 'failed']:
log.info('Ignored: %s', rel['name'])
@@ -328,13 +353,30 @@ class Release(Plugin):
log.info('Ignored, score to low: %s', rel['name'])
continue
rel['wait_for'] = False
if quality_custom.get('index') != 0 and quality_custom.get('wait_for', 0) > 0 and rel.get('age') <= quality_custom.get('wait_for', 0):
rel['wait_for'] = True
else:
let_through = True
filtered_results.append(rel)
# Loop through filtered results
for rel in filtered_results:
# Only wait if not a single release is old enough
if rel.get('wait_for') and not let_through:
log.info('Ignored, waiting %s days: %s', (quality_custom.get('wait_for') - rel.get('age'), rel['name']))
wait_for = True
continue
downloaded = fireEvent('release.download', data = rel, media = media, manual = manual, single = True)
if downloaded is True:
return True
elif downloaded != 'try_next':
break
return False
return wait_for
def createFromSearch(self, search_results, media, quality):
@@ -406,7 +448,7 @@ class Release(Plugin):
rel = db.get('id', release_id)
if rel and rel.get('status') != status:
release_name = rel.get('name')
release_name = rel['info'].get('name')
if rel.get('files'):
for file_type in rel.get('files', {}):
if file_type == 'movie':

View File

@@ -112,7 +112,7 @@ class Renamer(Plugin):
return
if not base_folder:
base_folder = self.conf('from')
base_folder = sp(self.conf('from'))
from_folder = sp(self.conf('from'))
to_folder = sp(self.conf('to'))
@@ -314,8 +314,14 @@ class Renamer(Plugin):
'cd': '',
'cd_nr': '',
'mpaa': media['info'].get('mpaa', ''),
'mpaa_only': media['info'].get('mpaa', ''),
'category': category_label,
'3d': '3D' if group['meta_data']['quality'].get('is_3d', 0) else '',
'3d_type': group['meta_data'].get('3d_type'),
}
if replacements['mpaa_only'] not in ('G', 'PG', 'PG-13', 'R', 'NC-17'):
replacements['mpaa_only'] = 'Not Rated'
for file_type in group['files']:
@@ -412,8 +418,12 @@ class Renamer(Plugin):
# Don't add language if multiple languages in 1 subtitle file
if len(sub_langs) == 1:
sub_name = sub_name.replace(replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext']))
rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)
sub_suffix = '%s.%s' % (sub_langs[0], replacements['ext'])
# Don't add language to subtitle file it it's already there
if not sub_name.endswith(sub_suffix):
sub_name = sub_name.replace(replacements['ext'], sub_suffix)
rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)
rename_files = mergeDicts(rename_files, rename_extras)
@@ -440,17 +450,15 @@ class Renamer(Plugin):
remove_leftovers = True
# Mark movie "done" once it's found the quality with the finish check
profile = None
try:
if media.get('status') == 'active' and media.get('profile_id'):
profile = db.get('id', media['profile_id'])
if group['meta_data']['quality']['identifier'] in profile.get('qualities', []):
nr = profile['qualities'].index(group['meta_data']['quality']['identifier'])
finish = profile['finish'][nr]
if finish:
mdia = db.get('id', media['_id'])
mdia['status'] = 'done'
mdia['last_edit'] = int(time.time())
db.update(mdia)
if fireEvent('quality.isfinish', group['meta_data']['quality'], profile, single = True):
mdia = db.get('id', media['_id'])
mdia['status'] = 'done'
mdia['last_edit'] = int(time.time())
db.update(mdia)
except Exception as e:
log.error('Failed marking movie finished: %s', (traceback.format_exc()))
@@ -461,18 +469,19 @@ class Renamer(Plugin):
# When a release already exists
if release.get('status') == 'done':
release_order = quality_order.index(release['quality'])
group_quality_order = quality_order.index(group['meta_data']['quality']['identifier'])
# This is where CP removes older, lesser quality releases or releases that are not wanted anymore
is_higher = fireEvent('quality.ishigher', \
group['meta_data']['quality'], {'identifier': release['quality'], 'is_3d': release.get('is_3d', 0)}, profile, single = True)
# This is where CP removes older, lesser quality releases
if release_order > group_quality_order:
log.info('Removing lesser quality %s for %s.', (media_title, release.get('quality')))
if is_higher == 'higher':
log.info('Removing lesser or not wanted quality %s for %s.', (media_title, release.get('quality')))
for file_type in release.get('files', {}):
for release_file in release['files'][file_type]:
remove_files.append(release_file)
remove_releases.append(release)
# Same quality, but still downloaded, so maybe repack/proper/unrated/directors cut etc
elif release_order == group_quality_order:
elif is_higher == 'equal':
log.info('Same quality release already exists for %s, with quality %s. Assuming repack.', (media_title, release.get('quality')))
for file_type in release.get('files', {}):
for release_file in release['files'][file_type]:
@@ -824,7 +833,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
def replaceDoubles(self, string):
replaces = [
('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '),
('\.+', '.'), ('_+', '_'), ('-+', '-'), ('\s+', ' '), (' \\\\', '\\\\'), (' /', '/'),
('(\s\.)+', '.'), ('(-\.)+', '.'), ('(\s-)+', '-'),
]
@@ -1056,6 +1065,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
release_download.update({
'imdb_id': getIdentifier(media),
'quality': rls['quality'],
'is_3d': rls['is_3d'],
'protocol': rls.get('info', {}).get('protocol') or rls.get('info', {}).get('type'),
'release_id': rls['_id'],
})
@@ -1195,6 +1205,8 @@ rename_options = {
'first': 'First letter (M)',
'quality': 'Quality (720p)',
'quality_type': '(HD) or (SD)',
'3d': '3D',
'3d_type': '3D Type (Full SBS)',
'video': 'Video (x264)',
'audio': 'Audio (DTS)',
'group': 'Releasegroup name',
@@ -1207,7 +1219,8 @@ rename_options = {
'imdb_id': 'IMDB id (tt0123456)',
'cd': 'CD number (cd1)',
'cd_nr': 'Just the cd nr. (1)',
'mpaa': 'MPAA Rating',
'mpaa': 'MPAA or other certification',
'mpaa_only': 'MPAA only certification (G|PG|PG-13|R|NC-17|Not Rated)',
'category': 'Category label',
},
}

View File

@@ -6,7 +6,7 @@ import traceback
from couchpotato import get_db
from couchpotato.core.event import fireEvent, addEvent
from couchpotato.core.helpers.encoding import toUnicode, simplifyString, sp
from couchpotato.core.helpers.encoding import toUnicode, simplifyString, sp, ss
from couchpotato.core.helpers.variable import getExt, getImdb, tryInt, \
splitString, getIdentifier
from couchpotato.core.logger import CPLog
@@ -40,6 +40,17 @@ class Scanner(Plugin):
'trailer': ['mov', 'mp4', 'flv']
}
threed_types = {
'Half SBS': [('half', 'sbs'), ('h', 'sbs'), 'hsbs'],
'Full SBS': [('full', 'sbs'), ('f', 'sbs'), 'fsbs'],
'SBS': ['sbs'],
'Half OU': [('half', 'ou'), ('h', 'ou'), 'hou'],
'Full OU': [('full', 'ou'), ('h', 'ou'), 'fou'],
'OU': ['ou'],
'Frame Packed': ['mvc', ('complete', 'bluray')],
'3D': ['3d']
}
file_types = {
'subtitle': ('subtitle', 'subtitle'),
'subtitle_extra': ('subtitle', 'subtitle_extra'),
@@ -64,6 +75,16 @@ class Scanner(Plugin):
'video': ['x264', 'H264', 'DivX', 'Xvid']
}
resolutions = {
'1080p': {'resolution_width': 1920, 'resolution_height': 1080, 'aspect': 1.78},
'1080i': {'resolution_width': 1920, 'resolution_height': 1080, 'aspect': 1.78},
'720p': {'resolution_width': 1280, 'resolution_height': 720, 'aspect': 1.78},
'720i': {'resolution_width': 1280, 'resolution_height': 720, 'aspect': 1.78},
'480p': {'resolution_width': 640, 'resolution_height': 480, 'aspect': 1.33},
'480i': {'resolution_width': 640, 'resolution_height': 480, 'aspect': 1.33},
'default': {'resolution_width': 0, 'resolution_height': 0, 'aspect': 1},
}
audio_codec_map = {
0x2000: 'AC3',
0x2001: 'DTS',
@@ -85,8 +106,8 @@ class Scanner(Plugin):
'HDTV': ['hdtv']
}
clean = '[ _\,\.\(\)\[\]\-]?(3d|hsbs|sbs|extended.cut|directors.cut|french|swedisch|danish|dutch|swesub|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip' \
'|hdtvrip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|cd[1-9]|\[.*\])([ _\,\.\(\)\[\]\-]|$)'
clean = '([ _\,\.\(\)\[\]\-]|^)(3d|hsbs|sbs|ou|extended.cut|directors.cut|french|fr|swedisch|sw|danish|dutch|nl|swesub|subs|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip' \
'|hdtvrip|webdl|web.dl|webrip|web.rip|internal|limited|multisubs|ntsc|ogg|ogm|pal|pdtv|proper|repack|rerip|retail|r3|r5|bd5|se|svcd|swedish|german|read.nfo|nfofix|unrated|ws|telesync|ts|telecine|tc|brrip|bdrip|video_ts|audio_ts|480p|480i|576p|576i|720p|720i|1080p|1080i|hrhd|hrhdtv|hddvd|bluray|x264|h264|xvid|xvidvd|xxx|www.www|hc|\[.*\])(?=[ _\,\.\(\)\[\]\-]|$)'
multipart_regex = [
'[ _\.-]+cd[ _\.-]*([0-9a-d]+)', #*cd1
'[ _\.-]+dvd[ _\.-]*([0-9a-d]+)', #*dvd1
@@ -164,7 +185,7 @@ class Scanner(Plugin):
identifiers = [identifier]
# Identifier with quality
quality = fireEvent('quality.guess', [file_path], single = True) if not is_dvd_file else {'identifier':'dvdr'}
quality = fireEvent('quality.guess', files = [file_path], size = self.getFileSize(file_path), single = True) if not is_dvd_file else {'identifier':'dvdr'}
if quality:
identifier_with_quality = '%s %s' % (identifier, quality.get('identifier', ''))
identifiers = [identifier_with_quality, identifier]
@@ -431,28 +452,39 @@ class Scanner(Plugin):
for cur_file in files:
if not self.filesizeBetween(cur_file, self.file_sizes['movie']): continue # Ignore smaller files
meta = self.getMeta(cur_file)
if not data.get('audio'): # Only get metadata from first media file
meta = self.getMeta(cur_file)
try:
data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video']))
data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio']))
data['resolution_width'] = meta.get('resolution_width', 720)
data['resolution_height'] = meta.get('resolution_height', 480)
data['audio_channels'] = meta.get('audio_channels', 2.0)
data['aspect'] = round(float(meta.get('resolution_width', 720)) / meta.get('resolution_height', 480), 2)
except:
log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc()))
pass
try:
data['video'] = meta.get('video', self.getCodec(cur_file, self.codecs['video']))
data['audio'] = meta.get('audio', self.getCodec(cur_file, self.codecs['audio']))
data['audio_channels'] = meta.get('audio_channels', 2.0)
if meta.get('resolution_width'):
data['resolution_width'] = meta.get('resolution_width')
data['resolution_height'] = meta.get('resolution_height')
data['aspect'] = round(float(meta.get('resolution_width')) / meta.get('resolution_height', 1), 2)
else:
data.update(self.getResolution(cur_file))
except:
log.debug('Error parsing metadata: %s %s', (cur_file, traceback.format_exc()))
pass
if data.get('audio'): break
data['size'] = data.get('size', 0) + self.getFileSize(cur_file)
# Use the quality guess first, if that failes use the quality we wanted to download
data['quality'] = None
quality = fireEvent('quality.guess', size = data['size'], files = files, extra = data, single = True)
# Use the quality that we snatched but check if it matches our guess
if release_download and release_download.get('quality'):
data['quality'] = fireEvent('quality.single', release_download.get('quality'), single = True)
data['quality']['is_3d'] = release_download.get('is_3d', 0)
if data['quality']['identifier'] != quality['identifier']:
log.info('Different quality snatched than detected for %s: %s vs. %s. Assuming snatched quality is correct.', (files[0], data['quality']['identifier'], quality['identifier']))
if data['quality']['is_3d'] != quality['is_3d']:
log.info('Different 3d snatched than detected for %s: %s vs. %s. Assuming snatched 3d is correct.', (files[0], data['quality']['is_3d'], quality['is_3d']))
if not data['quality']:
data['quality'] = fireEvent('quality.guess', files = files, extra = data, single = True)
data['quality'] = quality
if not data['quality']:
data['quality'] = fireEvent('quality.single', 'dvdr' if group['is_dvd'] else 'dvdrip', single = True)
@@ -462,9 +494,25 @@ class Scanner(Plugin):
filename = re.sub('(.cp\(tt[0-9{7}]+\))', '', files[0])
data['group'] = self.getGroup(filename[len(folder):])
data['source'] = self.getSourceMedia(filename)
if data['quality'].get('is_3d', 0):
data['3d_type'] = self.get3dType(filename)
return data
def get3dType(self, filename):
filename = ss(filename)
words = re.split('\W+', filename.lower())
for key in self.threed_types:
tags = self.threed_types.get(key, [])
for tag in tags:
if (isinstance(tag, tuple) and '.'.join(tag) in '.'.join(words)) or (isinstance(tag, (str, unicode)) and ss(tag.lower()) in words):
log.debug('Found %s in %s', (tag, filename))
return key
return ''
def getMeta(self, filename):
try:
@@ -708,19 +756,26 @@ class Scanner(Plugin):
if not file_size: file_size = []
try:
return (file_size.get('min', 0) * 1048576) < os.path.getsize(file) < (file_size.get('max', 100000) * 1048576)
return file_size.get('min', 0) < self.getFileSize(file) < file_size.get('max', 100000)
except:
log.error('Couldn\'t get filesize of %s.', file)
return False
def createStringIdentifier(self, file_path, folder = '', exclude_filename = False):
def getFileSize(self, file):
try:
return os.path.getsize(file) / 1024 / 1024
except:
return None
year = self.findYear(file_path)
def createStringIdentifier(self, file_path, folder = '', exclude_filename = False):
identifier = file_path.replace(folder, '').lstrip(os.path.sep) # root folder
identifier = os.path.splitext(identifier)[0] # ext
# Make sure the identifier is lower case as all regex is with lower case tags
identifier = identifier.lower()
try:
path_split = splitString(identifier, os.path.sep)
identifier = path_split[-2] if len(path_split) > 1 and len(path_split[-2]) > len(path_split[-1]) else path_split[-1] # Only get filename
@@ -735,8 +790,13 @@ class Scanner(Plugin):
# remove cptag
identifier = self.removeCPTag(identifier)
# groups, release tags, scenename cleaner, regex isn't correct
identifier = re.sub(self.clean, '::', simplifyString(identifier)).strip(':')
# simplify the string
identifier = simplifyString(identifier)
year = self.findYear(file_path)
# groups, release tags, scenename cleaner
identifier = re.sub(self.clean, '::', identifier).strip(':')
# Year
if year and identifier[:4] != year:
@@ -785,6 +845,14 @@ class Scanner(Plugin):
except:
return ''
def getResolution(self, filename):
try:
for key in self.resolutions:
if key in filename.lower() and key != 'default':
return self.resolutions[key]
except:
return self.resolutions['default']
def getGroup(self, file):
try:
match = re.findall('\-([A-Z0-9]+)[\.\/]', file, re.I)

View File

@@ -32,7 +32,7 @@ class Subtitle(Plugin):
for lang in self.getLanguages():
if lang not in available_languages:
download = subliminal.download_subtitles(files, multi = True, force = False, languages = [lang], services = self.services, cache_dir = Env.get('cache_dir'))
download = subliminal.download_subtitles(files, multi = True, force = self.conf('force'), languages = [lang], services = self.services, cache_dir = Env.get('cache_dir'))
for subtitle in download:
downloaded.extend(download[subtitle])
@@ -72,6 +72,14 @@ config = [{
'name': 'languages',
'description': ('Comma separated, 2 letter country code.', 'Example: en, nl. See the codes at <a href="http://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">on Wikipedia</a>'),
},
{
'advanced': True,
'name': 'force',
'label': 'Force',
'description': ('Force download all languages (including embedded).', 'This will also <strong>overwrite</strong> all existing subtitles.'),
'default': False,
'type': 'bool',
},
],
},
],

View File

@@ -2,6 +2,7 @@ Page.Userscript = new Class({
Extends: PageBase,
order: 80,
name: 'userscript',
has_tab: false,

View File

@@ -2,6 +2,7 @@ Page.Wizard = new Class({
Extends: Page.Settings,
order: 70,
name: 'wizard',
has_tab: false,
wizard_only: true,

View File

@@ -262,8 +262,14 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
# Go go go!
from tornado.ioloop import IOLoop
from tornado.autoreload import add_reload_hook
loop = IOLoop.current()
# Reload hook
def test():
fireEvent('app.shutdown')
add_reload_hook(test)
# Some logging and fire load event
try: log.info('Starting server on port %(port)s', config)
except: pass

View File

@@ -137,19 +137,34 @@
createPages: function(){
var self = this;
var pages = [];
Object.each(Page, function(page_class, class_name){
var pg = new Page[class_name](self, {});
self.pages[class_name] = pg;
self.fireEvent('load'+class_name);
$(pg).inject(self.content);
pages.include({
'order': pg.order,
'name': class_name,
'class': pg
});
});
pages.stableSort(self.sortPageByOrder).each(function(page){
page['class'].load();
self.fireEvent('load'+page.name);
$(page['class']).inject(self.content);
});
delete pages;
self.fireEvent('load');
},
sortPageByOrder: function(a, b){
return (a.order || 100) - (b.order || 100)
},
openPage: function(url) {
var self = this;

View File

@@ -6,6 +6,7 @@ var PageBase = new Class({
},
order: 1,
has_tab: true,
name: '',
@@ -16,6 +17,10 @@ var PageBase = new Class({
// Create main page container
self.el = new Element('div.page.'+self.name);
},
load: function(){
var self = this;
// Create tab for page
if(self.has_tab){
@@ -26,6 +31,7 @@ var PageBase = new Class({
'text': self.name.capitalize()
});
}
},
open: function(action, params){

View File

@@ -104,7 +104,7 @@ Page.Home = new Class({
// Make all thumbnails the same size
self.soon_list.addEvent('loaded', function(){
var images = $(self.soon_list).getElements('.poster'),
var images = $(self.soon_list).getElements('.poster, .no_thumbnail'),
timer,
highest = 100;

View File

@@ -2,6 +2,7 @@ Page.Settings = new Class({
Extends: PageBase,
order: 50,
name: 'settings',
title: 'Change settings.',
wizard_only: false,
@@ -112,7 +113,7 @@ Page.Settings = new Class({
},
sortByOrder: function(a, b){
return (a.order || 100) - (b.order || 100)
return (a.order || 100) - (b.order || 100)
},
create: function(json){