Merge branch 'develop' of github.com:RuudBurger/CouchPotatoServer into develop

couchpotato/core/downloaders/rtorrent_.py

@@ -154,19 +154,13 @@ class rTorrent(DownloaderBase):
             return False
 
     def getTorrentStatus(self, torrent):
-        if torrent.hashing or torrent.hash_checking or torrent.message:
-            return 'busy'
-
         if not torrent.complete:
             return 'busy'
 
-        if not torrent.open:
+        if torrent.open:
-            return 'completed'
-
-        if torrent.state and torrent.active:
             return 'seeding'
 
-        return 'busy'
+        return 'completed'
 
     def getAllDownloadStatus(self, ids):
         log.debug('Checking rTorrent download status.')

couchpotato/core/media/_base/media/index.py

@@ -99,7 +99,7 @@ from couchpotato.core.helpers.encoding import simplifyString"""
 
 
 class TitleIndex(TreeBasedIndex):
-    _version = 2
+    _version = 3
 
     custom_header = """from CodernityDB.tree_index import TreeBasedIndex
 from string import ascii_letters
@@ -123,7 +123,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
         nr_prefix = '' if title and len(title) > 0 and title[0] in ascii_letters else '#'
         title = simplifyString(title)
 
-        for prefix in ['the ']:
+        for prefix in ['the ', 'an ', 'a ']:
             if prefix == title[:len(prefix)]:
                 title = title[len(prefix):]
                 break
@@ -132,7 +132,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
 
 
 class StartsWithIndex(TreeBasedIndex):
-    _version = 2
+    _version = 3
 
     custom_header = """from CodernityDB.tree_index import TreeBasedIndex
 from string import ascii_letters
@@ -153,7 +153,7 @@ from couchpotato.core.helpers.encoding import toUnicode, simplifyString"""
         title = toUnicode(title)
         title = simplifyString(title)
 
-        for prefix in ['the ']:
+        for prefix in ['the ', 'an ', 'a ']:
             if prefix == title[:len(prefix)]:
                 title = title[len(prefix):]
                 break

couchpotato/core/media/_base/providers/base.py

@@ -200,7 +200,7 @@ class YarrProvider(Provider):
             self._search(media, quality, results)
         # Search possible titles
         else:
-            media_title = fireEvent('library.query', media, single = True)
+            media_title = fireEvent('library.query', media, include_year = False, single = True)
 
             for title in possibleTitles(media_title):
                 self._searchOnTitle(title, media, quality, results)

couchpotato/core/media/_base/providers/nzb/binsearch.py

@@ -2,7 +2,7 @@ import re
 import traceback
 
 from bs4 import BeautifulSoup
-from couchpotato.core.helpers.variable import tryInt
+from couchpotato.core.helpers.variable import tryInt, simplifyString
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.providers.nzb.base import NZBProvider
 
@@ -65,7 +65,7 @@ class Base(NZBProvider):
 
                     results.append({
                         'id': nzb_id,
-                        'name': title.text,
+                        'name': simplifyString(title.text),
                         'age': tryInt(age),
                         'size': self.parseSize(size_match.group('size')),
                         'url': self.urls['download'] % nzb_id,

couchpotato/core/media/_base/providers/nzb/newznab.py

@@ -2,6 +2,7 @@ from urllib2 import HTTPError
 from urlparse import urlparse
 import time
 import traceback
+import re
 import urllib2
 
 from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
@@ -20,10 +21,11 @@ log = CPLog(__name__)
 class Base(NZBProvider, RSS):
 
     urls = {
-        'detail': 'details&id=%s',
+        'detail': 'details/%s',
         'download': 't=get&id=%s'
     }
 
+    passwords_regex = 'password|wachtwoord'
     limits_reached = {}
 
     http_time_between_calls = 1  # Seconds
@@ -44,9 +46,7 @@ class Base(NZBProvider, RSS):
     def _searchOnHost(self, host, media, quality, results):
 
         query = self.buildUrl(media, host['api_key'])
-
         url = '%s&%s' % (self.getUrl(host['host']), query)
-
         nzbs = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})
 
         for nzb in nzbs:
@@ -79,6 +79,23 @@ class Base(NZBProvider, RSS):
             if spotter:
                 name_extra = spotter
 
+            description = ''
+            if "@spot.net" in nzb_id:
+                try:
+                    # Get details for extended description to retrieve passwords
+                    query = self.buildDetailsUrl(nzb_id, host['api_key'])
+                    url = '%s&%s' % (self.getUrl(host['host']), query)
+                    nzb_details = self.getRSSData(url, cache_timeout = 1800, headers = {'User-Agent': Env.getIdentifier()})[0]
+
+                    description = self.getTextElement(nzb_details, 'description')
+
+                    # Extract a password from the description
+                    password = re.search('(?:' + self.passwords_regex + ')(?: *)(?:\:|\=)(?: *)(.*?)\<br\>|\n|$', description, flags = re.I).group(1)
+                    if password:
+                        name = name + ' {{%s}}' % password.strip()
+                except:
+                    log.debug('Error getting details of "%s": %s', (name, traceback.format_exc()))
+
             results.append({
                 'id': nzb_id,
                 'provider_extra': urlparse(host['host']).hostname or host['host'],
@@ -87,8 +104,9 @@ class Base(NZBProvider, RSS):
                 'age': self.calculateAge(int(time.mktime(parse(date).timetuple()))),
                 'size': int(self.getElement(nzb, 'enclosure').attrib['length']) / 1024 / 1024,
                 'url': ((self.getUrl(host['host']) + self.urls['download']) % tryUrlencode(nzb_id)) + self.getApiExt(host),
-                'detail_url': '%sdetails/%s' % (cleanHost(host['host']), tryUrlencode(nzb_id)),
+                'detail_url': (cleanHost(host['host']) + self.urls['detail']) % tryUrlencode(nzb_id),
                 'content': self.getTextElement(nzb, 'description'),
+                'description': description,
                 'score': host['extra_score'],
             })
 
@@ -191,6 +209,15 @@ class Base(NZBProvider, RSS):
 
         return 'try_next'
 
+    def buildDetailsUrl(self, nzb_id, api_key):
+        query = tryUrlencode({
+            't': 'details',
+            'id': nzb_id,
+            'apikey': api_key,
+        })
+        return query
+
+
 
 config = [{
     'name': 'newznab',

couchpotato/core/media/_base/providers/torrent/bithdtv.py

@@ -25,7 +25,7 @@ class Base(TorrentProvider):
 
     def _search(self, media, quality, results):
 
-        query = self.buildUrl(media)
+        query = self.buildUrl(media, quality)
 
         url = "%s&%s" % (self.urls['search'], query)
 

couchpotato/core/media/_base/providers/torrent/kickasstorrents.py

@@ -65,12 +65,13 @@ class Base(TorrentMagnetProvider):
                                 if column_name:
 
                                     if column_name == 'name':
-                                        link = td.find('div', {'class': 'torrentname'}).find_all('a')[1]
+                                        link = td.find('div', {'class': 'torrentname'}).find_all('a')[2]
-                                        new['id'] = temp.get('id')[-8:]
+                                        new['id'] = temp.get('id')[-7:]
                                         new['name'] = link.text
                                         new['url'] = td.find('a', 'imagnet')['href']
                                         new['detail_url'] = self.urls['detail'] % (self.getDomain(), link['href'][1:])
-                                        new['score'] = 20 if td.find('a', 'iverif') else 0
+                                        new['verified'] = True if td.find('a', 'iverify') else False
+                                        new['score'] = 100 if new['verified'] else 0
                                     elif column_name is 'size':
                                         new['size'] = self.parseSize(td.text)
                                     elif column_name is 'age':
@@ -82,6 +83,10 @@ class Base(TorrentMagnetProvider):
 
                                 nr += 1
 
+                            # Only store verified torrents
+                            if self.conf('only_verified') and not new['verified']:
+                                continue
+
                             results.append(new)
                     except:
                         log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc())
@@ -151,6 +156,13 @@ config = [{
                     'default': 40,
                     'description': 'Will not be (re)moved until this seed time (in hours) is met.',
                 },
+                {
+                    'name': 'only_verified',
+                    'advanced': True,
+                    'type': 'bool',
+                    'default': False,
+                    'description': 'Only search for verified releases.'
+                },
                 {
                     'name': 'extra_score',
                     'advanced': True,

couchpotato/core/media/_base/providers/torrent/thepiratebay.py

@@ -65,7 +65,7 @@ class Base(TorrentMagnetProvider):
                         pass
 
                     entries = results_table.find_all('tr')
-                    for result in entries[2:]:
+                    for result in entries[1:]:
                         link = result.find(href = re.compile('torrent\/\d+\/'))
                         download = result.find(href = re.compile('magnet:'))
 
@@ -109,7 +109,11 @@ class Base(TorrentMagnetProvider):
         full_description = self.getCache('tpb.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
         html = BeautifulSoup(full_description)
         nfo_pre = html.find('div', attrs = {'class': 'nfo'})
-        description = toUnicode(nfo_pre.text) if nfo_pre else ''
+        description = ''
+        try: 
+            description = toUnicode(nfo_pre.text)
+        except:
+            pass
 
         item['description'] = description
         return item

couchpotato/core/media/_base/providers/torrent/yify.py

@@ -43,12 +43,10 @@ class Base(TorrentProvider):
             try:
                 for result in data.get('MovieList'):
 
-                    try:
+                    if result['Quality'] and result['Quality'] not in result['MovieTitle']:
-                        title = result['TorrentUrl'].split('/')[-1][:-8].replace('_', '.').strip('._')
+                        title = result['MovieTitle'] + ' ' + result['Quality']
-                        title = title.replace('.-.', '-')
+                    else: 
-                        title = title.replace('..', '.')
+                        title = result['MovieTitle']
-                    except:
-                        continue
 
                     results.append({
                         'id': result['MovieID'],

couchpotato/core/media/movie/_base/static/movie.js

@@ -250,6 +250,10 @@ var Movie = new Class({
 	getUnprefixedTitle: function(t){
 		if(t.substr(0, 4).toLowerCase() == 'the ')
 			t = t.substr(4) + ', The';
+		else if(t.substr(0, 3).toLowerCase() == 'an ')
+			t = t.substr(3) + ', An';
+		else if(t.substr(0, 2).toLowerCase() == 'a ')
+			t = t.substr(2) + ', A';
 		return t;
 	},
 

couchpotato/core/media/movie/charts/__init__.py

@@ -28,6 +28,20 @@ config = [{
                     'advanced': True,
                     'description': '(hours)',
                 },
+                {
+                    'name': 'hide_wanted',
+                    'default': False,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Hide the chart movies that are already in your wanted list.',
+                },
+                {
+                    'name': 'hide_library',
+                    'default': False,
+                    'type': 'bool',
+                    'advanced': True,
+                    'description': 'Hide the chart movies that are already in your library.',
+                },
             ],
         },
     ],

couchpotato/core/media/movie/charts/main.py

@@ -49,6 +49,9 @@ class Charts(Plugin):
         try:
             self.update_in_progress = True
             charts = fireEvent('automation.get_chart_list', merge = True)
+            for chart in charts:
+                chart['hide_wanted'] = self.conf('hide_wanted')
+                chart['hide_library'] = self.conf('hide_library')
             self.setCache('charts_cached', charts, timeout = 7200 * tryInt(self.conf('update_interval', default = 12)))
         except:
             log.error('Failed refreshing charts')

couchpotato/core/media/movie/charts/static/charts.css

@@ -11,6 +11,13 @@
 	display: inline-block;
 	width: 50%;
 	vertical-align: top;
+    max-height: 510px;
+    overflow-y: auto;
+    scrollbar-base-color: #4e5969;
+}
+
+.charts .chart .media_result.hidden {
+    display: none;
 }
 
 .charts .refresh {

couchpotato/core/media/movie/charts/static/charts.js

@@ -89,7 +89,7 @@ var Charts = new Class({
 						}
 					});
 
-					var in_database_class = movie.in_wanted ? 'chart_in_wanted' : (movie.in_library ? 'chart_in_library' : ''),
+					var in_database_class = (chart.hide_wanted && movie.in_wanted) ? 'hidden' : (movie.in_wanted ? 'chart_in_wanted' : ((chart.hide_library && movie.in_library) ? 'hidden': (movie.in_library ? 'chart_in_library' : ''))),
 						in_database_title = movie.in_wanted ? 'Movie in wanted list' : (movie.in_library ? 'Movie in library' : '');
 
 					m.el

couchpotato/core/media/movie/providers/automation/bluray.py

@@ -82,6 +82,7 @@ class Bluray(Automation, RSS):
     def getChartList(self):
         # Nearly identical to 'getIMDBids', but we don't care about minimalMovie and return all movie data (not just id)
         movie_list = {'name': 'Blu-ray.com - New Releases', 'url': self.display_url, 'order': self.chart_order, 'list': []}
+        movie_ids = []
         max_items = int(self.conf('max_items', section='charts', default=5))
         rss_movies = self.getRSSData(self.rss_url)
 
@@ -95,6 +96,9 @@ class Bluray(Automation, RSS):
             movie = self.search(name, year)
 
             if movie:
+                if movie.get('imdb') in movie_ids:
+                    continue
+                movie_ids.append(movie.get('imdb'))
                 movie_list['list'].append( movie )
                 if len(movie_list['list']) >= max_items:
                     break

couchpotato/core/media/movie/providers/torrent/bithdtv.py

@@ -10,10 +10,14 @@ autoload = 'BiTHDTV'
 
 
 class BiTHDTV(MovieProvider, Base):
+    cat_ids = [
+        ([2], ['bd50']),
+    ]
+    cat_backup_id = 7 # Movies
 
-    def buildUrl(self, media):
+    def buildUrl(self, media, quality):
         query = tryUrlencode({
             'search': fireEvent('library.query', media, single = True),
-            'cat': 7  # Movie cat
+            'cat': self.getCatId(quality)[0]
         })
         return query

couchpotato/core/media/movie/providers/torrent/publichd.py

@@ -11,4 +11,4 @@ autoload = 'PublicHD'
 class PublicHD(MovieProvider, Base):
 
     def buildUrl(self, media):
-        return fireEvent('library.query', media, single = True)
+        return fireEvent('library.query', media, single = True).replace(':', '')

couchpotato/core/media/movie/providers/torrent/thepiratebay.py

@@ -13,7 +13,7 @@ class ThePirateBay(MovieProvider, Base):
 
     cat_ids = [
         ([209], ['3d']),
-        ([207], ['720p', '1080p']),
+        ([207], ['720p', '1080p', 'bd50']),
         ([201], ['cam', 'ts', 'dvdrip', 'tc', 'r5', 'scr']),
         ([201, 207], ['brrip']),
         ([202], ['dvdr'])

couchpotato/core/media/movie/providers/torrent/torrentshack.py

@@ -31,6 +31,6 @@ class TorrentShack(MovieProvider, Base):
 
     def buildUrl(self, media, quality):
         query = (tryUrlencode(fireEvent('library.query', media, single = True)),
-                 self.getCatId(quality)[0],
+                 self.getSceneOnly(),
-                 self.getSceneOnly())
+                 self.getCatId(quality)[0])
         return query

couchpotato/core/media/movie/providers/userscript/filmstarts.py

@@ -0,0 +1,29 @@
+from BeautifulSoup import BeautifulSoup
+from couchpotato.core.media._base.providers.userscript.base import UserscriptBase
+
+autoload = 'Filmstarts'
+
+class Filmstarts(UserscriptBase):
+
+	includes = ['*://www.filmstarts.de/kritiken/*']
+	
+	def getMovie(self, url):
+		try:
+			data = self.getUrl(url)
+		except:
+			return
+			
+		html = BeautifulSoup(data)
+		table = html.find("table", attrs={"class": "table table-standard thead-standard table-striped_2 fs11"})
+		
+		if table.find(text='Originaltitel'):
+			# Get original film title from the table specified above
+			name = table.find("div", text="Originaltitel").parent.parent.parent.td.text
+		else:
+			# If none is available get the title from the meta data
+			name = html.find("meta", {"property":"og:title"})['content']
+			
+		# Year of production is not available in the meta data, so get it from the table
+		year = table.find("tr", text="Produktionsjahr").parent.parent.parent.td.text
+		
+		return self.search(name, year)

couchpotato/core/plugins/renamer.py

@@ -290,8 +290,10 @@ class Renamer(Plugin):
 
                 # Put 'The' at the end
                 name_the = movie_name
-                if movie_name[:4].lower() == 'the ':
+                for prefix in ['the ', 'an ', 'a ']:
-                    name_the = movie_name[4:] + ', The'
+                    if prefix == movie_name[:len(prefix)].lower():
+                        name_the = movie_name[len(prefix):] + ', ' + prefix.strip().capitalize()
+                        break
 
                 replacements = {
                     'ext': 'mkv',
@@ -312,8 +314,12 @@ class Renamer(Plugin):
                     'cd': '',
                     'cd_nr': '',
                     'mpaa': media['info'].get('mpaa', ''),
+                    'mpaa_only': media['info'].get('mpaa', ''),
                     'category': category_label,
                 }
+                
+                if replacements['mpaa_only'] not in ('G', 'PG', 'PG-13', 'R', 'NC-17'):
+                    replacements['mpaa_only'] = 'Not Rated'
 
                 for file_type in group['files']:
 
@@ -410,8 +416,12 @@ class Renamer(Plugin):
 
                             # Don't add language if multiple languages in 1 subtitle file
                             if len(sub_langs) == 1:
-                                sub_name = sub_name.replace(replacements['ext'], '%s.%s' % (sub_langs[0], replacements['ext']))
+                                sub_suffix = '%s.%s' % (sub_langs[0], replacements['ext'])
-                                rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)
+
+                                # Don't add language to subtitle file it it's already there
+                                if not sub_name.endswith(sub_suffix):
+                                    sub_name = sub_name.replace(replacements['ext'], sub_suffix)
+                                    rename_files[current_file] = os.path.join(destination, final_folder_name, sub_name)
 
                             rename_files = mergeDicts(rename_files, rename_extras)
 
@@ -652,7 +662,7 @@ Remove it if you want it to be renamed (again, or at least let it try again)
         elif isinstance(release_download, dict):
             # Tag download_files if they are known
             if release_download.get('files', []):
-                tag_files = release_download.get('files', [])
+                tag_files = [filename for filename in release_download.get('files', []) if os.path.exists(filename)]
 
             # Tag all files in release folder
             elif release_download['folder']:
@@ -1205,7 +1215,8 @@ rename_options = {
         'imdb_id': 'IMDB id (tt0123456)',
         'cd': 'CD number (cd1)',
         'cd_nr': 'Just the cd nr. (1)',
-        'mpaa': 'MPAA Rating',
+        'mpaa': 'MPAA or other certification',
+        'mpaa_only': 'MPAA only certification (G|PG|PG-13|R|NC-17|Not Rated)',
         'category': 'Category label',
     },
 }

couchpotato/core/plugins/scanner.py

@@ -60,29 +60,29 @@ class Scanner(Plugin):
     }
 
     codecs = {
-        'audio': ['dts', 'ac3', 'ac3d', 'mp3'],
+        'audio': ['DTS', 'AC3', 'AC3D', 'MP3'],
-        'video': ['x264', 'h264', 'divx', 'xvid']
+        'video': ['x264', 'H264', 'DivX', 'Xvid']
     }
 
     audio_codec_map = {
-        0x2000: 'ac3',
+        0x2000: 'AC3',
-        0x2001: 'dts',
+        0x2001: 'DTS',
-        0x0055: 'mp3',
+        0x0055: 'MP3',
-        0x0050: 'mp2',
+        0x0050: 'MP2',
-        0x0001: 'pcm',
+        0x0001: 'PCM',
-        0x003: 'pcm',
+        0x003: 'WAV',
-        0x77a1: 'tta1',
+        0x77a1: 'TTA1',
-        0x5756: 'wav',
+        0x5756: 'WAV',
-        0x6750: 'vorbis',
+        0x6750: 'Vorbis',
-        0xF1AC: 'flac',
+        0xF1AC: 'FLAC',
-        0x00ff: 'aac',
+        0x00ff: 'AAC',
     }
 
     source_media = {
-        'bluray': ['bluray', 'blu-ray', 'brrip', 'br-rip'],
+        'Blu-ray': ['bluray', 'blu-ray', 'brrip', 'br-rip'],
-        'hddvd': ['hddvd', 'hd-dvd'],
+        'HD DVD': ['hddvd', 'hd-dvd'],
-        'dvd': ['dvd'],
+        'DVD': ['dvd'],
-        'hdtv': ['hdtv']
+        'HDTV': ['hdtv']
     }
 
     clean = '[ _\,\.\(\)\[\]\-]?(3d|hsbs|sbs|extended.cut|directors.cut|french|swedisch|danish|dutch|swesub|spanish|german|ac3|dts|custom|dc|divx|divx5|dsr|dsrip|dutch|dvd|dvdr|dvdrip|dvdscr|dvdscreener|screener|dvdivx|cam|fragment|fs|hdtv|hdrip' \
@@ -471,7 +471,7 @@ class Scanner(Plugin):
             p = enzyme.parse(filename)
 
             # Video codec
-            vc = ('h264' if p.video[0].codec == 'AVC1' else p.video[0].codec).lower()
+            vc = ('H264' if p.video[0].codec == 'AVC1' else p.video[0].codec)
 
             # Audio codec
             ac = p.audio[0].codec

couchpotato/core/plugins/subtitle.py

@@ -32,7 +32,7 @@ class Subtitle(Plugin):
 
             for lang in self.getLanguages():
                 if lang not in available_languages:
-                    download = subliminal.download_subtitles(files, multi = True, force = False, languages = [lang], services = self.services, cache_dir = Env.get('cache_dir'))
+                    download = subliminal.download_subtitles(files, multi = True, force = self.conf('force'), languages = [lang], services = self.services, cache_dir = Env.get('cache_dir'))
                     for subtitle in download:
                         downloaded.extend(download[subtitle])
 
@@ -72,6 +72,14 @@ config = [{
                     'name': 'languages',
                     'description': ('Comma separated, 2 letter country code.', 'Example: en, nl. See the codes at <a href="http://en.wikipedia.org/wiki/List_of_ISO_639-1_codes">on Wikipedia</a>'),
                 },
+                {
+                    'advanced': True,
+                    'name': 'force',
+                    'label': 'Force',
+                    'description': ('Force download all languages (including embedded).', 'This will also <strong>overwrite</strong> all existing subtitles.'),
+                    'default': False,
+                    'type': 'bool',
+                },
             ],
         },
     ],