From cb774bc31bc1fde67fe217a13bab3a612a3a9366 Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 21 Apr 2012 16:39:34 +0200 Subject: [PATCH] Refactored updater to make git-less updating possible. closes #78 --- couchpotato/core/_base/updater/main.py | 316 ++++++++++++++++++------- 1 file changed, 237 insertions(+), 79 deletions(-) diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index 8bf26359..97210123 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -4,9 +4,13 @@ from couchpotato.core.helpers.request import jsonified from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env -from git.repository import LocalRepository from datetime import datetime +from dateutil.parser import parse +from git.repository import LocalRepository +import json import os +import shutil +import tarfile import time import traceback @@ -15,19 +19,14 @@ log = CPLog(__name__) class Updater(Plugin): - repo_name = 'RuudBurger/CouchPotatoServer' - - version = None - update_failed = False - update_version = None - last_check = 0 - def __init__(self): - self.repo = LocalRepository(Env.get('app_dir'), command = self.conf('git_command', default = 'git')) + if os.path.isdir(os.path.join(Env.get('app_dir'), '.git')): + self.updater = GitUpdater(self.conf('git_command', default = 'git')) + else: + self.updater = SourceUpdater() fireEvent('schedule.interval', 'updater.check', self.check, hours = 6) - addEvent('app.load', self.check) addApiView('updater.info', self.getInfo, docs = { @@ -35,7 +34,6 @@ class Updater(Plugin): 'return': { 'type': 'object', 'example': """{ - 'repo_name': "Name of used repository", 'last_check': "last checked for update", 'update_version': "available update version or empty", 'version': current_cp_version @@ -47,75 +45,88 @@ class Updater(Plugin): 'return': {'type': 'see updater.info'} }) - def getInfo(self): + def check(self): + if self.isDisabled(): + return + if self.updater.check(): + if self.conf('automatic') and not self.updater.update_failed: + if self.updater.doUpdate(): + fireEventAsync('app.crappy_restart') + else: + if self.conf('notification'): + fireEvent('updater.available', message = 'A new update is available', data = self.updater.getVersion()) + + def getInfo(self): + return jsonified(self.updater.info()) + + def checkView(self): + self.check() + return self.updater.getInfo() + + def doUpdateView(self): + return jsonified({ + 'success': self.updater.doUpdate() + }) + + +class BaseUpdater(Plugin): + + repo_user = 'RuudBurger' + repo_name = 'CouchPotatoServer' + branch = 'develop' + + version = None + update_failed = False + update_version = None + last_check = 0 + + def doUpdate(self): + pass + + def getInfo(self): return jsonified(self.info()) def info(self): return { - 'repo_name': self.repo_name, 'last_check': self.last_check, 'update_version': self.update_version, 'version': self.getVersion() } - def getVersion(self): - - if not self.version: - try: - output = self.repo.getHead() # Yes, please - log.debug('Git version output: %s' % output.hash) - self.version = { - 'hash': output.hash[:8], - 'date': output.getDate(), - } - except Exception, e: - log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s' % e) - return 'No GIT' - - return self.version - def check(self): + pass - if self.update_version or self.isDisabled(): - return + def deletePyc(self, only_excess = True): - log.info('Checking for new version on github for %s' % self.repo_name) - if not Env.get('dev'): - self.repo.fetch() + for root, dirs, files in os.walk(Env.get('app_dir')): - current_branch = self.repo.getCurrentBranch().name + pyc_files = filter(lambda filename: filename.endswith('.pyc'), files) + py_files = set(filter(lambda filename: filename.endswith('.py'), files)) + excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files - for branch in self.repo.getRemoteByName('origin').getBranches(): - if current_branch == branch.name: + for excess_pyc_file in excess_pyc_files: + full_path = os.path.join(root, excess_pyc_file) + log.debug('Removing old PYC file: %s' % full_path) + try: + os.remove(full_path) + except: + log.error('Couldn\'t remove %s: %s' % (full_path, traceback.format_exc())) - local = self.repo.getHead() - remote = branch.getHead() + for dir_name in dirs: + full_path = os.path.join(root, dir_name) + if len(os.listdir(full_path)) == 0: + try: + os.rmdir(full_path) + except: + log.error('Couldn\'t remove empty directory %s: %s' % (full_path, traceback.format_exc())) - log.info('Versions, local:%s, remote:%s' % (local.hash[:8], remote.hash[:8])) - if local.getDate() < remote.getDate(): - self.update_version = { - 'hash': remote.hash[:8], - 'date': remote.getDate(), - } - if self.conf('automatic') and not self.update_failed: - if self.doUpdate(): - fireEventAsync('app.crappy_restart') - else: - if self.conf('notification'): - fireEvent('updater.available', message = 'A new update is available', data = self.getVersion()) - self.last_check = time.time() +class GitUpdater(BaseUpdater): - def checkView(self): - self.check() - return self.getInfo() - - def doUpdateView(self): - return jsonified({ - 'success': self.doUpdate() - }) + def __init__(self, git_command): + self.repo = LocalRepository(Env.get('app_dir'), command = git_command) def doUpdate(self): try: @@ -141,29 +152,176 @@ class Updater(Plugin): return False - def deletePyc(self): + def getVersion(self): - for root, dirs, files in os.walk(Env.get('app_dir')): + if not self.version: + try: + output = self.repo.getHead() # Yes, please + log.debug('Git version output: %s' % output.hash) + self.version = { + 'hash': output.hash[:8], + 'date': output.getDate(), + } + except Exception, e: + log.error('Failed using GIT updater, running from source, you need to have GIT installed. %s' % e) + return 'No GIT' - pyc_files = filter(lambda filename: filename.endswith('.pyc'), files) - py_files = set(filter(lambda filename: filename.endswith('.py'), files)) - excess_pyc_files = filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) + return self.version - for excess_pyc_file in excess_pyc_files: - full_path = os.path.join(root, excess_pyc_file) - log.debug('Removing old PYC file: %s' % full_path) - try: - os.remove(full_path) - except: - log.error('Couldn\'t remove %s: %s' % (full_path, traceback.format_exc())) + def check(self): - for dir_name in dirs: - full_path = os.path.join(root, dir_name) - if len(os.listdir(full_path)) == 0: + if self.update_version: + return + + log.info('Checking for new version on github for %s' % self.repo_name) + if not Env.get('dev'): + self.repo.fetch() + + current_branch = self.repo.getCurrentBranch().name + + for branch in self.repo.getRemoteByName('origin').getBranches(): + if current_branch == branch.name: + + local = self.repo.getHead() + remote = branch.getHead() + + log.info('Versions, local:%s, remote:%s' % (local.hash[:8], remote.hash[:8])) + + if local.getDate() < remote.getDate(): + self.update_version = { + 'hash': remote.hash[:8], + 'date': remote.getDate(), + } + return True + + self.last_check = time.time() + return False + + + +class SourceUpdater(BaseUpdater): + + def __init__(self): + + # Create version file in cache + self.version_file = os.path.join(Env.get('cache_dir'), 'version') + if not os.path.isfile(self.version_file): + self.createFile(self.version_file, json.dumps(self.latestCommit())) + + def doUpdate(self): + + try: + url = 'https://github.com/%s/%s/tarball/%s' % (self.repo_user, self.repo_name, self.branch) + destination = os.path.join(Env.get('cache_dir'), self.update_version.get('hash') + '.tar.gz') + extracted_path = os.path.join(Env.get('cache_dir'), 'temp_updater') + + destination = fireEvent('file.download', url = url, dest = destination, single = True) + + # Cleanup leftover from last time + if os.path.isdir(extracted_path): + self.removeDir(extracted_path) + self.makeDir(extracted_path) + + # Extract + tar = tarfile.open(destination) + tar.extractall(path = extracted_path) + os.remove(destination) + + self.replaceWith(os.path.join(extracted_path, os.listdir(extracted_path)[0])) + self.removeDir(extracted_path) + + # Write update version to file + self.createFile(self.version_file, json.dumps(self.update_version)) + + return True + except: + log.error('Failed updating: %s' % traceback.format_exc()) + + self.update_failed = True + return False + + def replaceWith(self, path): + app_dir = Env.get('app_dir') + + # Get list of files we want to overwrite + self.deletePyc(only_excess = False) + existing_files = [] + for root, subfiles, filenames in os.walk(app_dir): + for filename in filenames: + existing_files.append(os.path.join(root, filename)) + + for root, subfiles, filenames in os.walk(path): + for filename in filenames: + fromfile = os.path.join(root, filename) + tofile = os.path.join(app_dir, fromfile.replace(path + os.path.sep, '')) + + if not Env.get('dev'): try: - os.rmdir(full_path) + os.remove(tofile) except: - log.error('Couldn\'t remove empty directory %s: %s' % (full_path, traceback.format_exc())) + pass - def isEnabled(self): - return super(Updater, self).isEnabled() and Env.get('uses_git') + try: + os.renames(fromfile, tofile) + try: + existing_files.remove(tofile) + except ValueError: + pass + except Exception, e: + log.error('Failed overwriting file: %s' % e) + + + def removeDir(self, path): + try: + if os.path.isdir(path): + shutil.rmtree(path) + except OSError, inst: + os.chmod(inst.filename, 0777) + self.removeDir(path) + + def getVersion(self): + + if not self.version: + try: + f = open(self.version_file, 'r') + output = json.loads(f.read()) + f.close() + + log.debug('Source version output: %s' % output) + self.version = output + except Exception, e: + log.error('Failed using source updater. %s' % e) + return {} + + return self.version + + def check(self): + + current_version = self.getVersion() + + try: + latest = self.latestCommit() + + if latest.get('hash') != current_version.get('hash') and latest.get('date') >= current_version.get('date'): + self.update_version = latest + + self.last_check = time.time() + except: + log.error('Failed updating via source: %s' % traceback.format_exc()) + + return self.update_version is not None + + def latestCommit(self): + try: + url = 'https://api.github.com/repos/%s/%s/commits?per_page=1&sha=%s' % (self.repo_user, self.repo_name, self.branch) + data = self.getCache('github.commit', url = url) + commit = json.loads(data)[0] + + return { + 'hash': commit['sha'], + 'date': int(time.mktime(parse(commit['commit']['committer']['date']).timetuple())), + } + except: + log.error('Failed getting latest request from github: %s' % traceback.format_exc()) + + return {}