From 7c1bb810fc0357c8eace7cdfb575da41f5e3806e Mon Sep 17 00:00:00 2001 From: mdipierro Date: Sat, 20 Apr 2019 13:53:43 -0700 Subject: [PATCH] cleaner fileutils and untarring/unzipping in fileutils, thanks Paolo --- gluon/fileutils.py | 138 ++++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 76 deletions(-) diff --git a/gluon/fileutils.py b/gluon/fileutils.py index c1644be7..9424d32e 100644 --- a/gluon/fileutils.py +++ b/gluon/fileutils.py @@ -18,12 +18,13 @@ import glob import time import datetime import logging +import shutil from gluon.http import HTTP from gzip import open as gzopen from gluon.recfile import generate from gluon._compat import PY2 -__all__ = [ +__all__ = ( 'parse_version', 'read_file', 'write_file', @@ -41,11 +42,11 @@ __all__ = [ 'check_credentials', 'w2p_pack', 'w2p_unpack', + 'create_app', 'w2p_pack_plugin', 'w2p_unpack_plugin', 'fix_newlines', - 'make_fake_file_like_object', -] +) def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"): @@ -58,7 +59,7 @@ def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"): tuple: Major, Minor, Patch, Release, Build Date """ - re_version = re.compile('(\d+)\.(\d+)\.(\d+)(\-(?P
[^\s+]*))?(\+(?P\S*))')
+    re_version = re.compile(r'(\d+)\.(\d+)\.(\d+)(-(?P
[^\s+]*))?(\+(?P\S*))')
     m = re_version.match(version.strip().split()[-1])
     if not m:
         return None
@@ -80,7 +81,7 @@ def parse_legacy(version="Version 1.99.0 (2011-09-19 08:23:26)"):
         tuple: Major, Minor, Patch, Release, Build Date
 
     """
-    re_version = re.compile('[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P.+?)\)\s*(?P[a-z]+)?')
+    re_version = re.compile(r'[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P.+?)\)\s*(?P[a-z]+)?')
     m = re_version.match(version)
     a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)),
     pre_release = m.group('type') or 'dev'
@@ -109,22 +110,16 @@ def read_file(filename, mode='r'):
     """Returns content from filename, making sure to close the file explicitly
     on exit.
     """
-    f = open_file(filename, mode)
-    try:
+    with open_file(filename, mode) as f:
         return f.read()
-    finally:
-        f.close()
 
 
 def write_file(filename, value, mode='w'):
     """Writes  to filename, making sure to close the file
     explicitly on exit.
     """
-    f = open_file(filename, mode)
-    try:
+    with open_file(filename, mode) as f:
         return f.write(value)
-    finally:
-        f.close()
 
 
 def readlines_file(filename, mode='r'):
@@ -200,10 +195,10 @@ def cleanpath(path):
 
     items = path.split('.')
     if len(items) > 1:
-        path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.'
+        path = re.sub(r'[^\w.]+', '_', '_'.join(items[:-1]) + '.'
                       + ''.join(items[-1:]))
     else:
-        path = re.sub('[^\w\.]+', '_', ''.join(items[-1:]))
+        path = re.sub(r'[^\w.]+', '_', ''.join(items[-1:]))
     return path
 
 
@@ -250,59 +245,64 @@ def w2p_pack(filename, path, compiled=False, filenames=None):
     path = abspath(path)
     tarname = filename + '.tar'
     if compiled:
-        tar_compiled(tarname, path, '^[\w\.\-]+$',
+        tar_compiled(tarname, path, r'^[\w.-]+$',
                      exclude_content_from=['cache', 'sessions', 'errors'])
     else:
-        tar(tarname, path, '^[\w\.\-]+$', filenames=filenames,
+        tar(tarname, path, r'^[\w.-]+$', filenames=filenames,
             exclude_content_from=['cache', 'sessions', 'errors'])
-    w2pfp = gzopen(filename, 'wb')
-    tarfp = open(tarname, 'rb')
-    w2pfp.write(tarfp.read())
-    w2pfp.close()
-    tarfp.close()
+    with open(tarname, 'rb') as tarfp, gzopen(filename, 'wb') as gzfp:
+        shutil.copyfileobj(tarfp, gzfp, 4194304) # 4 MB buffer
     os.unlink(tarname)
 
 
 def create_welcome_w2p():
-    is_newinstall_file = os.path.exists('NEWINSTALL')
-    if not os.path.exists('welcome.w2p') or is_newinstall_file:
+    is_newinstall = os.path.exists('NEWINSTALL')
+    if not os.path.exists('welcome.w2p') or is_newinstall:
+        logger = logging.getLogger("web2py")
         try:
             w2p_pack('welcome.w2p', 'applications/welcome')
-            logging.info("New installation: created welcome.w2p file")
+            logger.info("New installation: created welcome.w2p file")
         except:
-            logging.error("New installation error: unable to create welcome.w2p file")
+            logger.exception("New installation error: unable to create welcome.w2p file")
             return
-        if is_newinstall_file:
+        if is_newinstall:
             try:
                 os.unlink('NEWINSTALL')
-                logging.info("New installation: removed NEWINSTALL file")
+                logger.info("New installation: removed NEWINSTALL file")
             except:
-                logging.error("New installation error: unable to remove NEWINSTALL file")
+                logger.exception("New installation error: unable to remove NEWINSTALL file")
 
 
 def w2p_unpack(filename, path, delete_tar=True):
-
     if filename == 'welcome.w2p':
         create_welcome_w2p()
     filename = abspath(filename)
-    path = abspath(path)
-    if filename[-4:] == '.w2p' or filename[-3:] == '.gz':
-        if filename[-4:] == '.w2p':
-            tarname = filename[:-4] + '.tar'
-        else:
-            tarname = filename[:-3] + '.tar'
-        fgzipped = gzopen(filename, 'rb')
-        tarfile = open(tarname, 'wb')
-        tarfile.write(fgzipped.read())
-        tarfile.close()
-        fgzipped.close()
+    tarname = None
+    if filename.endswith('.w2p'):
+        tarname = filename[:-4] + '.tar'
+    elif filename.endswith('.gz'):
+        tarname = filename[:-3] + '.tar'
+    if tarname is not None:
+        with gzopen(filename, 'rb') as gzfp, open(tarname, 'wb') as tarfp:
+            shutil.copyfileobj(gzfp, tarfp, 4194304) # 4 MB buffer
     else:
         tarname = filename
+    path = abspath(path)
     untar(tarname, path)
     if delete_tar:
         os.unlink(tarname)
 
 
+def create_app(path):
+    w2p_unpack('welcome.w2p', path)
+    for subfolder in ('models', 'views', 'controllers', 'databases',
+                      'modules', 'cron', 'errors', 'sessions',
+                      'languages', 'static', 'private', 'uploads'):
+        subpath = os.path.join(path, subfolder)
+        if not os.path.exists(subpath):
+            os.mkdir(subpath)
+
+
 def w2p_pack_plugin(filename, path, plugin_name):
     """Packs the given plugin into a w2p file.
     Will match files at::
@@ -314,11 +314,10 @@ def w2p_pack_plugin(filename, path, plugin_name):
     filename = abspath(filename)
     path = abspath(path)
     if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name):
-        raise Exception("Not a web2py plugin name")
-    plugin_tarball = tarfile.open(filename, 'w:gz')
-    try:
+        raise ValueError('Not a web2py plugin')
+    with tarfile.open(filename, 'w:gz') as plugin_tarball:
         app_dir = path
-        while app_dir[-1] == '/':
+        while app_dir.endswith('/'):
             app_dir = app_dir[:-1]
         files1 = glob.glob(
             os.path.join(app_dir, '*/plugin_%s.*' % plugin_name))
@@ -326,15 +325,13 @@ def w2p_pack_plugin(filename, path, plugin_name):
             os.path.join(app_dir, '*/plugin_%s/*' % plugin_name))
         for file in files1 + files2:
             plugin_tarball.add(file, arcname=file[len(app_dir) + 1:])
-    finally:
-        plugin_tarball.close()
 
 
 def w2p_unpack_plugin(filename, path, delete_tar=True):
     filename = abspath(filename)
     path = abspath(path)
     if not os.path.basename(filename).startswith('web2py.plugin.'):
-        raise Exception("Not a web2py plugin")
+        raise ValueError('Not a web2py plugin')
     w2p_unpack(filename, path, delete_tar)
 
 
@@ -344,23 +341,22 @@ def tar_compiled(file, dir, expression='^.+$',
     The content of models, views, controllers is not stored in the tar file.
     """
 
-    tar = tarfile.TarFile(file, 'w')
-    for file in listdir(dir, expression, add_dirs=True,
-                        exclude_content_from=exclude_content_from):
-        filename = os.path.join(dir, file)
-        if os.path.islink(filename):
-            continue
-        if os.path.isfile(filename) and file[-4:] != '.pyc':
-            if file[:6] == 'models':
+    with tarfile.TarFile(file, 'w') as tar:
+        for file in listdir(dir, expression, add_dirs=True,
+                            exclude_content_from=exclude_content_from):
+            filename = os.path.join(dir, file)
+            if os.path.islink(filename):
                 continue
-            if file[:5] == 'views':
-                continue
-            if file[:11] == 'controllers':
-                continue
-            if file[:7] == 'modules':
-                continue
-        tar.add(filename, file, False)
-    tar.close()
+            if os.path.isfile(filename) and not file.endswith('.pyc'):
+                if file.startswith('models'):
+                    continue
+                if file.startswith('views'):
+                    continue
+                if file.startswith('controllers'):
+                    continue
+                if file.startswith('modules'):
+                    continue
+            tar.add(filename, file, False)
 
 
 def up(path):
@@ -378,7 +374,7 @@ def get_session(request, other_application='admin'):
         if not os.path.exists(session_filename):
             session_filename = generate(session_filename)
         osession = storage.load_storage(session_filename)
-    except Exception as e:
+    except:
         osession = storage.Storage()
     return osession
 
@@ -421,7 +417,7 @@ def fix_newlines(path):
     regex = re.compile(r'''(\r
 |\r|
 )''')
-    for filename in listdir(path, '.*\.(py|html)$', drop=False):
+    for filename in listdir(path, r'.*\.(py|html)$', drop=False):
         rdata = read_file(filename, 'r')
         wdata = regex.sub('\n', rdata)
         if wdata != rdata:
@@ -455,16 +451,6 @@ def copystream(
     return
 
 
-def make_fake_file_like_object():
-    class LogFile(object):
-        def write(self, value):
-            pass
-
-        def close(self):
-            pass
-    return LogFile()
-
-
 from gluon.settings import global_settings  # we need to import settings here because
                                       # settings imports fileutils too