dinislage/web2py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleaner fileutils and untarring/unzipping in fileutils, thanks Paolo

Browse Source
This commit is contained in:
mdipierro
2019-04-20 13:53:53 -07:00
parent 8a9b2d687e
commit 7c1bb810fc
1 changed files with 62 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gluon/fileutils.py
+62 -76
View File
@@ -18,12 +18,13 @@ import glob
import time import time
import datetime import datetime
import logging import logging
import shutil
from gluon.http import HTTP from gluon.http import HTTP
from gzip import open as gzopen from gzip import open as gzopen
from gluon.recfile import generate from gluon.recfile import generate
from gluon._compat import PY2 from gluon._compat import PY2
__all__ = [ __all__ = (
'parse_version', 'parse_version',
'read_file', 'read_file',
'write_file', 'write_file',
@@ -41,11 +42,11 @@ __all__ = [
'check_credentials', 'check_credentials',
'w2p_pack', 'w2p_pack',
'w2p_unpack', 'w2p_unpack',
'create_app',
'w2p_pack_plugin', 'w2p_pack_plugin',
'w2p_unpack_plugin', 'w2p_unpack_plugin',
'fix_newlines', 'fix_newlines',
'make_fake_file_like_object', )
]
def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"): def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"):
@@ -58,7 +59,7 @@ def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"):
tuple: Major, Minor, Patch, Release, Build Date tuple: Major, Minor, Patch, Release, Build Date
""" """
re_version = re.compile('(\d+)\.(\d+)\.(\d+)(\-(?P<pre>[^\s+]*))?(\+(?P<build>\S*))') re_version = re.compile(r'(\d+)\.(\d+)\.(\d+)(-(?P<pre>[^\s+]*))?(\+(?P<build>\S*))')
m = re_version.match(version.strip().split()[-1]) m = re_version.match(version.strip().split()[-1])
if not m: if not m:
return None return None
@@ -80,7 +81,7 @@ def parse_legacy(version="Version 1.99.0 (2011-09-19 08:23:26)"):
tuple: Major, Minor, Patch, Release, Build Date tuple: Major, Minor, Patch, Release, Build Date
""" """
re_version = re.compile('[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P<datetime>.+?)\)\s*(?P<type>[a-z]+)?') re_version = re.compile(r'[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P<datetime>.+?)\)\s*(?P<type>[a-z]+)?')
m = re_version.match(version) m = re_version.match(version)
a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)), a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)),
pre_release = m.group('type') or 'dev' pre_release = m.group('type') or 'dev'
@@ -109,22 +110,16 @@ def read_file(filename, mode='r'):
"""Returns content from filename, making sure to close the file explicitly """Returns content from filename, making sure to close the file explicitly
on exit. on exit.
""" """
f = open_file(filename, mode) with open_file(filename, mode) as f:
try:
return f.read() return f.read()
finally:
f.close()
def write_file(filename, value, mode='w'): def write_file(filename, value, mode='w'):
"""Writes <value> to filename, making sure to close the file """Writes <value> to filename, making sure to close the file
explicitly on exit. explicitly on exit.
""" """
f = open_file(filename, mode) with open_file(filename, mode) as f:
try:
return f.write(value) return f.write(value)
finally:
f.close()
def readlines_file(filename, mode='r'): def readlines_file(filename, mode='r'):
@@ -200,10 +195,10 @@ def cleanpath(path):
items = path.split('.') items = path.split('.')
if len(items) > 1: if len(items) > 1:
path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.' path = re.sub(r'[^\w.]+', '_', '_'.join(items[:-1]) + '.'
+ ''.join(items[-1:])) + ''.join(items[-1:]))
else: else:
path = re.sub('[^\w\.]+', '_', ''.join(items[-1:])) path = re.sub(r'[^\w.]+', '_', ''.join(items[-1:]))
return path return path
@@ -250,59 +245,64 @@ def w2p_pack(filename, path, compiled=False, filenames=None):
path = abspath(path) path = abspath(path)
tarname = filename + '.tar' tarname = filename + '.tar'
if compiled: if compiled:
tar_compiled(tarname, path, '^[\w\.\-]+$', tar_compiled(tarname, path, r'^[\w.-]+$',
exclude_content_from=['cache', 'sessions', 'errors']) exclude_content_from=['cache', 'sessions', 'errors'])
else: else:
tar(tarname, path, '^[\w\.\-]+$', filenames=filenames, tar(tarname, path, r'^[\w.-]+$', filenames=filenames,
exclude_content_from=['cache', 'sessions', 'errors']) exclude_content_from=['cache', 'sessions', 'errors'])
w2pfp = gzopen(filename, 'wb') with open(tarname, 'rb') as tarfp, gzopen(filename, 'wb') as gzfp:
tarfp = open(tarname, 'rb') shutil.copyfileobj(tarfp, gzfp, 4194304) # 4 MB buffer
w2pfp.write(tarfp.read())
w2pfp.close()
tarfp.close()
os.unlink(tarname) os.unlink(tarname)
def create_welcome_w2p(): def create_welcome_w2p():
is_newinstall_file = os.path.exists('NEWINSTALL') is_newinstall = os.path.exists('NEWINSTALL')
if not os.path.exists('welcome.w2p') or is_newinstall_file: if not os.path.exists('welcome.w2p') or is_newinstall:
logger = logging.getLogger("web2py")
try: try:
w2p_pack('welcome.w2p', 'applications/welcome') w2p_pack('welcome.w2p', 'applications/welcome')
logging.info("New installation: created welcome.w2p file") logger.info("New installation: created welcome.w2p file")
except: except:
logging.error("New installation error: unable to create welcome.w2p file") logger.exception("New installation error: unable to create welcome.w2p file")
return return
if is_newinstall_file: if is_newinstall:
try: try:
os.unlink('NEWINSTALL') os.unlink('NEWINSTALL')
logging.info("New installation: removed NEWINSTALL file") logger.info("New installation: removed NEWINSTALL file")
except: except:
logging.error("New installation error: unable to remove NEWINSTALL file") logger.exception("New installation error: unable to remove NEWINSTALL file")
def w2p_unpack(filename, path, delete_tar=True): def w2p_unpack(filename, path, delete_tar=True):
if filename == 'welcome.w2p': if filename == 'welcome.w2p':
create_welcome_w2p() create_welcome_w2p()
filename = abspath(filename) filename = abspath(filename)
path = abspath(path) tarname = None
if filename[-4:] == '.w2p' or filename[-3:] == '.gz': if filename.endswith('.w2p'):
if filename[-4:] == '.w2p': tarname = filename[:-4] + '.tar'
tarname = filename[:-4] + '.tar' elif filename.endswith('.gz'):
else: tarname = filename[:-3] + '.tar'
tarname = filename[:-3] + '.tar' if tarname is not None:
fgzipped = gzopen(filename, 'rb') with gzopen(filename, 'rb') as gzfp, open(tarname, 'wb') as tarfp:
tarfile = open(tarname, 'wb') shutil.copyfileobj(gzfp, tarfp, 4194304) # 4 MB buffer
tarfile.write(fgzipped.read())
tarfile.close()
fgzipped.close()
else: else:
tarname = filename tarname = filename
path = abspath(path)
untar(tarname, path) untar(tarname, path)
if delete_tar: if delete_tar:
os.unlink(tarname) os.unlink(tarname)
def create_app(path):
w2p_unpack('welcome.w2p', path)
for subfolder in ('models', 'views', 'controllers', 'databases',
'modules', 'cron', 'errors', 'sessions',
'languages', 'static', 'private', 'uploads'):
subpath = os.path.join(path, subfolder)
if not os.path.exists(subpath):
os.mkdir(subpath)
def w2p_pack_plugin(filename, path, plugin_name): def w2p_pack_plugin(filename, path, plugin_name):
"""Packs the given plugin into a w2p file. """Packs the given plugin into a w2p file.
Will match files at:: Will match files at::
@@ -314,11 +314,10 @@ def w2p_pack_plugin(filename, path, plugin_name):
filename = abspath(filename) filename = abspath(filename)
path = abspath(path) path = abspath(path)
if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name): if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name):
raise Exception("Not a web2py plugin name") raise ValueError('Not a web2py plugin')
plugin_tarball = tarfile.open(filename, 'w:gz') with tarfile.open(filename, 'w:gz') as plugin_tarball:
try:
app_dir = path app_dir = path
while app_dir[-1] == '/': while app_dir.endswith('/'):
app_dir = app_dir[:-1] app_dir = app_dir[:-1]
files1 = glob.glob( files1 = glob.glob(
os.path.join(app_dir, '*/plugin_%s.*' % plugin_name)) os.path.join(app_dir, '*/plugin_%s.*' % plugin_name))
@@ -326,15 +325,13 @@ def w2p_pack_plugin(filename, path, plugin_name):
os.path.join(app_dir, '*/plugin_%s/*' % plugin_name)) os.path.join(app_dir, '*/plugin_%s/*' % plugin_name))
for file in files1 + files2: for file in files1 + files2:
plugin_tarball.add(file, arcname=file[len(app_dir) + 1:]) plugin_tarball.add(file, arcname=file[len(app_dir) + 1:])
finally:
plugin_tarball.close()
def w2p_unpack_plugin(filename, path, delete_tar=True): def w2p_unpack_plugin(filename, path, delete_tar=True):
filename = abspath(filename) filename = abspath(filename)
path = abspath(path) path = abspath(path)
if not os.path.basename(filename).startswith('web2py.plugin.'): if not os.path.basename(filename).startswith('web2py.plugin.'):
raise Exception("Not a web2py plugin") raise ValueError('Not a web2py plugin')
w2p_unpack(filename, path, delete_tar) w2p_unpack(filename, path, delete_tar)
@@ -344,23 +341,22 @@ def tar_compiled(file, dir, expression='^.+$',
The content of models, views, controllers is not stored in the tar file. The content of models, views, controllers is not stored in the tar file.
""" """
tar = tarfile.TarFile(file, 'w') with tarfile.TarFile(file, 'w') as tar:
for file in listdir(dir, expression, add_dirs=True, for file in listdir(dir, expression, add_dirs=True,
exclude_content_from=exclude_content_from): exclude_content_from=exclude_content_from):
filename = os.path.join(dir, file) filename = os.path.join(dir, file)
if os.path.islink(filename): if os.path.islink(filename):
continue
if os.path.isfile(filename) and file[-4:] != '.pyc':
if file[:6] == 'models':
continue continue
if file[:5] == 'views': if os.path.isfile(filename) and not file.endswith('.pyc'):
continue if file.startswith('models'):
if file[:11] == 'controllers': continue
continue if file.startswith('views'):
if file[:7] == 'modules': continue
continue if file.startswith('controllers'):
tar.add(filename, file, False) continue
tar.close() if file.startswith('modules'):
continue
tar.add(filename, file, False)
def up(path): def up(path):
@@ -378,7 +374,7 @@ def get_session(request, other_application='admin'):
if not os.path.exists(session_filename): if not os.path.exists(session_filename):
session_filename = generate(session_filename) session_filename = generate(session_filename)
osession = storage.load_storage(session_filename) osession = storage.load_storage(session_filename)
except Exception as e: except:
osession = storage.Storage() osession = storage.Storage()
return osession return osession
@@ -421,7 +417,7 @@ def fix_newlines(path):
regex = re.compile(r'''(\r regex = re.compile(r'''(\r
|\r| |\r|
)''') )''')
for filename in listdir(path, '.*\.(py|html)$', drop=False): for filename in listdir(path, r'.*\.(py|html)$', drop=False):
rdata = read_file(filename, 'r') rdata = read_file(filename, 'r')
wdata = regex.sub('\n', rdata) wdata = regex.sub('\n', rdata)
if wdata != rdata: if wdata != rdata:
@@ -455,16 +451,6 @@ def copystream(
return return
def make_fake_file_like_object():
class LogFile(object):
def write(self, value):
pass
def close(self):
pass
return LogFile()
from gluon.settings import global_settings # we need to import settings here because from gluon.settings import global_settings # we need to import settings here because
# settings imports fileutils too # settings imports fileutils too