|
Package Dropbox ::
Package web2py ::
Package gluon ::
Module fileutils
|
|
1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8 """
9
10 import sys
11 import storage
12 import os
13 import re
14 import tarfile
15 import glob
16 import time
17 import datetime
18 import logging
19 from http import HTTP
20 from gzip import open as gzopen
21
22
23 __all__ = [
24 'parse_version',
25 'read_file',
26 'write_file',
27 'readlines_file',
28 'up',
29 'abspath',
30 'mktree',
31 'listdir',
32 'recursive_unlink',
33 'cleanpath',
34 'tar',
35 'untar',
36 'tar_compiled',
37 'get_session',
38 'check_credentials',
39 'w2p_pack',
40 'w2p_unpack',
41 'w2p_pack_plugin',
42 'w2p_unpack_plugin',
43 'fix_newlines',
44 'make_fake_file_like_object',
45 ]
46
47
48 -def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"):
49 "http://semver.org/"
50 re_version = re.compile('Version (\d+)\.(\d+)\.(\d+)(\-(?P<pre>[^\s+]*))?(\+(?P<build>\S*))')
51 m = re_version.match(version)
52 if not m:
53 return None
54 a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3))
55 pre_release = m.group('pre') or ''
56 build = m.group('build') or ''
57 if build.startswith('timestamp'):
58 build = datetime.datetime.strptime(build.split('.',1)[1], '%Y.%m.%d.%H.%M.%S')
59 return (a, b, c, pre_release, build)
60
61 -def parse_legacy(version="Version 1.99.0 (2011-09-19 08:23:26)"):
62 re_version = re.compile('[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P<datetime>.+?)\)\s*(?P<type>[a-z]+)?')
63 m = re_version.match(version)
64 a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)),
65 pre_release = m.group('type') or 'dev'
66 build = datetime.datetime.strptime(m.group('datetime'), '%Y-%m-%d %H:%M:%S')
67 return (a, b, c, pre_release, build)
68
70 version_tuple = parse_semantic(version)
71 if not version_tuple:
72 version_tuple = parse_legacy(version)
73 return version_tuple
74
76 "returns content from filename, making sure to close the file explicitly on exit."
77 f = open(filename, mode)
78 try:
79 return f.read()
80 finally:
81 f.close()
82
83
85 "writes <value> to filename, making sure to close the file explicitly on exit."
86 f = open(filename, mode)
87 try:
88 return f.write(value)
89 finally:
90 f.close()
91
92
94 "applies .split('\n') to the output of read_file()"
95 return read_file(filename, mode).split('\n')
96
97
99 head, tail = os.path.split(path)
100 if head:
101 if tail:
102 mktree(head)
103 if not os.path.exists(head):
104 os.mkdir(head)
105
106
107 -def listdir(
108 path,
109 expression='^.+$',
110 drop=True,
111 add_dirs=False,
112 sort=True,
113 ):
114 """
115 like os.listdir() but you can specify a regex pattern to filter files.
116 if add_dirs is True, the returned items will have the full path.
117 """
118 if path[-1:] != os.path.sep:
119 path = path + os.path.sep
120 if drop:
121 n = len(path)
122 else:
123 n = 0
124 regex = re.compile(expression)
125 items = []
126 for (root, dirs, files) in os.walk(path, topdown=True):
127 for dir in dirs[:]:
128 if dir.startswith('.'):
129 dirs.remove(dir)
130 if add_dirs:
131 items.append(root[n:])
132 for file in sorted(files):
133 if regex.match(file) and not file.startswith('.'):
134 items.append(os.path.join(root, file)[n:])
135 if sort:
136 return sorted(items)
137 else:
138 return items
139
140
148
149
151 """
152 turns any expression/path into a valid filename. replaces / with _ and
153 removes special characters.
154 """
155
156 items = path.split('.')
157 if len(items) > 1:
158 path = re.sub('[^\w\.]+', '_', '_'.join(items[:-1]) + '.'
159 + ''.join(items[-1:]))
160 else:
161 path = re.sub('[^\w\.]+', '_', ''.join(items[-1:]))
162 return path
163
164
166 if not hasattr(tarfile.TarFile, 'extractall'):
167 from tarfile import ExtractError
168
169 class TarFile(tarfile.TarFile):
170
171 def extractall(self, path='.', members=None):
172 """Extract all members from the archive to the current working
173 directory and set owner, modification time and permissions on
174 directories afterwards. `path' specifies a different directory
175 to extract to. `members' is optional and must be a subset of the
176 list returned by getmembers().
177 """
178
179 directories = []
180 if members is None:
181 members = self
182 for tarinfo in members:
183 if tarinfo.isdir():
184
185
186
187
188 try:
189 os.makedirs(os.path.join(path,
190 tarinfo.name), 0777)
191 except EnvironmentError:
192 pass
193 directories.append(tarinfo)
194 else:
195 self.extract(tarinfo, path)
196
197
198
199 directories.sort(lambda a, b: cmp(a.name, b.name))
200 directories.reverse()
201
202
203
204 for tarinfo in directories:
205 path = os.path.join(path, tarinfo.name)
206 try:
207 self.chown(tarinfo, path)
208 self.utime(tarinfo, path)
209 self.chmod(tarinfo, path)
210 except ExtractError, e:
211 if self.errorlevel > 1:
212 raise
213 else:
214 self._dbg(1, 'tarfile: %s' % e)
215
216 _cls = TarFile
217 else:
218 _cls = tarfile.TarFile
219
220 tar = _cls(filename, 'r')
221 ret = tar.extractall(path, members)
222 tar.close()
223 return ret
224
225
226 -def tar(file, dir, expression='^.+$'):
227 """
228 tars dir into file, only tars file that match expression
229 """
230
231 tar = tarfile.TarFile(file, 'w')
232 try:
233 for file in listdir(dir, expression, add_dirs=True):
234 tar.add(os.path.join(dir, file), file, False)
235 finally:
236 tar.close()
237
238
240 """
241 untar file into dir
242 """
243
244 _extractall(file, dir)
245
246
247 -def w2p_pack(filename, path, compiled=False):
248 filename = abspath(filename)
249 path = abspath(path)
250 tarname = filename + '.tar'
251 if compiled:
252 tar_compiled(tarname, path, '^[\w\.\-]+$')
253 else:
254 tar(tarname, path, '^[\w\.\-]+$')
255 w2pfp = gzopen(filename, 'wb')
256 tarfp = open(tarname, 'rb')
257 w2pfp.write(tarfp.read())
258 w2pfp.close()
259 tarfp.close()
260 os.unlink(tarname)
261
263 if not os.path.exists('welcome.w2p') or os.path.exists('NEWINSTALL'):
264 try:
265 w2p_pack('welcome.w2p', 'applications/welcome')
266 os.unlink('NEWINSTALL')
267 logging.info("New installation: created welcome.w2p file")
268 except:
269 logging.error("New installation error: unable to create welcome.w2p file")
270
271
273
274 if filename=='welcome.w2p':
275 create_welcome_w2p()
276 filename = abspath(filename)
277 path = abspath(path)
278 if filename[-4:] == '.w2p' or filename[-3:] == '.gz':
279 if filename[-4:] == '.w2p':
280 tarname = filename[:-4] + '.tar'
281 else:
282 tarname = filename[:-3] + '.tar'
283 fgzipped = gzopen(filename, 'rb')
284 tarfile = open(tarname, 'wb')
285 tarfile.write(fgzipped.read())
286 tarfile.close()
287 fgzipped.close()
288 else:
289 tarname = filename
290 untar(tarname, path)
291 if delete_tar:
292 os.unlink(tarname)
293
294
296 """Pack the given plugin into a w2p file.
297 Will match files at:
298 <path>/*/plugin_[name].*
299 <path>/*/plugin_[name]/*
300 """
301 filename = abspath(filename)
302 path = abspath(path)
303 if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name):
304 raise Exception("Not a web2py plugin name")
305 plugin_tarball = tarfile.open(filename, 'w:gz')
306 try:
307 app_dir = path
308 while app_dir[-1] == '/':
309 app_dir = app_dir[:-1]
310 files1 = glob.glob(
311 os.path.join(app_dir, '*/plugin_%s.*' % plugin_name))
312 files2 = glob.glob(
313 os.path.join(app_dir, '*/plugin_%s/*' % plugin_name))
314 for file in files1 + files2:
315 plugin_tarball.add(file, arcname=file[len(app_dir) + 1:])
316 finally:
317 plugin_tarball.close()
318
319
326
327
329 """
330 used to tar a compiled application.
331 the content of models, views, controllers is not stored in the tar file.
332 """
333
334 tar = tarfile.TarFile(file, 'w')
335 for file in listdir(dir, expression, add_dirs=True):
336 filename = os.path.join(dir, file)
337 if os.path.islink(filename):
338 continue
339 if os.path.isfile(filename) and file[-4:] != '.pyc':
340 if file[:6] == 'models':
341 continue
342 if file[:5] == 'views':
343 continue
344 if file[:11] == 'controllers':
345 continue
346 if file[:7] == 'modules':
347 continue
348 tar.add(filename, file, False)
349 tar.close()
350
351
353 return os.path.dirname(os.path.normpath(path))
354
355
357 """ checks that user is authorized to access other_application"""
358 if request.application == other_application:
359 raise KeyError
360 try:
361 session_id = request.cookies['session_id_' + other_application].value
362 osession = storage.load_storage(os.path.join(
363 up(request.folder), other_application, 'sessions', session_id))
364 except Exception, e:
365 osession = storage.Storage()
366 return osession
367
368
370 """ checks that user is authorized to access other_application"""
371 if request.env.web2py_runtime_gae:
372 from google.appengine.api import users
373 if users.is_current_user_admin():
374 return True
375 else:
376 login_html = '<a href="%s">Sign in with your google account</a>.' \
377 % users.create_login_url(request.env.path_info)
378 raise HTTP(200, '<html><body>%s</body></html>' % login_html)
379 else:
380 dt = time.time() - expiration
381 s = get_session(request, other_application)
382 return (s.authorized and s.last_time and s.last_time > dt)
383
384
386 regex = re.compile(r'''(\r
387 |\r|
388 )''')
389 for filename in listdir(path, '.*\.(py|html)$', drop=False):
390 rdata = read_file(filename, 'rb')
391 wdata = regex.sub('\n', rdata)
392 if wdata != rdata:
393 write_file(filename, wdata, 'wb')
394
395
396 -def copystream(
397 src,
398 dest,
399 size,
400 chunk_size=10 ** 5,
401 ):
402 """
403 this is here because I think there is a bug in shutil.copyfileobj
404 """
405 while size > 0:
406 if size < chunk_size:
407 data = src.read(size)
408 else:
409 data = src.read(chunk_size)
410 length = len(data)
411 if length > size:
412 (data, length) = (data[:size], size)
413 size -= length
414 if length == 0:
415 break
416 dest.write(data)
417 if length < chunk_size:
418 break
419 dest.seek(0)
420 return
421
422
424 class LogFile(object):
425 def write(self, value):
426 pass
427
428 def close(self):
429 pass
430 return LogFile()
431
432
433 from settings import global_settings
434
435
436
438 "convert relative path to absolute path based (by default) on applications_parent"
439 path = os.path.join(*relpath)
440 gluon = base.get('gluon', False)
441 if os.path.isabs(path):
442 return path
443 if gluon:
444 return os.path.join(global_settings.gluon_parent, path)
445 return os.path.join(global_settings.applications_parent, path)
446