dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update Tornado lib

Browse Source
This commit is contained in:
Ruud
2012-12-03 13:31:34 +01:00
parent f0a3358561
commit 802338a934
33 changed files with 2510 additions and 1336 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/tornado/options.py
+195 -197
View File
@@ -16,7 +16,8 @@
"""A command line parsing module that lets modules define their own options.
Each module defines its own options, e.g.::
Each module defines its own options which are added to the global
option namespace, e.g.::
from tornado.options import define, options
@@ -30,12 +31,15 @@ Each module defines its own options, e.g.::
The main() method of your application does not need to be aware of all of
the options used throughout your program; they are all automatically loaded
when the modules are loaded. Your main() method can parse the command line
or parse a config file with::
when the modules are loaded. However, all modules that define options
must have been imported before the command line is parsed.
Your main() method can parse the command line or parse a config file with
either::
import tornado.options
tornado.options.parse_config_file("/etc/server.conf")
tornado.options.parse_command_line()
# or
tornado.options.parse_config_file("/etc/server.conf")
Command line formats are what you would expect ("--myoption=myvalue").
Config files are just Python files. Global names become options, e.g.::
@@ -46,26 +50,24 @@ Config files are just Python files. Global names become options, e.g.::
We support datetimes, timedeltas, ints, and floats (just pass a 'type'
kwarg to define). We also accept multi-value options. See the documentation
for define() below.
`tornado.options.options` is a singleton instance of `OptionParser`, and
the top-level functions in this module (`define`, `parse_command_line`, etc)
simply call methods on it. You may create additional `OptionParser`
instances to define isolated sets of options, such as for subcommands.
"""
from __future__ import absolute_import, division, with_statement
import datetime
import logging
import logging.handlers
import re
import sys
import os
import time
import textwrap
from tornado.escape import _unicode
# For pretty log messages, if available
try:
import curses
except ImportError:
curses = None
from tornado.log import define_logging_options
from tornado import stack_context
class Error(Exception):
@@ -73,27 +75,68 @@ class Error(Exception):
pass
class _Options(dict):
class OptionParser(object):
"""A collection of options, a dictionary with object-like access.
Normally accessed via static functions in the `tornado.options` module,
which reference a global instance.
"""
def __init__(self):
# we have to use self.__dict__ because we override setattr.
self.__dict__['_options'] = {}
self.__dict__['_parse_callbacks'] = []
self.define("help", type=bool, help="show this help information",
callback=self._help_callback)
def __getattr__(self, name):
if isinstance(self.get(name), _Option):
return self[name].value()
if isinstance(self._options.get(name), _Option):
return self._options[name].value()
raise AttributeError("Unrecognized option %r" % name)
def __setattr__(self, name, value):
if isinstance(self.get(name), _Option):
return self[name].set(value)
if isinstance(self._options.get(name), _Option):
return self._options[name].set(value)
raise AttributeError("Unrecognized option %r" % name)
def define(self, name, default=None, type=None, help=None, metavar=None,
multiple=False, group=None):
if name in self:
multiple=False, group=None, callback=None):
"""Defines a new command line option.
If type is given (one of str, float, int, datetime, or timedelta)
or can be inferred from the default, we parse the command line
arguments based on the given type. If multiple is True, we accept
comma-separated values, and the option value is always a list.
For multi-value integers, we also accept the syntax x:y, which
turns into range(x, y) - very useful for long integer ranges.
help and metavar are used to construct the automatically generated
command line help string. The help message is formatted like::
--name=METAVAR help string
group is used to group the defined options in logical
groups. By default, command line options are grouped by the
file in which they are defined.
Command line option names must be unique globally. They can be parsed
from the command line with parse_command_line() or parsed from a
config file with parse_config_file.
If a callback is given, it will be run with the new value whenever
the option is changed. This can be used to combine command-line
and file-based options::
define("config", type=str, help="path to config file",
callback=lambda path: parse_config_file(path, final=False))
With this definition, options in the file specified by ``--config`` will
override options set earlier on the command line, but can be overridden
by later flags.
"""
if name in self._options:
raise Error("Option %r already defined in %s", name,
self[name].file_name)
self._options[name].file_name)
frame = sys._getframe(0)
options_file = frame.f_code.co_filename
file_name = frame.f_back.f_code.co_filename
@@ -108,11 +151,23 @@ class _Options(dict):
group_name = group
else:
group_name = file_name
self[name] = _Option(name, file_name=file_name, default=default,
type=type, help=help, metavar=metavar,
multiple=multiple, group_name=group_name)
self._options[name] = _Option(name, file_name=file_name,
default=default, type=type, help=help,
metavar=metavar, multiple=multiple,
group_name=group_name,
callback=callback)
def parse_command_line(self, args=None):
def parse_command_line(self, args=None, final=True):
"""Parses all options given on the command line (defaults to sys.argv).
Note that args[0] is ignored since it is the program name in sys.argv.
We return a list of all arguments that are not parsed as options.
If ``final`` is ``False``, parse callbacks will not be run.
This is useful for applications that wish to combine configurations
from multiple sources.
"""
if args is None:
args = sys.argv
remaining = []
@@ -127,40 +182,46 @@ class _Options(dict):
arg = args[i].lstrip("-")
name, equals, value = arg.partition("=")
name = name.replace('-', '_')
if not name in self:
print_help()
if not name in self._options:
self.print_help()
raise Error('Unrecognized command line option: %r' % name)
option = self[name]
option = self._options[name]
if not equals:
if option.type == bool:
value = "true"
else:
raise Error('Option %r requires a value' % name)
option.parse(value)
if self.help:
print_help()
sys.exit(0)
# Set up log level and pretty console logging by default
if self.logging != 'none':
logging.getLogger().setLevel(getattr(logging, self.logging.upper()))
enable_pretty_logging()
if final:
self.run_parse_callbacks()
return remaining
def parse_config_file(self, path):
def parse_config_file(self, path, final=True):
"""Parses and loads the Python config file at the given path.
If ``final`` is ``False``, parse callbacks will not be run.
This is useful for applications that wish to combine configurations
from multiple sources.
"""
config = {}
execfile(path, config, config)
for name in config:
if name in self:
self[name].set(config[name])
if name in self._options:
self._options[name].set(config[name])
def print_help(self, file=sys.stdout):
"""Prints all the command line options to stdout."""
if final:
self.run_parse_callbacks()
def print_help(self, file=None):
"""Prints all the command line options to stderr (or another file)."""
if file is None:
file = sys.stderr
print >> file, "Usage: %s [OPTIONS]" % sys.argv[0]
print >> file, "\nOptions:\n"
by_group = {}
for option in self.itervalues():
for option in self._options.itervalues():
by_group.setdefault(option.group_name, []).append(option)
for filename, o in sorted(by_group.items()):
@@ -182,10 +243,67 @@ class _Options(dict):
print >> file, "%-34s %s" % (' ', line)
print >> file
def _help_callback(self, value):
if value:
self.print_help()
sys.exit(0)
def add_parse_callback(self, callback):
"""Adds a parse callback, to be invoked when option parsing is done."""
self._parse_callbacks.append(stack_context.wrap(callback))
def run_parse_callbacks(self):
for callback in self._parse_callbacks:
callback()
def mockable(self):
"""Returns a wrapper around self that is compatible with `mock.patch`.
The `mock.patch` function (included in the standard library
`unittest.mock` package since Python 3.3, or in the
third-party `mock` package for older versions of Python) is
incompatible with objects like ``options`` that override
``__getattr__`` and ``__setattr__``. This function returns an
object that can be used with `mock.patch.object` to modify
option values::
with mock.patch.object(options.mockable(), 'name', value):
assert options.name == value
"""
return _Mockable(self)
class _Mockable(object):
"""`mock.patch` compatible wrapper for `OptionParser`.
As of ``mock`` version 1.0.1, when an object uses ``__getattr__``
hooks instead of ``__dict__``, ``patch.__exit__`` tries to delete
the attribute it set instead of setting a new one (assuming that
the object does not catpure ``__setattr__``, so the patch
created a new attribute in ``__dict__``).
_Mockable's getattr and setattr pass through to the underlying
OptionParser, and delattr undoes the effect of a previous setattr.
"""
def __init__(self, options):
# Modify __dict__ directly to bypass __setattr__
self.__dict__['_options'] = options
self.__dict__['_originals'] = {}
def __getattr__(self, name):
return getattr(self._options, name)
def __setattr__(self, name, value):
assert name not in self._originals, "don't reuse mockable objects"
self._originals[name] = getattr(self._options, name)
setattr(self._options, name, value)
def __delattr__(self, name):
setattr(self._options, name, self._originals.pop(name))
class _Option(object):
def __init__(self, name, default=None, type=basestring, help=None, metavar=None,
multiple=False, file_name=None, group_name=None):
def __init__(self, name, default=None, type=basestring, help=None,
metavar=None, multiple=False, file_name=None, group_name=None,
callback=None):
if default is None and multiple:
default = []
self.name = name
@@ -195,6 +313,7 @@ class _Option(object):
self.multiple = multiple
self.file_name = file_name
self.group_name = group_name
self.callback = callback
self.default = default
self._value = None
@@ -221,6 +340,8 @@ class _Option(object):
self._value.append(_parse(part))
else:
self._value = _parse(value)
if self.callback is not None:
self.callback(self._value)
return self.value()
def set(self, value):
@@ -237,6 +358,8 @@ class _Option(object):
raise Error("Option %r is required to be a %s (%s given)" %
(self.name, self.type.__name__, type(value)))
self._value = value
if self.callback is not None:
self.callback(self._value)
# Supported date/time formats in our options
_DATETIME_FORMATS = [
@@ -303,179 +426,54 @@ class _Option(object):
return _unicode(value)
options = _Options()
"""Global options dictionary.
options = OptionParser()
"""Global options object.
Supports both attribute-style and dict-style access.
All defined options are available as attributes on this object.
"""
def define(name, default=None, type=None, help=None, metavar=None,
multiple=False, group=None):
"""Defines a new command line option.
multiple=False, group=None, callback=None):
"""Defines an option in the global namespace.
If type is given (one of str, float, int, datetime, or timedelta)
or can be inferred from the default, we parse the command line
arguments based on the given type. If multiple is True, we accept
comma-separated values, and the option value is always a list.
For multi-value integers, we also accept the syntax x:y, which
turns into range(x, y) - very useful for long integer ranges.
help and metavar are used to construct the automatically generated
command line help string. The help message is formatted like::
--name=METAVAR help string
group is used to group the defined options in logical groups. By default,
command line options are grouped by the defined file.
Command line option names must be unique globally. They can be parsed
from the command line with parse_command_line() or parsed from a
config file with parse_config_file.
See `OptionParser.define`.
"""
return options.define(name, default=default, type=type, help=help,
metavar=metavar, multiple=multiple, group=group)
metavar=metavar, multiple=multiple, group=group,
callback=callback)
def parse_command_line(args=None):
"""Parses all options given on the command line (defaults to sys.argv).
def parse_command_line(args=None, final=True):
"""Parses global options from the command line.
Note that args[0] is ignored since it is the program name in sys.argv.
We return a list of all arguments that are not parsed as options.
See `OptionParser.parse_command_line`.
"""
return options.parse_command_line(args)
return options.parse_command_line(args, final=final)
def parse_config_file(path):
"""Parses and loads the Python config file at the given path."""
return options.parse_config_file(path)
def parse_config_file(path, final=True):
"""Parses global options from a config file.
See `OptionParser.parse_config_file`.
"""
return options.parse_config_file(path, final=final)
def print_help(file=sys.stdout):
"""Prints all the command line options to stdout."""
def print_help(file=None):
"""Prints all the command line options to stderr (or another file).
See `OptionParser.print_help`.
"""
return options.print_help(file)
def add_parse_callback(callback):
"""Adds a parse callback, to be invoked when option parsing is done.
def enable_pretty_logging(options=options):
"""Turns on formatted logging output as configured.
This is called automatically by `parse_command_line`.
See `OptionParser.add_parse_callback`
"""
root_logger = logging.getLogger()
if options.log_file_prefix:
channel = logging.handlers.RotatingFileHandler(
filename=options.log_file_prefix,
maxBytes=options.log_file_max_size,
backupCount=options.log_file_num_backups)
channel.setFormatter(_LogFormatter(color=False))
root_logger.addHandler(channel)
if (options.log_to_stderr or
(options.log_to_stderr is None and not root_logger.handlers)):
# Set up color if we are in a tty and curses is installed
color = False
if curses and sys.stderr.isatty():
try:
curses.setupterm()
if curses.tigetnum("colors") > 0:
color = True
except Exception:
pass
channel = logging.StreamHandler()
channel.setFormatter(_LogFormatter(color=color))
root_logger.addHandler(channel)
class _LogFormatter(logging.Formatter):
def __init__(self, color, *args, **kwargs):
logging.Formatter.__init__(self, *args, **kwargs)
self._color = color
if color:
# The curses module has some str/bytes confusion in
# python3. Until version 3.2.3, most methods return
# bytes, but only accept strings. In addition, we want to
# output these strings with the logging module, which
# works with unicode strings. The explicit calls to
# unicode() below are harmless in python2 but will do the
# right conversion in python 3.
fg_color = (curses.tigetstr("setaf") or
curses.tigetstr("setf") or "")
if (3, 0) < sys.version_info < (3, 2, 3):
fg_color = unicode(fg_color, "ascii")
self._colors = {
logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue
"ascii"),
logging.INFO: unicode(curses.tparm(fg_color, 2), # Green
"ascii"),
logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow
"ascii"),
logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red
"ascii"),
}
self._normal = unicode(curses.tigetstr("sgr0"), "ascii")
def format(self, record):
try:
record.message = record.getMessage()
except Exception, e:
record.message = "Bad message (%r): %r" % (e, record.__dict__)
assert isinstance(record.message, basestring) # guaranteed by logging
record.asctime = time.strftime(
"%y%m%d %H:%M:%S", self.converter(record.created))
prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \
record.__dict__
if self._color:
prefix = (self._colors.get(record.levelno, self._normal) +
prefix + self._normal)
# Encoding notes: The logging module prefers to work with character
# strings, but only enforces that log messages are instances of
# basestring. In python 2, non-ascii bytestrings will make
# their way through the logging framework until they blow up with
# an unhelpful decoding error (with this formatter it happens
# when we attach the prefix, but there are other opportunities for
# exceptions further along in the framework).
#
# If a byte string makes it this far, convert it to unicode to
# ensure it will make it out to the logs. Use repr() as a fallback
# to ensure that all byte strings can be converted successfully,
# but don't do it by default so we don't add extra quotes to ascii
# bytestrings. This is a bit of a hacky place to do this, but
# it's worth it since the encoding errors that would otherwise
# result are so useless (and tornado is fond of using utf8-encoded
# byte strings whereever possible).
try:
message = _unicode(record.message)
except UnicodeDecodeError:
message = repr(record.message)
formatted = prefix + " " + message
if record.exc_info:
if not record.exc_text:
record.exc_text = self.formatException(record.exc_info)
if record.exc_text:
formatted = formatted.rstrip() + "\n" + record.exc_text
return formatted.replace("\n", "\n ")
options.add_parse_callback(callback)
# Default options
define("help", type=bool, help="show this help information")
define("logging", default="info",
help=("Set the Python log level. If 'none', tornado won't touch the "
"logging configuration."),
metavar="debug|info|warning|error|none")
define("log_to_stderr", type=bool, default=None,
help=("Send log output to stderr (colorized if possible). "
"By default use stderr if --log_file_prefix is not set and "
"no other logging is configured."))
define("log_file_prefix", type=str, default=None, metavar="PATH",
help=("Path prefix for log files. "
"Note that if you are running multiple tornado processes, "
"log_file_prefix must be different for each of them (e.g. "
"include the port number)"))
define("log_file_max_size", type=int, default=100 * 1000 * 1000,
help="max size of log files before rollover")
define("log_file_num_backups", type=int, default=10,
help="number of log files to keep")
define_logging_options(options)