dinislage/CouchPotatoServer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update libs

Browse Source
This commit is contained in:
Ruud
2013-11-22 16:09:15 +01:00
parent f53364eb6c
commit 88d6148500
119 changed files with 18448 additions and 8923 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libs/pyutil/_version.py
+1 -1
View File
@@ -6,7 +6,7 @@
# pyutil.version_class for a description of what the different fields mean.
__pkgname__ = "pyutil"
verstr = "1.9.3"
verstr = "1.9.7"
try:
from pyutil.version_class import Version as pyutil_Version
__version__ = pyutil_Version(verstr)
libs/pyutil/benchutil.py
+44 -36
View File
@@ -1,4 +1,4 @@
# Copyright (c) 2002-2012 Zooko Wilcox-O'Hearn
# Copyright (c) 2002-2013 Zooko Wilcox-O'Hearn
# This file is part of pyutil; see README.rst for licensing terms.
"""
@@ -21,10 +21,10 @@ the second, e.g.:
>>> rep_bench(fib, 25, UNITS_PER_SECOND=1000)
best: 1.968e+00, 3th-best: 1.987e+00, mean: 2.118e+00, 3th-worst: 2.175e+00, worst: 2.503e+00 (of 10)
The output is reporting the number of milliseconds that executing the function
took, divided by N, from ten different invocations of fib(). It reports the
best, worst, M-th best, M-th worst, and mean, where "M" is the natural log of
the number of invocations (in this case 10).
The output is reporting the number of milliseconds that executing the
function took, divided by N, from ten different invocations of
fib(). It reports the best, worst, M-th best, M-th worst, and mean,
where "M" is 1/4 of the number of invocations (in this case 10).
2. Now run it with different values of N and look for patterns:
@@ -74,10 +74,12 @@ and the main function is to make them be methods of the same object, e.g.:
4. Things to fix:
a. I used to have it hooked up to use the "hotshot" profiler on the code being
measured. I recently tried to change it to use the newer cProfile profiler
instead, but I don't understand the interface to cProfiler so it just gives an
exception if you pass profile=True. Please fix this and send me a patch.
a. I used to have it hooked up to use the "hotshot" profiler on the
code being measured. I recently tried to change it to use the newer
cProfile profiler instead, but I don't understand the interface to
cProfiler so it just gives an exception if you pass
profile=True. Please fix this and send me a patch. xxx change it to
statprof
b. Wouldn't it be great if this script emitted results in a json format that
was understood by a tool to make pretty interactive explorable graphs? The
@@ -122,7 +124,7 @@ def mult(a, b):
except TypeError:
return to_decimal(a) * to_decimal(b)
def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False):
def rep_bench(func, n, runtime=1.0, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False):
"""
Will run the func up to MAXREPS times, but won't start a new run if MAXTIME
(wall-clock time) has already elapsed (unless MAXTIME is None).
@@ -130,33 +132,43 @@ def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, p
@param quiet Don't print anything--just return the results dict.
"""
assert isinstance(n, int), (n, type(n))
global worstemptymeasure
emsta = clock()
do_nothing(2**32)
emstop = clock()
empty = emstop - emsta
if empty > worstemptymeasure:
worstemptymeasure = empty
if (worstemptymeasure*2) >= runtime:
raise BadMeasure("Apparently simply invoking an empty Python function can take as long as %0.10f seconds, and we were running iterations for only about %0.10f seconds. So the measurement of the runtime of the code under benchmark is not reliable. Please pass a higher number for the 'runtime' argument to bench_it().")
startwallclocktime = time.time()
tls = [] # elapsed time in seconds
tls = [] # (elapsed time per iter in seconds, iters)
bmes = []
while ((len(tls) < MAXREPS) or (MAXREPS is None)) and ((MAXTIME is None) or ((time.time() - startwallclocktime) < MAXTIME)):
if initfunc:
initfunc(n)
try:
tl = bench_it(func, n, profile=profile, profresults=profresults)
tl, iters = bench_it(func, n, runtime=runtime, profile=profile, profresults=profresults)
except BadMeasure, bme:
bmes.append(bme)
else:
tls.append(tl)
tls.append((tl, iters))
if len(tls) == 0:
raise Exception("Couldn't get any measurements within time limits or number-of-attempts limits. Maybe something is wrong with your clock? %s" % (bmes,))
sumtls = reduce(operator.__add__, tls)
sumtls = sum([tl for (tl, iters) in tls])
mean = sumtls / len(tls)
tls.sort()
worst = tls[-1]
best = tls[0]
_assert(best > worstemptymeasure*MARGINOFERROR, "%s(n=%s) took %0.10f seconds, but we cannot measure times much less than about %0.10f seconds. Try a more time-consuming variant (such as higher n)." % (func, n, best, worstemptymeasure*MARGINOFERROR,))
worst = tls[-1][0]
best = tls[0][0]
m = len(tls)/4
if m > 0:
mthbest = tls[m-1]
mthworst = tls[-m]
mthbest = tls[m-1][0]
mthworst = tls[-m][0]
else:
mthbest = tls[0]
mthworst = tls[-1]
mthbest = tls[0][0]
mthworst = tls[-1][0]
# The +/-0 index is the best/worst, the +/-1 index is the 2nd-best/worst,
# etc, so we use mp1 to name it.
@@ -196,26 +208,22 @@ class BadMeasure(Exception):
def do_nothing(n):
pass
def bench_it(func, n, profile=False, profresults="pyutil-benchutil.prof"):
def bench_it(func, n, runtime=1.0, profile=False, profresults="pyutil-benchutil.prof"):
if profile:
st = clock()
cProfile.run('func(n)', profresults)
sto = clock()
raise NotImplementedException()
else:
iters = 0
st = clock()
func(n)
deadline = st + runtime
sto = clock()
while sto < deadline:
func(n)
iters += 1
sto = clock()
timeelapsed = sto - st
if timeelapsed <= 0:
raise BadMeasure(timeelapsed)
global worstemptymeasure
emsta = clock()
do_nothing(2**32)
emstop = clock()
empty = emstop - emsta
if empty > worstemptymeasure:
worstemptymeasure = empty
return timeelapsed
if (timeelapsed <= 0) or (iters == 0):
raise BadMeasure((timeelapsed, iters))
return (timeelapsed / iters, iters)
def bench(func, initfunc=None, TOPXP=21, MAXREPS=5, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", outputjson=False, jsonresultsfname="pyutil-benchutil-results.json", UNITS_PER_SECOND=1):
BSIZES = []
libs/pyutil/benchutil.py~
+28 -7
View File
@@ -1,4 +1,4 @@
# Copyright (c) 2002-2012 Zooko Wilcox-O'Hearn
# Copyright (c) 2002-2013 Zooko Wilcox-O'Hearn
# This file is part of pyutil; see README.rst for licensing terms.
"""
@@ -104,6 +104,24 @@ def makeg(func):
func()
return blah
def to_decimal(x):
"""
See if D(x) returns something. If instead it raises TypeError, x must have been a float, so convert it to Decimal by way of string. (In Python >= 2.7, D(x) does this automatically.
"""
try:
return D(x)
except TypeError:
return D("%0.54f" % (x,))
def mult(a, b):
"""
If we get TypeError from * (possibly because one is float and the other is Decimal), then promote them both to Decimal.
"""
try:
return a * b
except TypeError:
return to_decimal(a) * to_decimal(b)
def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, profresults="pyutil-benchutil.prof", UNITS_PER_SECOND=1, quiet=False):
"""
Will run the func up to MAXREPS times, but won't start a new run if MAXTIME
@@ -144,12 +162,12 @@ def rep_bench(func, n, initfunc=None, MAXREPS=10, MAXTIME=60.0, profile=False, p
# etc, so we use mp1 to name it.
mp1 = m+1
res = {
'worst': (worst*UNITS_PER_SECOND)/n,
'best': (best*UNITS_PER_SECOND)/n,
'worst': mult(worst, UNITS_PER_SECOND)/n,
'best': mult(best, UNITS_PER_SECOND)/n,
'mp1': mp1,
'mth-best': (mthbest*UNITS_PER_SECOND)/n,
'mth-worst': (mthworst*UNITS_PER_SECOND)/n,
'mean': (mean*UNITS_PER_SECOND)/n,
'mth-best': mult(mthbest, UNITS_PER_SECOND)/n,
'mth-worst': mult(mthworst, UNITS_PER_SECOND)/n,
'mean': mult(mean, UNITS_PER_SECOND)/n,
'num': len(tls),
}
@@ -178,7 +196,10 @@ class BadMeasure(Exception):
def do_nothing(n):
pass
def bench_it(func, n, profile=False, profresults="pyutil-benchutil.prof"):
def bench_it(func, n, runtime=0.1, profile=False, profresults="pyutil-benchutil.prof"):
"""
runtime is how many seconds to
"""
if profile:
st = clock()
cProfile.run('func(n)', profresults)
libs/pyutil/data/wordlist.txt
+7248
View File
File diff suppressed because it is too large Load Diff
libs/pyutil/fileutil.py~
-271
View File
@@ -1,271 +0,0 @@
# Copyright (c) 2002-2010 Zooko Wilcox-O'Hearn
# This file is part of pyutil; see README.rst for licensing terms.
"""
Futz with files like a pro.
"""
import errno, exceptions, os, stat, tempfile
try:
import bsddb
except ImportError:
DBNoSuchFileError = None
else:
DBNoSuchFileError = bsddb.db.DBNoSuchFileError
# read_file() and write_file() copied from Mark Seaborn's blog post. Please
# read it for complete rationale:
# http://lackingrhoticity.blogspot.com/2009/12/readfile-and-writefile-in-python.html
def read_file(filename, mode='rb'):
""" Read the contents of the file named filename and return it in
a string. This function closes the file handle before it returns
(even if the underlying Python implementation's garbage collector
doesn't). """
fh = open(filename, mode)
try:
return fh.read()
finally:
fh.close()
def write_file(filename, data, mode='wb'):
""" Write the string data into a file named filename. This
function closes the file handle (ensuring that the written data is
flushed from the perspective of the Python implementation) before
it returns (even if the underlying Python implementation's garbage
collector doesn't)."""
fh = open(filename, mode)
try:
fh.write(data)
finally:
fh.close()
# For backwards-compatibility in case someone is using these names. We used to
# have a superkludge in fileutil.py under these names.
def rename(src, dst, tries=4, basedelay=0.1):
return os.rename(src, dst)
def remove(f, tries=4, basedelay=0.1):
return os.remove(f)
def rmdir(f, tries=4, basedelay=0.1):
return os.rmdir(f)
class _Dir(object):
"""
Hold a set of files and subdirs and clean them all up when asked to.
"""
def __init__(self, name, cleanup=True):
self.name = name
self.cleanup = cleanup
self.files = []
self.subdirs = set()
def file(self, fname, mode=None):
"""
Create a file in the tempdir and remember it so as to close() it
before attempting to cleanup the temp dir.
@rtype: file
"""
ffn = os.path.join(self.name, fname)
if mode is not None:
fo = open(ffn, mode)
else:
fo = open(ffn)
self.register_file(fo)
return fo
def subdir(self, dirname):
"""
Create a subdirectory in the tempdir and remember it so as to call
shutdown() on it before attempting to clean up.
@rtype: _Dir instance
"""
ffn = os.path.join(self.name, dirname)
sd = _Dir(ffn, self.cleanup)
self.register_subdir(sd)
make_dirs(sd.name)
return sd
def register_file(self, fileobj):
"""
Remember the file object and call close() on it before attempting to
clean up.
"""
self.files.append(fileobj)
def register_subdir(self, dirobj):
"""
Remember the _Dir object and call shutdown() on it before attempting
to clean up.
"""
self.subdirs.add(dirobj)
def shutdown(self):
if self.cleanup:
for subdir in hasattr(self, 'subdirs') and self.subdirs or []:
subdir.shutdown()
for fileobj in hasattr(self, 'files') and self.files or []:
if DBNoSuchFileError is None:
fileobj.close() # "close()" is idempotent so we don't need to catch exceptions here
else:
try:
fileobj.close()
except DBNoSuchFileError:
# Ah, except that the bsddb module's file-like object (a DB object) has a non-idempotent close...
pass
if hasattr(self, 'name'):
rm_dir(self.name)
def __repr__(self):
return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name)
def __str__(self):
return self.__repr__()
def __del__(self):
try:
self.shutdown()
except:
import traceback
traceback.print_exc()
class NamedTemporaryDirectory(_Dir):
"""
Call tempfile.mkdtemp(), store the name of the dir in self.name, and
rm_dir() when it gets garbage collected or "shutdown()".
Also keep track of file objects for files within the tempdir and call
close() on them before rm_dir(). This is a convenient way to open temp
files within the directory, and it is very helpful on Windows because you
can't delete a directory which contains a file which is currently open.
"""
def __init__(self, cleanup=True, *args, **kwargs):
""" If cleanup, then the directory will be rmrf'ed when the object is shutdown. """
name = tempfile.mkdtemp(*args, **kwargs)
_Dir.__init__(self, name, cleanup)
class ReopenableNamedTemporaryFile:
"""
This uses tempfile.mkstemp() to generate a secure temp file. It then closes
the file, leaving a zero-length file as a placeholder. You can get the
filename with ReopenableNamedTemporaryFile.name. When the
ReopenableNamedTemporaryFile instance is garbage collected or its shutdown()
method is called, it deletes the file.
"""
def __init__(self, *args, **kwargs):
fd, self.name = tempfile.mkstemp(*args, **kwargs)
os.close(fd)
def __repr__(self):
return "<%s instance at %x %s>" % (self.__class__.__name__, id(self), self.name)
def __str__(self):
return self.__repr__()
def __del__(self):
self.shutdown()
def shutdown(self):
remove(self.name)
def make_dirs(dirname, mode=0777):
"""
An idempotent version of os.makedirs(). If the dir already exists, do
nothing and return without raising an exception. If this call creates the
dir, return without raising an exception. If there is an error that
prevents creation or if the directory gets deleted after make_dirs() creates
it and before make_dirs() checks that it exists, raise an exception.
"""
tx = None
try:
os.makedirs(dirname, mode)
except OSError, x:
tx = x
if not os.path.isdir(dirname):
if tx:
raise tx
raise exceptions.IOError, "unknown error prevented creation of directory, or deleted the directory immediately after creation: %s" % dirname # careful not to construct an IOError with a 2-tuple, as that has a special meaning...
def rmtree(dirname):
"""
A threadsafe and idempotent version of shutil.rmtree(). If the dir is
already gone, do nothing and return without raising an exception. If this
call removes the dir, return without raising an exception. If there is an
error that prevents deletion or if the directory gets created again after
rm_dir() deletes it and before rm_dir() checks that it is gone, raise an
exception.
"""
excs = []
try:
os.chmod(dirname, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD)
for f in os.listdir(dirname):
fullname = os.path.join(dirname, f)
if os.path.isdir(fullname):
rm_dir(fullname)
else:
remove(fullname)
os.rmdir(dirname)
except EnvironmentError, le:
# Ignore "No such file or directory", collect any other exception.
if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT):
excs.append(le)
except Exception, le:
excs.append(le)
# Okay, now we've recursively removed everything, ignoring any "No
# such file or directory" errors, and collecting any other errors.
if os.path.exists(dirname):
if len(excs) == 1:
raise excs[0]
if len(excs) == 0:
raise OSError, "Failed to remove dir for unknown reason."
raise OSError, excs
def rm_dir(dirname):
# Renamed to be like shutil.rmtree and unlike rmdir.
return rmtree(dirname)
def remove_if_possible(f):
try:
remove(f)
except EnvironmentError:
pass
def remove_if_present(f):
try:
remove(f)
except EnvironmentError, le:
# Ignore "No such file or directory", re-raise any other exception.
if (le.args[0] != 2 and le.args[0] != 3) or (le.args[0] != errno.ENOENT):
raise
def rmdir_if_possible(f):
try:
rmdir(f)
except EnvironmentError:
pass
def open_or_create(fname, binarymode=True):
try:
f = open(fname, binarymode and "r+b" or "r+")
except EnvironmentError:
f = open(fname, binarymode and "w+b" or "w+")
return f
def du(basedir):
size = 0
for root, dirs, files in os.walk(basedir):
for f in files:
fn = os.path.join(root, f)
size += os.path.getsize(fn)
return size
libs/pyutil/iputil.py
+65 -90
View File
@@ -1,22 +1,12 @@
# portions extracted from ipaddresslib by Autonomous Zone Industries, LGPL (author: Greg Smith)
# portions adapted from nattraverso.ipdiscover
# portions authored by Brian Warner, working for Allmydata
# most recent version authored by Zooko O'Whielacronx, working for Allmydata
# from the Python Standard Library
import os, re, socket, sys
import os, re, socket, sys, subprocess
# from Twisted
from twisted.internet import defer, reactor
from twisted.python import failure
from twisted.internet import defer, threads, reactor
from twisted.internet.protocol import DatagramProtocol
from twisted.internet.utils import getProcessOutput
from twisted.python.procutils import which
from twisted.python import log
# from pyutil
import observer
try:
import resource
def increase_rlimits():
@@ -77,6 +67,7 @@ except ImportError:
# since one might be shadowing the other. This hack appeases pyflakes.
increase_rlimits = _increase_rlimits
def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET
"""
Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad
@@ -121,14 +112,16 @@ def get_local_ip_for(target):
except socket.gaierror:
# DNS isn't running, or somehow we encountered an error
# note: if an interface is configured and up, but nothing is connected to it,
# gethostbyname("A.ROOT-SERVERS.NET") will take 20 seconds to raise socket.gaierror
# . This is synchronous and occurs for each node being started, so users of certain unit
# tests will see something like 120s of delay, which may be enough to hit the default
# trial timeouts. For that reason, get_local_addresses_async() was changed to default to
# the numerical ip address for A.ROOT-SERVERS.NET, to avoid this DNS lookup. This also
# makes node startup a tad faster.
# note: if an interface is configured and up, but nothing is
# connected to it, gethostbyname("A.ROOT-SERVERS.NET") will take 20
# seconds to raise socket.gaierror . This is synchronous and occurs
# for each node being started, so users of
# test.common.SystemTestMixin (like test_system) will see something
# like 120s of delay, which may be enough to hit the default trial
# timeouts. For that reason, get_local_addresses_async() was changed
# to default to the numerical ip address for A.ROOT-SERVERS.NET, to
# avoid this DNS lookup. This also makes node startup fractionally
# faster.
return None
udpprot = DatagramProtocol()
port = reactor.listenUDP(0, udpprot)
@@ -146,16 +139,29 @@ _platform_map = {
"linux-i386": "linux", # redhat
"linux-ppc": "linux", # redhat
"linux2": "linux", # debian
"linux3": "linux", # debian
"win32": "win32",
"irix6-n32": "irix",
"irix6-n64": "irix",
"irix6": "irix",
"openbsd2": "bsd",
"openbsd3": "bsd",
"openbsd4": "bsd",
"openbsd5": "bsd",
"darwin": "bsd", # Mac OS X
"freebsd4": "bsd",
"freebsd5": "bsd",
"freebsd6": "bsd",
"freebsd7": "bsd",
"freebsd8": "bsd",
"freebsd9": "bsd",
"netbsd1": "bsd",
"netbsd2": "bsd",
"netbsd3": "bsd",
"netbsd4": "bsd",
"netbsd5": "bsd",
"netbsd6": "bsd",
"dragonfly2": "bsd",
"sunos5": "sunos",
"cygwin": "cygwin",
}
@@ -173,12 +179,12 @@ _win32_re = re.compile('^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\
# These work in Redhat 6.x and Debian 2.2 potato
_linux_path = '/sbin/ifconfig'
_linux_re = re.compile('^\s*inet addr:(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S)
_linux_re = re.compile('^\s*inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S)
# originally NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X, FreeBSD, OpenBSD
_bsd_path = '/sbin/ifconfig'
_bsd_args = ('-a',)
_bsd_re = re.compile('^\s+inet (?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S)
# NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X
_netbsd_path = '/sbin/ifconfig'
_netbsd_args = ('-a',)
_netbsd_re = re.compile('^\s+inet [a-zA-Z]*:?(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S)
# Irix 6.5
_irix_path = '/usr/etc/ifconfig'
@@ -186,39 +192,6 @@ _irix_path = '/usr/etc/ifconfig'
# Solaris 2.x
_sunos_path = '/usr/sbin/ifconfig'
class SequentialTrier(object):
""" I hold a list of executables to try and try each one in turn
until one gives me a list of IP addresses."""
def __init__(self, exebasename, args, regex):
assert not os.path.isabs(exebasename)
self.exes_left_to_try = which(exebasename)
self.exes_left_to_try.reverse()
self.args = args
self.regex = regex
self.o = observer.OneShotObserverList()
self._try_next()
def _try_next(self):
if not self.exes_left_to_try:
self.o.fire(None)
else:
exe = self.exes_left_to_try.pop()
d2 = _query(exe, self.args, self.regex)
def cb(res):
if res:
self.o.fire(res)
else:
self._try_next()
def eb(why):
self._try_next()
d2.addCallbacks(cb, eb)
def when_tried(self):
return self.o.when_fired()
# k: platform string as provided in the value of _platform_map
# v: tuple of (path_to_tool, args, regex,)
@@ -226,19 +199,22 @@ _tool_map = {
"linux": (_linux_path, (), _linux_re,),
"win32": (_win32_path, _win32_args, _win32_re,),
"cygwin": (_win32_path, _win32_args, _win32_re,),
"bsd": (_bsd_path, _bsd_args, _bsd_re,),
"irix": (_irix_path, _bsd_args, _bsd_re,),
"sunos": (_sunos_path, _bsd_args, _bsd_re,),
"bsd": (_netbsd_path, _netbsd_args, _netbsd_re,),
"irix": (_irix_path, _netbsd_args, _netbsd_re,),
"sunos": (_sunos_path, _netbsd_args, _netbsd_re,),
}
def _find_addresses_via_config():
# originally by Greg Smith, hacked by Zooko to conform to Brian Warner's API.
return threads.deferToThread(_synchronously_find_addresses_via_config)
def _synchronously_find_addresses_via_config():
# originally by Greg Smith, hacked by Zooko to conform to Brian's API
platform = _platform_map.get(sys.platform)
(pathtotool, args, regex,) = _tool_map.get(platform, ('ifconfig', _bsd_args, _bsd_re,))
if not platform:
raise UnsupportedPlatformError(sys.platform)
# If the platform isn't known then we attempt BSD-style ifconfig. If it
# turns out that we don't get anything resembling a dotted quad IPv4 address
# out of it, then we'll raise UnsupportedPlatformError.
(pathtotool, args, regex,) = _tool_map[platform]
# If pathtotool is a fully qualified path then we just try that.
# If it is merely an executable name then we use Twisted's
@@ -246,34 +222,33 @@ def _find_addresses_via_config():
# gives us something that resembles a dotted-quad IPv4 address.
if os.path.isabs(pathtotool):
d = _query(pathtotool, args, regex)
return _query(pathtotool, args, regex)
else:
d = SequentialTrier(pathtotool, args, regex).when_tried()
d.addCallback(_check_result)
return d
def _check_result(result):
if not result and not _platform_map.has_key(sys.platform):
return failure.Failure(UnsupportedPlatformError(sys.platform))
else:
return result
exes_to_try = which(pathtotool)
for exe in exes_to_try:
try:
addresses = _query(exe, args, regex)
except Exception:
addresses = []
if addresses:
return addresses
return []
def _query(path, args, regex):
d = getProcessOutput(path, args)
def _parse(output):
addresses = []
outputsplit = output.split('\n')
for outline in outputsplit:
m = regex.match(outline)
if m:
addr = m.groupdict()['address']
if addr not in addresses:
addresses.append(addr)
env = {'LANG': 'en_US.UTF-8'}
p = subprocess.Popen([path] + list(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
(output, err) = p.communicate()
return addresses
d.addCallback(_parse)
return d
addresses = []
outputsplit = output.split('\n')
for outline in outputsplit:
m = regex.match(outline)
if m:
addr = m.groupdict()['address']
if addr not in addresses:
addresses.append(addr)
return addresses
def _cygwin_hack_find_addresses(target):
addresses = []
libs/pyutil/iputil.py~
+288
View File
@@ -0,0 +1,288 @@
# portions extracted from ipaddresslib by Autonomous Zone Industries, LGPL (author: Greg Smith)
# portions adapted from nattraverso.ipdiscover
# portions authored by Brian Warner, working for Allmydata
# most recent version authored by Zooko O'Whielacronx, working for Allmydata
# from the Python Standard Library
import os, re, socket, sys
# from Twisted
from twisted.internet import defer, reactor
from twisted.python import failure
from twisted.internet.protocol import DatagramProtocol
from twisted.internet.utils import getProcessOutput
from twisted.python.procutils import which
from twisted.python import log
# from pyutil
import observer
try:
import resource
def increase_rlimits():
# We'd like to raise our soft resource.RLIMIT_NOFILE, since certain
# systems (OS-X, probably solaris) start with a relatively low limit
# (256), and some unit tests want to open up more sockets than this.
# Most linux systems start with both hard and soft limits at 1024,
# which is plenty.
# unfortunately the values to pass to setrlimit() vary widely from
# one system to another. OS-X reports (256, HUGE), but the real hard
# limit is 10240, and accepts (-1,-1) to mean raise it to the
# maximum. Cygwin reports (256, -1), then ignores a request of
# (-1,-1): instead you have to guess at the hard limit (it appears to
# be 3200), so using (3200,-1) seems to work. Linux reports a
# sensible (1024,1024), then rejects (-1,-1) as trying to raise the
# maximum limit, so you could set it to (1024,1024) but you might as
# well leave it alone.
try:
current = resource.getrlimit(resource.RLIMIT_NOFILE)
except AttributeError:
# we're probably missing RLIMIT_NOFILE
return
if current[0] >= 1024:
# good enough, leave it alone
return
try:
if current[1] > 0 and current[1] < 1000000:
# solaris reports (256, 65536)
resource.setrlimit(resource.RLIMIT_NOFILE,
(current[1], current[1]))
else:
# this one works on OS-X (bsd), and gives us 10240, but
# it doesn't work on linux (on which both the hard and
# soft limits are set to 1024 by default).
resource.setrlimit(resource.RLIMIT_NOFILE, (-1,-1))
new = resource.getrlimit(resource.RLIMIT_NOFILE)
if new[0] == current[0]:
# probably cygwin, which ignores -1. Use a real value.
resource.setrlimit(resource.RLIMIT_NOFILE, (3200,-1))
except ValueError:
log.msg("unable to set RLIMIT_NOFILE: current value %s"
% (resource.getrlimit(resource.RLIMIT_NOFILE),))
except:
# who knows what. It isn't very important, so log it and continue
log.err()
except ImportError:
def _increase_rlimits():
# TODO: implement this for Windows. Although I suspect the
# solution might be "be running under the iocp reactor and
# make this function be a no-op".
pass
# pyflakes complains about two 'def FOO' statements in the same time,
# since one might be shadowing the other. This hack appeases pyflakes.
increase_rlimits = _increase_rlimits
def get_local_addresses_async(target="198.41.0.4"): # A.ROOT-SERVERS.NET
"""
Return a Deferred that fires with a list of IPv4 addresses (as dotted-quad
strings) that are currently configured on this host, sorted in descending
order of how likely we think they are to work.
@param target: we want to learn an IP address they could try using to
connect to us; The default value is fine, but it might help if you
pass the address of a host that you are actually trying to be
reachable to.
"""
addresses = []
local_ip = get_local_ip_for(target)
if local_ip:
addresses.append(local_ip)
if sys.platform == "cygwin":
d = _cygwin_hack_find_addresses(target)
else:
d = _find_addresses_via_config()
def _collect(res):
for addr in res:
if addr != "0.0.0.0" and not addr in addresses:
addresses.append(addr)
return addresses
d.addCallback(_collect)
return d
def get_local_ip_for(target):
"""Find out what our IP address is for use by a given target.
@return: the IP address as a dotted-quad string which could be used by
to connect to us. It might work for them, it might not. If
there is no suitable address (perhaps we don't currently have an
externally-visible interface), this will return None.
"""
try:
target_ipaddr = socket.gethostbyname(target)
except socket.gaierror:
# DNS isn't running, or somehow we encountered an error
# note: if an interface is configured and up, but nothing is connected to it,
# gethostbyname("A.ROOT-SERVERS.NET") will take 20 seconds to raise socket.gaierror
# . This is synchronous and occurs for each node being started, so users of certain unit
# tests will see something like 120s of delay, which may be enough to hit the default
# trial timeouts. For that reason, get_local_addresses_async() was changed to default to
# the numerical ip address for A.ROOT-SERVERS.NET, to avoid this DNS lookup. This also
# makes node startup a tad faster.
return None
udpprot = DatagramProtocol()
port = reactor.listenUDP(0, udpprot)
try:
udpprot.transport.connect(target_ipaddr, 7)
localip = udpprot.transport.getHost().host
except socket.error:
# no route to that host
localip = None
port.stopListening() # note, this returns a Deferred
return localip
# k: result of sys.platform, v: which kind of IP configuration reader we use
_platform_map = {
"linux-i386": "linux", # redhat
"linux-ppc": "linux", # redhat
"linux2": "linux", # debian
"win32": "win32",
"irix6-n32": "irix",
"irix6-n64": "irix",
"irix6": "irix",
"openbsd2": "bsd",
"darwin": "bsd", # Mac OS X
"freebsd4": "bsd",
"freebsd5": "bsd",
"freebsd6": "bsd",
"netbsd1": "bsd",
"sunos5": "sunos",
"cygwin": "cygwin",
}
class UnsupportedPlatformError(Exception):
pass
# Wow, I'm really amazed at home much mileage we've gotten out of calling
# the external route.exe program on windows... It appears to work on all
# versions so far. Still, the real system calls would much be preferred...
# ... thus wrote Greg Smith in time immemorial...
_win32_path = 'route.exe'
_win32_args = ('print',)
_win32_re = re.compile('^\s*\d+\.\d+\.\d+\.\d+\s.+\s(?P<address>\d+\.\d+\.\d+\.\d+)\s+(?P<metric>\d+)\s*$', flags=re.M|re.I|re.S)
# These work in Redhat 6.x and Debian 2.2 potato
_linux_path = '/sbin/ifconfig'
_linux_re = re.compile('^\s*inet addr:(?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S)
# originally NetBSD 1.4 (submitted by Rhialto), Darwin, Mac OS X, FreeBSD, OpenBSD
_bsd_path = '/sbin/ifconfig'
_bsd_args = ('-a',)
_bsd_re = re.compile('^\s+inet (?P<address>\d+\.\d+\.\d+\.\d+)\s.+$', flags=re.M|re.I|re.S)
# Irix 6.5
_irix_path = '/usr/etc/ifconfig'
# Solaris 2.x
_sunos_path = '/usr/sbin/ifconfig'
class SequentialTrier(object):
""" I hold a list of executables to try and try each one in turn
until one gives me a list of IP addresses."""
def __init__(self, exebasename, args, regex):
assert not os.path.isabs(exebasename)
self.exes_left_to_try = which(exebasename)
self.exes_left_to_try.reverse()
self.args = args
self.regex = regex
self.o = observer.OneShotObserverList()
self._try_next()
def _try_next(self):
if not self.exes_left_to_try:
self.o.fire(None)
else:
exe = self.exes_left_to_try.pop()
d2 = _query(exe, self.args, self.regex)
def cb(res):
if res:
self.o.fire(res)
else:
self._try_next()
def eb(why):
self._try_next()
d2.addCallbacks(cb, eb)
def when_tried(self):
return self.o.when_fired()
# k: platform string as provided in the value of _platform_map
# v: tuple of (path_to_tool, args, regex,)
_tool_map = {
"linux": (_linux_path, (), _linux_re,),
"win32": (_win32_path, _win32_args, _win32_re,),
"cygwin": (_win32_path, _win32_args, _win32_re,),
"bsd": (_bsd_path, _bsd_args, _bsd_re,),
"irix": (_irix_path, _bsd_args, _bsd_re,),
"sunos": (_sunos_path, _bsd_args, _bsd_re,),
}
def _find_addresses_via_config():
# originally by Greg Smith, hacked by Zooko to conform to Brian Warner's API.
platform = _platform_map.get(sys.platform)
(pathtotool, args, regex,) = _tool_map.get(platform, ('ifconfig', _bsd_args, _bsd_re,))
# If the platform isn't known then we attempt BSD-style ifconfig. If it
# turns out that we don't get anything resembling a dotted quad IPv4 address
# out of it, then we'll raise UnsupportedPlatformError.
# If pathtotool is a fully qualified path then we just try that.
# If it is merely an executable name then we use Twisted's
# "which()" utility and try each executable in turn until one
# gives us something that resembles a dotted-quad IPv4 address.
if os.path.isabs(pathtotool):
d = _query(pathtotool, args, regex)
else:
d = SequentialTrier(pathtotool, args, regex).when_tried()
d.addCallback(_check_result)
return d
def _check_result(result):
if not result and not _platform_map.has_key(sys.platform):
return failure.Failure(UnsupportedPlatformError(sys.platform))
else:
return result
def _query(path, args, regex):
d = getProcessOutput(path, args)
def _parse(output):
addresses = []
outputsplit = output.split('\n')
for outline in outputsplit:
m = regex.match(outline)
if m:
addr = m.groupdict()['address']
if addr not in addresses:
addresses.append(addr)
return addresses
d.addCallback(_parse)
return d
def _cygwin_hack_find_addresses(target):
addresses = []
for h in [target, "localhost", "127.0.0.1",]:
try:
addr = get_local_ip_for(h)
if addr not in addresses:
addresses.append(addr)
except socket.gaierror:
pass
return defer.succeed(addresses)
libs/pyutil/mathutil.py
+1 -1
View File
@@ -11,7 +11,7 @@ def div_ceil(n, d):
"""
The smallest integer k such that k*d >= n.
"""
return (n/d) + (n%d != 0)
return int((n//d) + (n%d != 0))
def next_multiple(n, k):
"""
libs/pyutil/mathutil.py~
+106
View File
@@ -0,0 +1,106 @@
# Copyright (c) 2005-2010 Zooko Wilcox-O'Hearn
# This file is part of pyutil; see README.rst for licensing terms.
"""
A few commonly needed functions.
"""
import math
def div_ceil(n, d):
"""
The smallest integer k such that k*d >= n.
"""
return (n/d) + (n%d != 0)
def next_multiple(n, k):
"""
The smallest multiple of k which is >= n. Note that if n is 0 then the
answer is 0.
"""
return div_ceil(n, k) * k
def pad_size(n, k):
"""
The smallest number that has to be added to n to equal a multiple of k.
"""
if n%k:
return k - n%k
else:
return 0
def is_power_of_k(n, k):
return k**int(math.log(n, k) + 0.5) == n
def next_power_of_k(n, k):
p = 1
while p < n:
p *= k
return p
def ave(l):
return sum(l) / len(l)
def log_ceil(n, b):
"""
The smallest integer k such that b^k >= n.
log_ceil(n, 2) is the number of bits needed to store any of n values, e.g.
the number of bits needed to store any of 128 possible values is 7.
"""
p = 1
k = 0
while p < n:
p *= b
k += 1
return k
def log_floor(n, b):
"""
The largest integer k such that b^k <= n.
"""
p = 1
k = 0
while p <= n:
p *= b
k += 1
return k - 1
def linear_fit_slope(ps):
"""
Single-independent-variable linear regression -- least squares method.
At least, I *think* this function computes that answer. I no longer
remember where I learned this trick and at the moment I can't prove to
myself that this is correct.
@param ps a sequence of tuples of (x, y)
"""
avex = ave([x for (x, y) in ps])
avey = ave([y for (x, y) in ps])
sxy = sum([ (x - avex) * (y - avey) for (x, y) in ps ])
sxx = sum([ (x - avex) ** 2 for (x, y) in ps ])
if sxx == 0:
return None
return sxy / sxx
def permute(l):
"""
Return all possible permutations of l.
@type l: sequence
@rtype a set of sequences
"""
if len(l) == 1:
return [l,]
res = []
for i in range(len(l)):
l2 = list(l[:])
x = l2.pop(i)
for l3 in permute(l2):
l3.append(x)
res.append(l3)
return res
libs/pyutil/odict.py~
-552
View File
@@ -1,552 +0,0 @@
# Copyright (c) 2002-2009 Zooko "Zooko" Wilcox-O'Hearn
"""
This module offers a Ordered Dict, which is a dict that preserves
insertion order. See PEP 372 for description of the problem. This
implementation uses a linked-list to get good O(1) asymptotic
performance. (Actually it is O(hashtable-update-cost), but whatever.)
Warning: if -O optimizations are not turned on then OrderedDict performs
extensive self-analysis in every function call, which can take minutes
and minutes for a large cache. Turn on -O, or comment out assert
self._assert_invariants()
"""
import operator
from assertutil import _assert, precondition
from humanreadable import hr
class OrderedDict:
"""
An efficient ordered dict.
Adding an item that is already in the dict *does not* make it the
most- recently-added item although it may change the state of the
dict itself (if the value is different than the previous value).
See also SmallOrderedDict (below), which is faster in some cases.
"""
class ItemIterator:
def __init__(self, c):
self.c = c
self.i = c.d[c.ts][1]
def __iter__(self):
return self
def next(self):
if self.i is self.c.hs:
raise StopIteration
k = self.i
precondition(self.c.d.has_key(k), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", k, self.c)
(v, p, n,) = self.c.d[k]
self.i = p
return (k, v,)
class KeyIterator:
def __init__(self, c):
self.c = c
self.i = c.d[c.ts][1]
def __iter__(self):
return self
def next(self):
if self.i is self.c.hs:
raise StopIteration
k = self.i
precondition(self.c.d.has_key(k), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", k, self.c)
(v, p, n,) = self.c.d[k]
self.i = p
return k
class ValIterator:
def __init__(self, c):
self.c = c
self.i = c.d[c.ts][1]
def __iter__(self):
return self
def next(self):
if self.i is self.c.hs:
raise StopIteration
precondition(self.c.d.has_key(self.i), "The iterated OrderedDict doesn't have the next key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c)
(v, p, n,) = self.c.d[self.i]
self.i = p
return v
class Sentinel:
def __init__(self, msg):
self.msg = msg
def __repr__(self):
return "<%s %s>" % (self.__class__.__name__, self.msg,)
def __init__(self, initialdata={}):
self.d = {} # k: k, v: [v, prev, next,] # the dict
self.hs = OrderedDict.Sentinel("hs")
self.ts = OrderedDict.Sentinel("ts")
self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes.
self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes.
self.update(initialdata)
assert self._assert_invariants()
def __repr_n__(self, n=None):
s = ["{",]
try:
iter = self.iteritems()
x = iter.next()
s.append(str(x[0])); s.append(": "); s.append(str(x[1]))
i = 1
while (n is None) or (i < n):
x = iter.next()
s.append(", "); s.append(str(x[0])); s.append(": "); s.append(str(x[1]))
except StopIteration:
pass
s.append("}")
return ''.join(s)
def __repr__(self):
return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(),)
def __str__(self):
return "<%s %s>" % (self.__class__.__name__, self.__repr_n__(16),)
def _assert_invariants(self):
_assert((len(self.d) > 2) == (self.d[self.hs][2] is not self.ts) == (self.d[self.ts][1] is not self.hs), "Head and tail point to something other than each other if and only if there is at least one element in the dictionary.", self.hs, self.ts, len(self.d))
foundprevsentinel = 0
foundnextsentinel = 0
for (k, (v, p, n,)) in self.d.iteritems():
_assert(v not in (self.hs, self.ts,))
_assert(p is not self.ts, "A reference to the tail sentinel may not appear in prev.", k, v, p, n)
_assert(n is not self.hs, "A reference to the head sentinel may not appear in next.", k, v, p, n)
_assert(p in self.d, "Each prev is required to appear as a key in the dict.", k, v, p, n)
_assert(n in self.d, "Each next is required to appear as a key in the dict.", k, v, p, n)
if p is self.hs:
foundprevsentinel += 1
_assert(foundprevsentinel <= 2, "No more than two references to the head sentinel may appear as a prev.", k, v, p, n)
if n is self.ts:
foundnextsentinel += 1
_assert(foundnextsentinel <= 2, "No more than one reference to the tail sentinel may appear as a next.", k, v, p, n)
_assert(foundprevsentinel == 2, "A reference to the head sentinel is required appear as a prev (plus a self-referential reference).")
_assert(foundnextsentinel == 2, "A reference to the tail sentinel is required appear as a next (plus a self-referential reference).")
count = 0
for (k, v,) in self.iteritems():
_assert(k not in (self.hs, self.ts,), k, self.hs, self.ts)
count += 1
_assert(count == len(self.d)-2, count, len(self.d)) # -2 for the sentinels
return True
def move_to_most_recent(self, k, strictkey=False):
assert self._assert_invariants()
if not self.d.has_key(k):
if strictkey:
raise KeyError, k
return
node = self.d[k]
# relink
self.d[node[1]][2] = node[2]
self.d[node[2]][1] = node[1]
# move to front
hnode = self.d[self.hs]
node[1] = self.hs
node[2] = hnode[2]
hnode[2] = k
self.d[node[2]][1] = k
assert self._assert_invariants()
def iteritems(self):
return OrderedDict.ItemIterator(self)
def itervalues(self):
return OrderedDict.ValIterator(self)
def iterkeys(self):
return self.__iter__()
def __iter__(self):
return OrderedDict.KeyIterator(self)
def __getitem__(self, key, default=None, strictkey=True):
node = self.d.get(key)
if not node:
if strictkey:
raise KeyError, key
return default
return node[0]
def __setitem__(self, k, v=None):
assert self._assert_invariants()
node = self.d.get(k)
if node:
node[0] = v
return
hnode = self.d[self.hs]
n = hnode[2]
self.d[k] = [v, self.hs, n,]
hnode[2] = k
self.d[n][1] = k
assert self._assert_invariants()
return v
def __delitem__(self, key, default=None, strictkey=True):
"""
@param strictkey: True if you want a KeyError in the case that
key is not there, False if you want a reference to default
in the case that key is not there
@param default: the object to return if key is not there; This
is ignored if strictkey.
@return: the value removed or default if there is not item by
that key and strictkey is False
"""
assert self._assert_invariants()
if self.d.has_key(key):
node = self.d[key]
# relink
self.d[node[1]][2] = node[2]
self.d[node[2]][1] = node[1]
del self.d[key]
assert self._assert_invariants()
return node[0]
elif strictkey:
assert self._assert_invariants()
raise KeyError, key
else:
assert self._assert_invariants()
return default
def has_key(self, key):
assert self._assert_invariants()
if self.d.has_key(key):
assert self._assert_invariants()
return True
else:
assert self._assert_invariants()
return False
def clear(self):
assert self._assert_invariants()
self.d.clear()
self.d[self.hs] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes.
self.d[self.ts] = [None, self.hs, self.ts,] # This allows us to use sentinels as normal nodes.
assert self._assert_invariants()
def update(self, otherdict):
"""
@return: self
"""
assert self._assert_invariants()
for (k, v,) in otherdict.iteritems():
assert self._assert_invariants()
self[k] = v
assert self._assert_invariants()
def pop(self):
assert self._assert_invariants()
if len(self.d) < 2: # the +2 is for the sentinels
raise KeyError, 'popitem(): dictionary is empty'
k = self.d[self.hs][2]
self.remove(k)
assert self._assert_invariants()
return k
def popitem(self):
assert self._assert_invariants()
if len(self.d) < 2: # the +2 is for the sentinels
raise KeyError, 'popitem(): dictionary is empty'
k = self.d[self.hs][2]
val = self.remove(k)
assert self._assert_invariants()
return (k, val,)
def keys_unsorted(self):
assert self._assert_invariants()
t = self.d.copy()
del t[self.hs]
del t[self.ts]
assert self._assert_invariants()
return t.keys()
def keys(self):
res = [None] * len(self)
i = 0
for k in self.iterkeys():
res[i] = k
i += 1
return res
def values_unsorted(self):
assert self._assert_invariants()
t = self.d.copy()
del t[self.hs]
del t[self.ts]
assert self._assert_invariants()
return map(operator.__getitem__, t.values(), [0]*len(t))
def values(self):
res = [None] * len(self)
i = 0
for v in self.itervalues():
res[i] = v
i += 1
return res
def items(self):
res = [None] * len(self)
i = 0
for it in self.iteritems():
res[i] = it
i += 1
return res
def __len__(self):
return len(self.d) - 2
def insert(self, key, val=None):
assert self._assert_invariants()
result = self.__setitem__(key, val)
assert self._assert_invariants()
return result
def setdefault(self, key, default=None):
assert self._assert_invariants()
if not self.has_key(key):
self[key] = default
assert self._assert_invariants()
return self[key]
def get(self, key, default=None):
return self.__getitem__(key, default, strictkey=False)
def remove(self, key, default=None, strictkey=True):
assert self._assert_invariants()
result = self.__delitem__(key, default, strictkey)
assert self._assert_invariants()
return result
class SmallOrderedDict(dict):
"""
SmallOrderedDict is faster than OrderedDict for small sets. How small? That
depends on your machine and which operations you use most often. Use
performance profiling to determine whether the cache class that you are
using makes any difference to the performance of your program, and if it
does, then run "quick_bench()" in test/test_cache.py to see which cache
implementation is faster for the size of your datasets.
A simple least-recently-used cache. It keeps an LRU queue, and
when the number of items in the cache reaches maxsize, it removes
the least recently used item.
"Looking" at an item or a key such as with "has_key()" makes that
item become the most recently used item.
You can also use "refresh()" to explicitly make an item become the most
recently used item.
Adding an item that is already in the dict *does* make it the
most- recently-used item although it does not change the state of
the dict itself.
"""
class ItemIterator:
def __init__(self, c):
self.c = c
self.i = 0
def __iter__(self):
return self
def next(self):
precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c)
precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c)
if self.i == len(self.c._lru):
raise StopIteration
k = self.i
self.i += 1
return (k, dict.__getitem__(self.c, k),)
class KeyIterator:
def __init__(self, c):
self.c = c
self.i = 0
def __iter__(self):
return self
def next(self):
precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c)
precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c)
if self.i == len(self.c._lru):
raise StopIteration
k = self.i
self.i += 1
return k
class ValueIterator:
def __init__(self, c):
self.c = c
self.i = 0
def __iter__(self):
return self
def next(self):
precondition(self.i <= len(self.c._lru), "The iterated SmallOrderedDict doesn't have this many elements. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c)
precondition(dict.has_key(self.c, self.c._lru[self.i]), "The iterated SmallOrderedDict doesn't have this key. Most likely this is because someone altered the contents of the OrderedDict while the iteration was in progress.", self.i, self.c._lru[self.i], self.c)
if self.i == len(self.c._lru):
raise StopIteration
k = self.i
self.i += 1
return dict.__getitem__(self.c, k)
def __init__(self, initialdata={}, maxsize=128):
dict.__init__(self, initialdata)
self._lru = initialdata.keys() # contains keys
self._maxsize = maxsize
over = len(self) - self._maxsize
if over > 0:
map(dict.__delitem__, [self]*over, self._lru[:over])
del self._lru[:over]
assert self._assert_invariants()
def _assert_invariants(self):
_assert(len(self._lru) <= self._maxsize, "Size is required to be <= maxsize.")
_assert(len(filter(lambda x: dict.has_key(self, x), self._lru)) == len(self._lru), "Each key in self._lru is required to be in dict.", filter(lambda x: not dict.has_key(self, x), self._lru), len(self._lru), self._lru, len(self), self)
_assert(len(filter(lambda x: x in self._lru, self.keys())) == len(self), "Each key in dict is required to be in self._lru.", filter(lambda x: x not in self._lru, self.keys()), len(self._lru), self._lru, len(self), self)
_assert(len(self._lru) == len(self), "internal consistency", filter(lambda x: x not in self.keys(), self._lru), len(self._lru), self._lru, len(self), self)
_assert(len(self._lru) <= self._maxsize, "internal consistency", len(self._lru), self._lru, self._maxsize)
return True
def insert(self, key, item=None):
assert self._assert_invariants()
result = self.__setitem__(key, item)
assert self._assert_invariants()
return result
def setdefault(self, key, default=None):
assert self._assert_invariants()
if not self.has_key(key):
self[key] = default
assert self._assert_invariants()
return self[key]
def __setitem__(self, key, item=None):
assert self._assert_invariants()
if dict.has_key(self, key):
self._lru.remove(key)
else:
if len(self._lru) == self._maxsize:
# If this insert is going to increase the size of the cache to bigger than maxsize:
killkey = self._lru.pop(0)
dict.__delitem__(self, killkey)
dict.__setitem__(self, key, item)
self._lru.append(key)
assert self._assert_invariants()
return item
def remove(self, key, default=None, strictkey=True):
assert self._assert_invariants()
result = self.__delitem__(key, default, strictkey)
assert self._assert_invariants()
return result
def __delitem__(self, key, default=None, strictkey=True):
"""
@param strictkey: True if you want a KeyError in the case that
key is not there, False if you want a reference to default
in the case that key is not there
@param default: the object to return if key is not there; This
is ignored if strictkey.
@return: the object removed or default if there is not item by
that key and strictkey is False
"""
assert self._assert_invariants()
if dict.has_key(self, key):
val = dict.__getitem__(self, key)
dict.__delitem__(self, key)
self._lru.remove(key)
assert self._assert_invariants()
return val
elif strictkey:
assert self._assert_invariants()
raise KeyError, key
else:
assert self._assert_invariants()
return default
def clear(self):
assert self._assert_invariants()
dict.clear(self)
self._lru = []
assert self._assert_invariants()
def update(self, otherdict):
"""
@return: self
"""
assert self._assert_invariants()
if len(otherdict) > self._maxsize:
# Handling this special case here makes it possible to implement the
# other more common cases faster below.
dict.clear(self)
self._lru = []
if self._maxsize > (len(otherdict) - self._maxsize):
dict.update(self, otherdict)
while len(self) > self._maxsize:
dict.popitem(self)
else:
for k, v, in otherdict.iteritems():
if len(self) == self._maxsize:
break
dict.__setitem__(self, k, v)
self._lru = dict.keys(self)
assert self._assert_invariants()
return self
for k in otherdict.iterkeys():
if dict.has_key(self, k):
self._lru.remove(k)
self._lru.extend(otherdict.keys())
dict.update(self, otherdict)
over = len(self) - self._maxsize
if over > 0:
map(dict.__delitem__, [self]*over, self._lru[:over])
del self._lru[:over]
assert self._assert_invariants()
return self
def has_key(self, key):
assert self._assert_invariants()
if dict.has_key(self, key):
assert key in self._lru, "key: %s, self._lru: %s" % tuple(map(hr, (key, self._lru,)))
self._lru.remove(key)
self._lru.append(key)
assert self._assert_invariants()
return True
else:
assert self._assert_invariants()
return False
def refresh(self, key, strictkey=True):
"""
@param strictkey: raise a KeyError exception if key isn't present
"""
assert self._assert_invariants()
if not dict.has_key(self, key):
if strictkey:
raise KeyError, key
return
self._lru.remove(key)
self._lru.append(key)
def popitem(self):
if not self._lru:
raise KeyError, 'popitem(): dictionary is empty'
k = self._lru[-1]
obj = self.remove(k)
return (k, obj,)
libs/pyutil/randutil.py
+1 -2
View File
@@ -80,6 +80,5 @@ seed = randobj.seed
def randstr(n):
return ''.join(map(chr, map(randrange, [0]*n, [256]*n)))
import random as insecurerandom
def insecurerandstr(n):
return ''.join(map(chr, map(insecurerandom.randrange, [0]*n, [256]*n)))
return os.urandom(n)
libs/pyutil/randutil.py~
-85
View File
@@ -1,85 +0,0 @@
# Copyright (c) 2002-2010 Zooko Wilcox-O'Hearn
# This file is part of pyutil; see README.rst for licensing terms.
import warnings
import os, random
try:
import hashexpand
class SHA256Random(hashexpand.SHA256Expander, random.Random):
def __init__(self, seed=None, deterministic=True):
warnings.warn("deprecated", DeprecationWarning)
if not deterministic:
raise NotImplementedError, "SHA256Expander is always deterministic. For non-deterministic, try urandomRandom."
hashexpand.SHA256Expander.__init__(self)
random.Random.__init__(self, seed)
self.seed(seed)
def seed(self, seed=None):
if seed is None:
import increasing_timer
seed = repr(increasing_timer.time())
hashexpand.SHA256Expander.seed(self, seed)
class SHA256Random(hashexpand.SHA256Expander, random.Random):
def __init__(self, seed=""):
warnings.warn("deprecated", DeprecationWarning)
hashexpand.SHA256Expander.__init__(self)
self.seed(seed)
def seed(self, seed=None):
if seed is None:
seed = os.urandom(32)
hashexpand.SHA256Expander.seed(self, seed)
except ImportError, le:
class InsecureSHA256Random:
def __init__(self, seed=None):
raise ImportError, le
class SHA256Random:
def __init__(self, seed=""):
raise ImportError, le
class devrandomRandom(random.Random):
""" The problem with using this one, of course, is that it blocks. This
is, of course, a security flaw. (On Linux and probably on other
systems.) --Zooko 2005-03-04
Not repeatable.
"""
def __init__(self):
warnings.warn("deprecated", DeprecationWarning)
self.dr = open("/dev/random", "r")
def get(self, bytes):
return self.dr.read(bytes)
class devurandomRandom(random.Random):
""" The problem with using this one is that it gives answers even when it
has never been properly seeded, e.g. when you are booting from CD and have
just started up and haven't yet gathered enough entropy to actually be
unguessable. (On Linux and probably on other systems.) --Zooko 2005-03-04
Not repeatable.
"""
def get(self, bytes):
warnings.warn("deprecated", DeprecationWarning)
return os.urandom(bytes)
randobj = devurandomRandom()
get = randobj.get
random = randobj.random
randrange = randobj.randrange
shuffle = randobj.shuffle
choice = randobj.choice
seed = randobj.seed
def randstr(n):
return ''.join(map(chr, map(randrange, [0]*n, [256]*n)))
import random as insecurerandom
def insecurerandstr(n):
return ''.join(map(chr, map(insecurerandom.randrange, [0]*n, [256]*n)))
libs/pyutil/scripts/passphrase.py
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import argparse, math, random
from pyutil.mathutil import div_ceil
from pkg_resources import resource_stream
def recursive_subset_sum(entropy_needed, wordlists):
# Pick a minimalish set of numbers which sum to at least
# entropy_needed.
# Okay now what's the smallest number of words which will give us
# at least this much entropy?
entropy_of_biggest_wordlist = wordlists[-1][0]
assert isinstance(entropy_of_biggest_wordlist, float), wordlists[-1]
needed_words = div_ceil(entropy_needed, entropy_of_biggest_wordlist)
# How much entropy do we need from each word?
needed_entropy_per_word = entropy_needed / needed_words
# What's the smallest wordlist that offers at least this much
# entropy per word?
for (wlentropy, wl) in wordlists:
if wlentropy >= needed_entropy_per_word:
break
assert wlentropy >= needed_entropy_per_word, (wlentropy, needed_entropy_per_word)
result = [(wlentropy, wl)]
# If we need more, recurse...
if wlentropy < entropy_needed:
rest = recursive_subset_sum(entropy_needed - wlentropy, wordlists)
result.extend(rest)
return result
def gen_passphrase(entropy, allwords):
maxlenwords = []
i = 2 # The smallest set is words of length 1 or 2.
words = [x for x in allwords if len(x) <= i]
maxlenwords.append((math.log(len(words), 2), words))
while len(maxlenwords[-1][1]) < len(allwords):
i += 1
words = [x for x in allwords if len(x) <= i]
maxlenwords.append((math.log(len(words), 2), words))
sr = random.SystemRandom()
passphrase = []
wordlists_to_use = recursive_subset_sum(entropy, maxlenwords)
passphraseentropy = 0.0
for (wle, wl) in wordlists_to_use:
passphrase.append(sr.choice(wl))
passphraseentropy += wle
return (u".".join(passphrase), passphraseentropy)
def main():
parser = argparse.ArgumentParser(prog="chbs", description="Create a random passphrase by picking a few random words.")
parser.add_argument('-d', '--dictionary', help="what file to read a list of words from (or omit this option to use chbs's bundled dictionary)", type=argparse.FileType('rU'), metavar="DICT")
parser.add_argument('bits', help="how many bits of entropy minimum", type=float, metavar="BITS")
args = parser.parse_args()
dicti = args.dictionary
if not dicti:
dicti = resource_stream('pyutil', 'data/wordlist.txt')
allwords = set([x.decode('utf-8').strip().lower() for x in dicti.readlines()])
passphrase, bits = gen_passphrase(args.bits, allwords)
print u"Your new password is: '%s'. It is worth about %s bits." % (passphrase, bits)
libs/pyutil/scripts/time_comparisons.py
+209
View File
@@ -0,0 +1,209 @@
# If you run this file, it will make up a random secret and then crack it
# using timing information from a string comparison function. Maybe--if it
# gets lucky. It takes a long, long time to work.
# So, the thing I need help with is statistics. The way this thing works is
# extremely stupid. Suppose you want to know which function invocation takes
# longer: comparison(secret, guess1) or comparison(secret, guess2)?
# If you can correctly determine that one of them takes longer than the
# other, then (a) you can use that to crack the secret, and (b) this is a
# unit test demonstrating that comparison() is not timing-safe.
# So how does this script do it? Extremely stupidly. First of all, you can't
# reliably measure tiny times, so to measure the time that a function takes,
# we run that function 10,000 times in a row, measure how long that took, and
# divide by 10,000 to estimate how long any one run would have taken.
# Then, we do that 100 times in a row, and take the fastest of 100 runs. (I
# also experimented with taking the mean of 100 runs instead of the fastest.)
# Then, we just say whichever comparison took longer (for its fastest run of
# 100 runs of 10,000 executions per run) is the one we think is a closer
# guess to the secret.
# Now I would *like* to think that there is some kind of statistical analysis
# more sophisticated than "take the slowest of the fastest of 100 runs of
# 10,000 executions". Such improved statistical analysis would hopefully be
# able to answer these two questions:
# 1. Are these two function calls -- comparison(secret, guess1) and
# comparison(secret, guess2) -- drawing from the same distribution or
# different? If you can answer that question, then you've answered the
# question of whether "comparison" is timing-safe or not.
# And, this would also allow the cracker to recover from a false step. If it
# incorrectly decides the the prefix of the secret is ABCX, when the real
# secret is ABCD, then after that every next step it takes will be the
# "drawing from the same distribution" kind -- any difference between ABCXQ
# and ABCXR will be just due to noise, since both are equally far from the
# correct answer, which startsw with ABCD. If it could realize that there is
# no real difference between the distributions, then it could back-track and
# recover.
# 2. Giving the ability to measure, noisily, the time taken by comparison(),
# how can you most efficiently figure out which guess takes the longest? If
# you can do that more efficiently, you can crack secrets more efficiently.
# The script takes two arguments. The first is how many symbols in the
# secret, and the second is how big the alphabet from which the symbols are
# drawn. To prove that this script can *ever* work, try passing length 5 and
# alphabet size 2. Also try editing the code to let is use sillycomp. That'll
# definitely make it work. If you can improve this script (as per the thing
# above about "needing better statistics") to the degree that it can crack a
# secret with length 32 and alphabet size 256, then that would be awesome.
# See the result of this commandline:
# $ python -c 'import time_comparisons ; time_comparisons.print_measurements()'
from pyutil import benchutil
import hashlib, random, os
from decimal import Decimal
D=Decimal
p1 = 'a'*32
p1a = 'a'*32
p2 = 'a'*31+'b' # close, but no cigar
p3 = 'b'*32 # different in the first byte
def randstr(n, alphabetsize):
alphabet = [ chr(x) for x in range(alphabetsize) ]
return ''.join([random.choice(alphabet) for i in range(n)])
def compare(n, f, a, b):
for i in xrange(n):
f(a, b)
def eqeqcomp(a, b):
return a == b
def sillycomp(a, b):
# This exposes a lot of information in its timing about how many leading bytes match.
for i in range(len(a)):
if a[i] != b[i]:
return False
for i in xrange(2**9):
pass
if len(a) == len(b):
return True
else:
return False
def hashcomp(a, b):
# Brian Warner invented this for Tahoe-LAFS. It seems like it should be very safe agaist timing leakage of any kind, because of the inclusion of a new random randkey every time. Note that exposing the value of the hash (i.e. the output of md5(randkey+secret)) is *not* a security problem. You can post that on your web site and let all attackers have it, no problem. (Provided that the value of "randkey" remains secret.)
randkey = os.urandom(32)
return hashlib.md5(randkey+ a).digest() == hashlib.md5(randkey+b).digest()
def xorcomp(a, b):
# This appears to be the most popular timing-insensitive string comparison function. I'm not completely sure it is fully timing-insensitive. (There are all sorts of funny things inside Python, such as caching of integer objects < 100...)
if len(a) != len(b):
return False
result = 0
for x, y in zip(a, b):
result |= ord(x) ^ ord(y)
return result == 0
def print_measurements():
N=10**4
REPS=10**2
print "all times are in nanoseconds per comparison (in scientific notation)"
print
for comparator in [eqeqcomp, hashcomp, xorcomp, sillycomp]:
print "using comparator ", comparator
# for (a, b, desc) in [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')]:
trials = [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')]
random.shuffle(trials)
for (a, b, desc) in trials:
print "comparing two strings that are %s to each other" % (desc,)
def f(n):
compare(n, comparator, a, b)
benchutil.rep_bench(f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS)
print
def try_to_crack_secret(cracker, comparator, secretlen, alphabetsize):
secret = randstr(secretlen, alphabetsize)
def test_guess(x):
return comparator(secret, x)
print "Giving cracker %s a chance to figure out the secret. Don't tell him, but the secret is %s. Whenever he makes a guess, we'll use comparator %s to decide if his guess is right ..." % (cracker, secret.encode('hex'), comparator,)
guess = cracker(test_guess, secretlen, alphabetsize)
print "Cracker %s guessed %r" % (cracker, guess,)
if guess == secret:
print "HE FIGURED IT OUT!? HOW DID HE DO THAT."
else:
print "HAHA. Our secret is safe."
def byte_at_a_time_cracker(test_guess, secretlen, alphabetsize):
# If we were cleverer, we'd add some backtracking behaviour where, if we can't find any x such that ABCx stands out from the crowd as taking longer than all the other ABCy's, then we start to think that we've taken a wrong step and we go back to trying ABy's. Make sense? But we're not that clever. Once we take a step, we don't backtrack.
print
guess=[]
while len(guess) < secretlen:
best_next_byte = None
best_next_byte_time = None
# For each possible byte...
for next_byte in range(alphabetsize):
c = chr(next_byte)
# Construct a guess with our best candidate so far...
candidate_guess = guess[:]
# Plus that byte...
candidate_guess.append(c)
s = ''.join(candidate_guess)
# Plus random bytes...
s += os.urandom(32 - len(s))
# And see how long it takes the test_guess to consider it...
def f(n):
for i in xrange(n):
test_guess(s)
times = benchutil.rep_bench(f, 10**7, MAXREPS=10**3, quiet=True)
fastesttime = times['mean']
print "%s..."%(c.encode('hex'),),
if best_next_byte is None or fastesttime > best_next_byte_time:
print "new candidate for slowest next-char: %s, took: %s" % (c.encode('hex'), fastesttime,),
best_next_byte_time = fastesttime
best_next_byte = c
# Okay we've tried all possible next bytes. Our guess is this one (the one that took longest to be tested by test_guess):
guess.append(best_next_byte)
print "SLOWEST next-char %s! Current guess at secret: %s" % (best_next_byte.encode('hex'), ''.join(guess).encode('hex'),)
guess = ''.join(guess)
print "Our guess for the secret: %r" % (guess,)
return guess
if __name__ == '__main__':
import sys
secretlen = int(sys.argv[1])
alphabetsize = int(sys.argv[2])
if alphabetsize > 256:
raise Exception("We assume we can fit one element of the alphabet into a byte.")
print "secretlen: %d, alphabetsize: %d" % (secretlen, alphabetsize,)
# try_to_crack_secret(byte_at_a_time_cracker, sillycomp, secretlen, alphabetsize)
try_to_crack_secret(byte_at_a_time_cracker, eqeqcomp, secretlen, alphabetsize)
libs/pyutil/test/current/test_mathutil.py
+7
View File
@@ -42,6 +42,13 @@ class MathUtilTestCase(unittest.TestCase):
self.failUnlessEqual(f(5, 3), 2)
self.failUnlessEqual(f(6, 3), 2)
self.failUnlessEqual(f(7, 3), 3)
self.failUnless(isinstance(f(0.0, 1), int))
self.failUnlessEqual(f(7.0, 3.0), 3)
self.failUnlessEqual(f(7, 3.0), 3)
self.failUnlessEqual(f(7.0, 3), 3)
self.failUnlessEqual(f(6.0, 3.0), 2)
self.failUnlessEqual(f(6.0, 3), 2)
self.failUnlessEqual(f(6, 3.0), 2)
def test_next_multiple(self):
f = mathutil.next_multiple
libs/pyutil/test/current/test_mathutil.py~
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env python
import unittest
from pyutil import mathutil
from pyutil.assertutil import _assert
class MathUtilTestCase(unittest.TestCase):
def _help_test_is_power_of_k(self, k):
for i in range(2, 40):
_assert(mathutil.is_power_of_k(k**i, k), k, i)
def test_is_power_of_k(self):
for i in range(2, 5):
self._help_test_is_power_of_k(i)
def test_log_ceil(self):
f = mathutil.log_ceil
self.failUnlessEqual(f(1, 2), 0)
self.failUnlessEqual(f(1, 3), 0)
self.failUnlessEqual(f(2, 2), 1)
self.failUnlessEqual(f(2, 3), 1)
self.failUnlessEqual(f(3, 2), 2)
def test_log_floor(self):
f = mathutil.log_floor
self.failUnlessEqual(f(1, 2), 0)
self.failUnlessEqual(f(1, 3), 0)
self.failUnlessEqual(f(2, 2), 1)
self.failUnlessEqual(f(2, 3), 0)
self.failUnlessEqual(f(3, 2), 1)
def test_div_ceil(self):
f = mathutil.div_ceil
self.failUnlessEqual(f(0, 1), 0)
self.failUnlessEqual(f(0, 2), 0)
self.failUnlessEqual(f(0, 3), 0)
self.failUnlessEqual(f(1, 3), 1)
self.failUnlessEqual(f(2, 3), 1)
self.failUnlessEqual(f(3, 3), 1)
self.failUnlessEqual(f(4, 3), 2)
self.failUnlessEqual(f(5, 3), 2)
self.failUnlessEqual(f(6, 3), 2)
self.failUnlessEqual(f(7, 3), 3)
def test_next_multiple(self):
f = mathutil.next_multiple
self.failUnlessEqual(f(5, 1), 5)
self.failUnlessEqual(f(5, 2), 6)
self.failUnlessEqual(f(5, 3), 6)
self.failUnlessEqual(f(5, 4), 8)
self.failUnlessEqual(f(5, 5), 5)
self.failUnlessEqual(f(5, 6), 6)
self.failUnlessEqual(f(32, 1), 32)
self.failUnlessEqual(f(32, 2), 32)
self.failUnlessEqual(f(32, 3), 33)
self.failUnlessEqual(f(32, 4), 32)
self.failUnlessEqual(f(32, 5), 35)
self.failUnlessEqual(f(32, 6), 36)
self.failUnlessEqual(f(32, 7), 35)
self.failUnlessEqual(f(32, 8), 32)
self.failUnlessEqual(f(32, 9), 36)
self.failUnlessEqual(f(32, 10), 40)
self.failUnlessEqual(f(32, 11), 33)
self.failUnlessEqual(f(32, 12), 36)
self.failUnlessEqual(f(32, 13), 39)
self.failUnlessEqual(f(32, 14), 42)
self.failUnlessEqual(f(32, 15), 45)
self.failUnlessEqual(f(32, 16), 32)
self.failUnlessEqual(f(32, 17), 34)
self.failUnlessEqual(f(32, 18), 36)
self.failUnlessEqual(f(32, 589), 589)
def test_pad_size(self):
f = mathutil.pad_size
self.failUnlessEqual(f(0, 4), 0)
self.failUnlessEqual(f(1, 4), 3)
self.failUnlessEqual(f(2, 4), 2)
self.failUnlessEqual(f(3, 4), 1)
self.failUnlessEqual(f(4, 4), 0)
self.failUnlessEqual(f(5, 4), 3)
def test_is_power_of_k_part_2(self):
f = mathutil.is_power_of_k
for i in range(1, 100):
if i in (1, 2, 4, 8, 16, 32, 64):
self.failUnless(f(i, 2), "but %d *is* a power of 2" % i)
else:
self.failIf(f(i, 2), "but %d is *not* a power of 2" % i)
for i in range(1, 100):
if i in (1, 3, 9, 27, 81):
self.failUnless(f(i, 3), "but %d *is* a power of 3" % i)
else:
self.failIf(f(i, 3), "but %d is *not* a power of 3" % i)
def test_next_power_of_k(self):
f = mathutil.next_power_of_k
self.failUnlessEqual(f(0,2), 1)
self.failUnlessEqual(f(1,2), 1)
self.failUnlessEqual(f(2,2), 2)
self.failUnlessEqual(f(3,2), 4)
self.failUnlessEqual(f(4,2), 4)
for i in range(5, 8): self.failUnlessEqual(f(i,2), 8, "%d" % i)
for i in range(9, 16): self.failUnlessEqual(f(i,2), 16, "%d" % i)
for i in range(17, 32): self.failUnlessEqual(f(i,2), 32, "%d" % i)
for i in range(33, 64): self.failUnlessEqual(f(i,2), 64, "%d" % i)
for i in range(65, 100): self.failUnlessEqual(f(i,2), 128, "%d" % i)
self.failUnlessEqual(f(0,3), 1)
self.failUnlessEqual(f(1,3), 1)
self.failUnlessEqual(f(2,3), 3)
self.failUnlessEqual(f(3,3), 3)
for i in range(4, 9): self.failUnlessEqual(f(i,3), 9, "%d" % i)
for i in range(10, 27): self.failUnlessEqual(f(i,3), 27, "%d" % i)
for i in range(28, 81): self.failUnlessEqual(f(i,3), 81, "%d" % i)
for i in range(82, 200): self.failUnlessEqual(f(i,3), 243, "%d" % i)
def test_ave(self):
f = mathutil.ave
self.failUnlessEqual(f([1,2,3]), 2)
self.failUnlessEqual(f([0,0,0,4]), 1)
self.failUnlessAlmostEqual(f([0.0, 1.0, 1.0]), .666666666666)
def failUnlessEqualContents(self, a, b):
self.failUnlessEqual(sorted(a), sorted(b))
def test_permute(self):
f = mathutil.permute
self.failUnlessEqualContents(f([]), [])
self.failUnlessEqualContents(f([1]), [[1]])
self.failUnlessEqualContents(f([1,2]), [[1,2], [2,1]])
self.failUnlessEqualContents(f([1,2,3]),
[[1,2,3], [1,3,2],
[2,1,3], [2,3,1],
[3,1,2], [3,2,1]])
libs/pyutil/time_comparisons.py
-44
View File
@@ -1,44 +0,0 @@
from pyutil import benchutil
import hashlib, random, os
from decimal import Decimal
D=Decimal
p1 = 'a'*32
p1a = 'a'*32
p2 = 'a'*31+'b' # close, but no cigar
p3 = 'b'*32 # different in the first byte
def compare(n, f, a, b):
for i in xrange(n):
f(a, b)
def eqeqcomp(a, b):
return a == b
def hashcomp(a, b):
salt = os.urandom(32)
return hashlib.md5(salt+ a).digest() == hashlib.md5(salt+b).digest()
N=10**4
REPS=10**2
print "all times are in nanoseconds per comparison (scientific notation)"
print
for comparator in [eqeqcomp, hashcomp]:
print "using comparator ", comparator
# for (a, b, desc) in [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')]:
trials = [(p1, p1a, 'same'), (p1, p2, 'close'), (p1, p3, 'far')]
random.shuffle(trials)
for (a, b, desc) in trials:
print "comparing two strings that are %s to each other" % (desc,)
def f(n):
compare(n, comparator, a, b)
benchutil.rep_bench(f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS)
print
libs/pyutil/time_comparisons.py~
-72
View File
@@ -1,72 +0,0 @@
from pyutil import benchutil
import hashlib
import os
from decimal import Decimal
D=Decimal
p1 = 'a'*32
p1a = 'a'*32
p2 = 'a'*31+'b' # close, but no cigar
p3 = 'b'*32 # different in the first byte
def compare(n, f, a, b):
for i in xrange(n):
f(a, b)
def eqeq(a, b):
return a == b
def equalsequals_s(n):
# return compare(n, eqeq,
for i in xrange(n):
p1 == p1a
def equalsequals_c(n):
for i in xrange(n):
p1 == p2
def equalsequals_f(n):
for i in xrange(n):
p1 == p3
def hash_s(n):
for i in xrange(n):
salt = os.urandom(32)
hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p1a).digest()
def hash_c(n):
for i in xrange(n):
salt = os.urandom(32)
hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p2).digest()
def hash_f(n):
for i in xrange(n):
salt = os.urandom(32)
hashlib.md5(salt+ p1).digest() == hashlib.md5(salt+p3).digest()
N=10**4
REPS=10**2
print "using '=='"
print "same"
benchutil.rep_bench(equalsequals_s, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS)
print "close"
benchutil.rep_bench(equalsequals_c, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS)
print "far"
benchutil.rep_bench(equalsequals_f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS)
print "using hash"
print "same"
benchutil.rep_bench(hash_s, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS)
print "far"
benchutil.rep_bench(hash_f, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS)
print "close"
benchutil.rep_bench(hash_c, N, UNITS_PER_SECOND=10**9, MAXREPS=REPS)