from __future__ import print_function
import os
import sys
import functools
import contextlib
import hashlib
import numpy
import fcntl
def make_list(generator_func):
"""Create a list from a generator function.
"""
def wrapper(*args, **kwargs):
return list(generator_func(*args, **kwargs))
return functools.update_wrapper(wrapper, generator_func)
def vectorize(generator_func):
"""Create a numpy array from an generator function.
numpy.fromiter could have been used, but then the dtype would
have to be specified (or maybe guessed based on the first item).
This version is not super efficient, but easy to use.
>>> @vectorize
... def func():
... yield 1
... yield 2
... yield 3.4
>>> func()
array([ 1. , 2. , 3.4])
"""
def wrapper(*args, **kwargs):
return numpy.array(list(generator_func(*args, **kwargs)))
return functools.update_wrapper(wrapper, generator_func)
@contextlib.contextmanager
def flocked(file, exclusive=True):
fcntl.flock(file, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
try:
yield
finally:
fcntl.flock(file, fcntl.LOCK_UN)
@contextlib.contextmanager
def capture_stdout(file=None):
"""Capture output temporarily.
Use as:
>>> import io
>>> with capture_stdout(io.StringIO()) as c:
... print(u'hello, world')
>>> print(c.getvalue(), end='')
hello, world
Or simply as:
>>> def invoke_crazy_function_which_spews_garbage_on_stdout():
... print('x' * 1000)
>>> with capture_stdout():
... invoke_crazy_function_which_spews_garbage_on_stdout()
"""
stdout = sys.stdout
if file is None:
file = open(os.devnull, 'w')
sys.stdout = file
try:
yield file
finally:
sys.stdout = stdout
@contextlib.contextmanager
def chdired(dirname):
"""Change directory temporarily"""
old = os.getcwd()
os.chdir(dirname)
try:
yield
finally:
os.chdir(old)
@contextlib.contextmanager
def numpy_seterr(all=None, divide=None, over=None, under=None, invalid=None):
old = numpy.seterr(all=all, divide=divide, over=over, under=under,
invalid=invalid)
try:
yield
finally:
numpy.seterr(**old)
def disk_cached(basepath):
"""
Caches results in numpy files on disk.
Specified basepath must be under the control of the caller,
and entries must be cleaned manually. So this a basis for
a solution, not the final thing certainly.
>>> @disk_cached('/tmp/f_result_')
... def f(*args, **kwargs):
... print('f')
... ans = numpy.array([len(args), len(kwargs)])
... return ans
>>> f(3)
f
array([1, 0])
>>> f(3)
array([1, 0])
>>> f(*[3])
array([1, 0])
>>> f(a=1, b=2)
f
array([0, 2])
>>> f(b=2, a=1)
array([0, 2])
>>> f([1,2,3])
f
array([1, 0])
>>> f([1,2,3])
array([1, 0])
"""
def decorator(func):
def wrapper(*args, **kwargs):
key = (args, tuple(sorted(kwargs.items())))
tag = hashlib.sha1(repr(key)).hexdigest()
path = basepath + tag + '.npy'
try:
ans = numpy.load(path)
except TypeError:
print('warning: function arguments are unhashable')
return func(*args, **kwargs)
except IOError:
# value is not present in cache
ans = func(*args, **kwargs)
numpy.save(path, ans)
return ans
return functools.update_wrapper(wrapper, func)
return decorator