Коварный code type ITGM #9

Коварный CodeType,
или от segfault'а
к работающему коду
Андрей Захаревич
Дмитрий Алимов

С чего все началось
# Plain functions
if inspect.isroutine(obj):
if inspect.ismethod(obj):
has_self = True
argspec = inspect.getargspec(obj)
# Class constructors
elif inspect.isclass(obj):
try:
argspec = inspect.getargspec(obj.__new__)
except (AttributeError, TypeError):
argspec = inspect.getargspec(obj.__init__)
has_self = True
# obj objects
elif hasattr(obj, '__call__'):
method = obj.__call__
argspec = inspect.getargspec(method)
has_self = inspect.ismethod(method)
return argspec.args[1:] if has_self else args

Проблема
● Моя функция принимает переменное число аргументов (*args)
● Число передаваемых аргументов я узнаю в рантайме
● Объект функции в это время уже создан
Что же делать?

Метапрограммирование!

Ленивый вариант
compile('def func(a, b): f(a, b)', 'func.py', 'exec')
Не спортивно

Идем на stackoverflow
import types
def create_function(name, args):
def y(): pass
y_code = types.CodeType(args,
y.func_code.co_nlocals,
y.func_code.co_stacksize,
y.func_code.co_flags,
y.func_code.co_code,
y.func_code.co_consts,
y.func_code.co_names,
y.func_code.co_varnames,
y.func_code.co_filename,
name,
y.func_code.co_firstlineno,
y.func_code.co_lnotab)
return types.FunctionType(y_code, y.func_globals, name)
myfunc = create_function('myfunc', 3)
myfunc(1, 2, 3, 4)
# TypeError: myfunc() takes exactly 3 arguments (4 given)

Кажется должно работать
def create_function(name, nargs):
def y(*args):
print args
y_code = types.CodeType(nargs,
y.func_code.co_nlocals,
# Параметры
)
return types.FunctionType(y_code, y.func_globals,
name)
myfunc = create_function('myfunc', 3)
myfunc(1, 2, 3)

Но нет
[1] 8514 segmentation fault python func.py

Еще попытка
class Meta(type):
def __new__(mcls, name, bases, attrs):
nargs = 3
varnames = tuple(['self'] + [str(i) for i in range(nargs)])
ret_code = types.CodeType(
nargs + 1,
attrs['ret_func'].func_code.co_nlocals,
attrs['ret_func'].func_code.co_stacksize,
attrs['ret_func'].func_code.co_flags,
attrs['ret_func'].func_code.co_code,
attrs['ret_func'].func_code.co_consts,
attrs['ret_func'].func_code.co_names,
varnames,
attrs['ret_func'].func_code.co_filename,
'call',
attrs['ret_func'].func_code.co_firstlineno,
attrs['ret_func'].func_code.co_lnotab)
attrs['call'] = types.FunctionType(
ret_code,
attrs['ret_func'].func_globals, 'call')
return super(Meta, mcls).__new__(mcls, name, bases, attrs)
class CustomCall(object):
__metaclass__ = Meta
def __call__(self, *args):
self.call(*args)
def ret_func(self):
print self.call.__name__
obj = CustomCall()
obj.call(1, 2, 3)
# Если
закомментировать, то
все упадет
obj(1, 2, 3)

01eb3030 00000001 ob_refcnt
01eb3034 5886d830 python27!PyFrame_Type
01eb3038 00000002 ob_size
01eb303c 023b6b30 f_back
01eb3040 023a6b18 f_code
01eb3044 01e790c0 f_builtins
01eb3048 01e8aa50 f_globals
01eb304c 00000000 f_locals
01eb3050 01eb316c f_valuestack
01eb3054 01eb316c f_stacktop
01eb3058 00000000 f_trace
01eb305c 00000000 f_exc_type
01eb3060 00000000 f_exc_value
01eb3064 00000000 f_exc_traceback
01eb3068 01f72cf0 f_tstate
01eb306c ffffffff f_lasti
01eb3070 0000002f f_lineno
01eb3074 00000000 f_iblock
01eb3078 baadf00d f_blockstack[20] = {b_type,
01eb307c baadf00d b_handler
01eb3080 baadf00d b_level}
...
01eb3168 023bf550 f_localsplus // frame + 0x138
01eb316c 01f7d8a0
01eb3170 baadf00d
01eb3174 baadf00d
Что в памяти

PyObject *
PyEval_EvalCodeEx(PyCodeObject *co, PyObject *globals, PyObject *locals,
PyObject **args, int argcount, PyObject **kws, int kwcount,
PyObject **defs, int defcount, PyObject *closure) {
...
n = co->co_argcount;
for (i = 0; i < n; i++) {
x = args[i];
Py_INCREF(x);
SETLOCAL(i, x);
}
...
https://github.com/python/cpython/blob/2.7/Python/ceval.c
#define Py_INCREF(op) (
_Py_INC_REFTOTAL _Py_REF_DEBUG_COMMA
((PyObject*)(op))->ob_refcnt++)
#define GETLOCAL(i) (fastlocals[i])
#define SETLOCAL(i, value) do { PyObject *tmp = GETLOCAL(i);
GETLOCAL(i) = value;
Py_XDECREF(tmp); } while (0)

#define Py_XDECREF(op) do { if ((op) == NULL); else Py_DECREF(op); } while (0)
#define Py_DECREF(op)
do {
if (_Py_DEC_REFTOTAL _Py_REF_DEBUG_COMMA
--((PyObject*)(op))->ob_refcnt != 0)
_Py_CHECK_REFCNT(op)
else
_Py_Dealloc((PyObject *)(op));
} while (0)
#define _Py_Dealloc(op) (
_Py_INC_TPFREES(op) _Py_COUNT_ALLOCS_COMMA
(*Py_TYPE(op)->tp_dealloc)((PyObject *)(op)))
#endif /* !Py_TRACE_REFS */
https://github.com/python/cpython/blob/2.7/Include/object.h

typedef struct _frame {
PyObject_VAR_HEAD
struct _frame *f_back; /* previous frame, or NULL */
PyCodeObject *f_code; /* code segment */
PyObject *f_builtins; /* builtin symbol table (PyDictObject) */
PyObject *f_globals; /* global symbol table (PyDictObject) */
PyObject *f_locals; /* local symbol table (any mapping) */
PyObject **f_valuestack; /* points after the last local */
PyObject **f_stacktop;
PyObject *f_trace; /* Trace function */
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
PyThreadState *f_tstate;
int f_lasti; /* Last instruction if called */
int f_lineno; /* Current line number */
int f_iblock; /* index in f_blockstack */
PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */
PyObject *f_localsplus[1]; /* locals+stack, dynamically sized */
} PyFrameObject;
https://github.com/python/cpython/blob/2.7/Include/frameobject.h

ncells = PyTuple_GET_SIZE(code->co_cellvars);
nfrees = PyTuple_GET_SIZE(code->co_freevars);
extras = code->co_stacksize + code->co_nlocals + ncells + nfrees;
if (free_list == NULL) {
f = PyObject_GC_NewVar(PyFrameObject, &PyFrame_Type, extras);
...
https://github.com/python/cpython/blob/2.7/Objects/frameobject.c
co_cellvars – is a tuple with the names of local variables referenced by nested functions;
co_freevars – names of variables in the function, defined in an enclosing function scope;
Память для f_localsplus

Проблема найдена
ret_code = types.CodeType(
nargs + 1,
attrs['ret_func'].func_code.co_nlocals,
attrs['ret_func'].func_code.co_stacksize,
attrs['ret_func'].func_code.co_flags,
attrs['ret_func'].func_code.co_code,
attrs['ret_func'].func_code.co_consts,
attrs['ret_func'].func_code.co_names,
varnames,
attrs['ret_func'].func_code.co_filename,
'call',
attrs['ret_func'].func_code.co_firstlineno,
attrs['ret_func'].func_code.co_lnotab,
(), # freevars
() # cellvars
)

Попробуем применить
def create_func(fun, nargs):
def wrapper():
args = map(itemgetter(1),
sorted(((var_name, arg) for (var_name, arg)
in locals().items() if var_name != 'fun'),
key=itemgetter(0)))
fun(*args)
varnames = tuple(str(i) for i in range(0, nargs))
ret_code = types.CodeType(nargs,
wrapper.func_code.co_nlocals + nargs,
)
return types.FunctionType(ret_code, wrapper.func_globals,
wrapper.func_code.co_name)
def test_func(*args):
print args
fun = create_func(test_func, 3)
fun(1, 2, 3)

Как-то не очень
Traceback (most recent call last):
File "func.py", line 38, in <module>
fun(1,2,3)
File "func.py", line 16, in wrapper
fun(*args)
SystemError: Objects/cellobject.c:24: bad argument to internal function

Заработало!
def wrapper(fun):
key=itemgetter(0)))
fun(*args)
varnames = tuple(['fun'] + [str(i) for i in range(0, nargs)])
ret_code = types.CodeType(nargs + 1,
)
return partial(types.FunctionType(ret_code, wrapper.func_globals,
wrapper.func_code.co_name), fun)
print args
fun(1, 2, 3)

Что-то потеряли
>>> dir(wrapper.func_code)
['__class__', '__cmp__', '__delattr__', '__doc__', '__eq__', '__format__', '__ge__',
'__getattribute__', '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__ne__', '__new__',
'__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__',
'__subclasshook__', 'co_argcount', 'co_cellvars', 'co_code', 'co_consts', 'co_filename',
'co_firstlineno', 'co_flags', 'co_freevars', 'co_lnotab', 'co_name', 'co_names', 'co_nlocals',
'co_stacksize', 'co_varnames']

Добавим параметр
wrapper.func_code.co_freevars)
>>>Traceback (most recent call last):
File "func.py", line 39, in <module>
File "func.py", line 33, in create_func
TypeError: arg 5 (closure) must be tuple

def wrapper():
key=itemgetter(0)))
fun(*args)
varnames = tuple([str(i) for i in range(0, nargs)])
wrapper.func_code.co_freevars)
wrapper.func_code.co_name, None,
wrapper.func_closure)
print args
fun(1, 2, 3)
И наконец, замыкания работают!

… почти
from operator import itemgetter
def wrapper():
print locals()
key=itemgetter(0)))
print locals()
fun(*args)
{'1': 2, '0': 1, '2': 3, 'fun': <function test_func at 0x10cd62398>}
{'1': 2, '0': [1, 2, 3], '2': 3, 'fun': <function test_func at 0x10cd62398>}

Спасибо за внимание!
Вопросы?
SPb Python Interest Group
https://telegram.me/spbpython

Коварный code type ITGM #9

More Related Content

What's hot (20)

Similar to Коварный code type ITGM #9 (20)

Recently uploaded (20)

Коварный code type ITGM #9