Py interop fix (#23)

* Fix incref/decref

* Fixes

* Fix set __to_py__

* Add more Python conversion tests

* clang-format

* Make from_py/to_py use cobj instead of pyobj

* Remove unneeded increfs

* Fix cython

* Ignore __init_test__ in doc generation

* Add exception check

Co-authored-by: Ishak Numanagić <ishak.numanagic@gmail.com>
Co-authored-by: Ibrahim Numanagić <ibrahimpasa@gmail.com>
pull/27/head
A. R. Shajii 2022-04-07 00:33:49 -04:00 committed by GitHub
parent 12e8fe7666
commit 753151157d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 135 additions and 120 deletions

View File

@ -19,15 +19,11 @@ struct JITResult {
std::string data;
bool isError;
JITResult():
data(""), isError(false) {}
JITResult() : data(""), isError(false) {}
JITResult(const std::string &data, bool isError):
data(data), isError(isError) {}
JITResult(const std::string &data, bool isError) : data(data), isError(isError) {}
operator bool() {
return !this->isError;
}
operator bool() { return !this->isError; }
static JITResult success(const std::string &output) {
return JITResult(output, false);

View File

@ -156,6 +156,7 @@ std::vector<StmtPtr> DocVisitor::flatten(StmtPtr stmt, std::string *docstr, bool
std::shared_ptr<json> DocVisitor::transform(const ExprPtr &expr) {
DocVisitor v(ctx);
v.setSrcInfo(expr->getSrcInfo());
v.resultExpr = std::make_shared<json>();
expr->accept(v);
return v.resultExpr;
@ -163,6 +164,7 @@ std::shared_ptr<json> DocVisitor::transform(const ExprPtr &expr) {
std::string DocVisitor::transform(const StmtPtr &stmt) {
DocVisitor v(ctx);
v.setSrcInfo(stmt->getSrcInfo());
stmt->accept(v);
return v.resultStmt;
}
@ -290,6 +292,7 @@ void DocVisitor::visit(ClassStmt *stmt) {
for (auto &d : stmt->decorators)
if (auto e = d->getId())
isExtend |= (e->value == "extend");
if (isExtend) {
j->set("type", "extension");
auto i = ctx->find(stmt->name);

View File

@ -1399,7 +1399,8 @@ StmtPtr SimplifyVisitor::transformPythonImport(const Expr *what,
// or return retType.__from_py__(f(args...))
ExprPtr retExpr = N<CallExpr>(N<IdExpr>("f"), callArgs);
if (ret && !ret->isId("void"))
retExpr = N<CallExpr>(N<DotExpr>(ret->clone(), "__from_py__"), retExpr);
retExpr =
N<CallExpr>(N<DotExpr>(ret->clone(), "__from_py__"), N<DotExpr>(retExpr, "p"));
StmtPtr retStmt = nullptr;
if (ret && ret->isId("void"))
retStmt = N<ExprStmt>(retExpr);
@ -1745,30 +1746,30 @@ StmtPtr SimplifyVisitor::codegenMagic(const std::string &op, const Expr *typExpr
ret = I("int");
stmts.emplace_back(N<ReturnStmt>(N<IntExpr>(args.size())));
} else if (op == "to_py") {
// def __to_py__(self: T) -> pyobj:
// def __to_py__(self: T) -> cobj:
// o = pyobj._tuple_new(N) (number of args)
// o._tuple_set(1, self.arg1.__to_py__()) ...
// pyobj._tuple_set(o, 1, self.arg1.__to_py__()) ...
// return o
fargs.emplace_back(Param{"self", typExpr->clone()});
ret = I("pyobj");
ret = I("cobj");
stmts.emplace_back(
N<AssignStmt>(I("o"), N<CallExpr>(N<DotExpr>(I("pyobj"), "_tuple_new"),
N<IntExpr>(args.size()))));
for (int i = 0; i < args.size(); i++)
stmts.push_back(N<ExprStmt>(N<CallExpr>(
N<DotExpr>(I("o"), "_tuple_set"), N<IntExpr>(i),
N<DotExpr>(I("pyobj"), "_tuple_set"), I("o"), N<IntExpr>(i),
N<CallExpr>(N<DotExpr>(N<DotExpr>(I("self"), args[i].name), "__to_py__")))));
stmts.emplace_back(N<ReturnStmt>(I("o")));
} else if (op == "from_py") {
// def __from_py__(src: pyobj) -> T:
// return T(T1.__from_py__(src._tuple_get(1)), ...)
fargs.emplace_back(Param{"src", I("pyobj")});
// def __from_py__(src: cobj) -> T:
// return T(T1.__from_py__(pyobj._tuple_get(src, 1)), ...)
fargs.emplace_back(Param{"src", I("cobj")});
ret = typExpr->clone();
std::vector<ExprPtr> ar;
for (int i = 0; i < args.size(); i++)
ar.push_back(
N<CallExpr>(N<DotExpr>(clone(args[i].type), "__from_py__"),
N<CallExpr>(N<DotExpr>(I("src"), "_tuple_get"), N<IntExpr>(i))));
ar.push_back(N<CallExpr>(
N<DotExpr>(clone(args[i].type), "__from_py__"),
N<CallExpr>(N<DotExpr>(I("pyobj"), "_tuple_get"), I("src"), N<IntExpr>(i))));
stmts.emplace_back(N<ReturnStmt>(N<CallExpr>(typExpr->clone(), ar)));
} else if (op == "repr") {
// def __repr__(self: T) -> str:

View File

@ -23,7 +23,7 @@ def load_json(directory):
files=[]
for root,_,items in os.walk(directory):
for f in items:
if f.endswith('.codon'):
if f.endswith('.codon') and "__init_test__.codon" not in f:
files.append(os.path.abspath(os.path.join(root,f)))
files='\n'.join(files)
s=sp.run(['../../build/codon','doc'],stdout=sp.PIPE,input=files.encode('utf-8'))
@ -34,6 +34,7 @@ def load_json(directory):
j=load_json(root)
print(f" - Done with codon")
sys.exit(0)
# with open('x.json','w') as f:
# json.dump(j,f,indent=2)

View File

@ -38,7 +38,7 @@ def _wrapper_stub_header():
def _wrapper_stub_footer_ret():
PyObject_SetAttrString(module, "__codon_ret__".c_str(), ret.p)
PyObject_SetAttrString(module, "__codon_ret__".c_str(), ret)
def _wrapper_stub_footer_void():
@ -134,7 +134,7 @@ def _build_wrapper(obj, obj_name) -> str:
wrap = [f"def {wrap_name}():\n"]
wrap += inspect.getsourcelines(_wrapper_stub_header)[0][1:]
wrap += [
f" arg_{i} = {_type_str(arg_types[i])}.__from_py__(pyobj(PyTuple_GetItem(argt, {i})))\n"
f" arg_{i} = {_type_str(arg_types[i])}.__from_py__(PyTuple_GetItem(argt, {i}))\n"
for i in range(arg_count)
]
args = ", ".join([f"arg_{i}" for i in range(arg_count)])

View File

@ -194,64 +194,67 @@ class pyobj:
def __raw__(self) -> Ptr[byte]:
return __internal__.class_raw(self)
def __init__(self, p: Ptr[byte]):
def __init__(self, p: Ptr[byte], steal: bool = False):
self.p = p
if not steal:
self.incref()
def __del__(self):
self.decref()
def _getattr(self, name: str) -> pyobj:
return pyobj.exc_wrap(pyobj(PyObject_GetAttrString(self.p, name.c_str())))
def __setitem__(self, name: str, val: pyobj) -> pyobj:
return pyobj.exc_wrap(
pyobj(PyObject_SetAttrString(self.p, name.c_str(), val.p))
)
return pyobj(pyobj.exc_wrap(PyObject_GetAttrString(self.p, name.c_str())), steal=True)
def __len__(self) -> int:
return pyobj.exc_wrap(PyObject_Length(self.p))
def __to_py__(self) -> pyobj:
return self
def __to_py__(self) -> cobj:
return self.p
def __from_py__(self) -> pyobj:
return self
def __from_py__(p: cobj) -> pyobj:
return pyobj(p)
def __str__(self) -> str:
return str.__from_py__(self._getattr("__str__").__call__())
return str.__from_py__(self._getattr("__str__").__call__().p)
def __repr__(self) -> str:
return str.__from_py__(self._getattr("__repr__").__call__())
return str.__from_py__(self._getattr("__repr__").__call__().p)
def __iter__(self) -> Generator[pyobj]:
it = PyObject_GetIter(self.p)
if not it:
raise ValueError("Python object is not iterable")
while i := PyIter_Next(it):
yield pyobj(pyobj.exc_wrap(i))
yield pyobj(pyobj.exc_wrap(i), steal=True)
pyobj.decref(it)
pyobj.exc_check()
def to_str(self, errors: str, empty: str = "") -> str:
obj = PyUnicode_AsEncodedString(self.p, "utf-8".c_str(), errors.c_str())
def to_str(p: cobj, errors: str, empty: str = "") -> str:
obj = PyUnicode_AsEncodedString(p, "utf-8".c_str(), errors.c_str())
if obj == cobj():
return empty
bts = PyBytes_AsString(obj)
res = str.from_ptr(bts)
pyobj.decref(obj)
return str.from_ptr(bts)
return res
def to_str(self, errors: str, empty: str = "") -> str:
return pyobj.to_str(self.p, errors, empty)
def exc_check():
ptype, pvalue, ptraceback = cobj(), cobj(), cobj()
PyErr_Fetch(__ptr__(ptype), __ptr__(pvalue), __ptr__(ptraceback))
if ptype != cobj():
py_msg = PyObject_Str(pvalue) if pvalue != cobj() else pvalue
msg = pyobj(py_msg).to_str("ignore", "<empty Python message>")
typ = pyobj(PyObject_GetAttrString(ptype, "__name__".c_str())).to_str("ignore")
msg = pyobj.to_str(py_msg, "ignore", "<empty Python message>")
py_typ = PyObject_GetAttrString(ptype, "__name__".c_str())
typ = pyobj.to_str(py_typ, "ignore")
pyobj.decref(ptype)
pyobj.decref(pvalue)
pyobj.decref(ptraceback)
pyobj.decref(py_msg)
pyobj.decref(py_typ)
raise PyError(msg, typ)
@ -261,50 +264,42 @@ class pyobj:
def incref(self):
Py_IncRef(self.p)
def incref(obj: pyobj):
Py_IncRef(obj.p)
return self
def incref(ptr: Ptr[byte]):
Py_IncRef(ptr)
def decref(self):
Py_DecRef(self.p)
def decref(obj: pyobj):
Py_DecRef(obj.p)
return self
def decref(ptr: Ptr[byte]):
Py_DecRef(ptr)
def __call__(self, *args, **kwargs) -> pyobj:
names = iter(kwargs.__dict__())
kws = dict[str, pyobj]()
def __call__(self, *args, **kwargs):
args_py = args.__to_py__()
kws_py = cobj()
if staticlen(kwargs) > 0:
kws = {next(names): i.__to_py__() for i in kwargs}
return pyobj.exc_wrap(
pyobj(PyObject_Call(self.p, args.__to_py__().p, kws.__to_py__().p))
)
names = iter(kwargs.__dict__())
kws = {next(names): pyobj(i.__to_py__(), steal=True) for i in kwargs}
kws_py = kws.__to_py__()
return pyobj(pyobj.exc_wrap(PyObject_Call(self.p, args_py, kws_py)), steal=True)
def _tuple_new(length: int) -> pyobj:
t = PyTuple_New(length)
pyobj.exc_check()
return pyobj(t)
def _tuple_new(length: int):
return pyobj.exc_wrap(PyTuple_New(length))
def _tuple_set(self, idx: int, val: pyobj):
PyTuple_SetItem(self.p, idx, val.p)
def _tuple_set(p: cobj, idx: int, val: cobj):
PyTuple_SetItem(p, idx, val)
pyobj.exc_check()
def _tuple_get(self, idx: int) -> pyobj:
t = PyTuple_GetItem(self.p, idx)
pyobj.exc_check()
return pyobj(t)
def _tuple_get(p: cobj, idx: int) -> cobj:
return pyobj.exc_wrap(PyTuple_GetItem(p, idx))
def _import(name: str) -> pyobj:
ensure_initialized()
if name in _PY_MODULE_CACHE:
return _PY_MODULE_CACHE[name]
m = pyobj.exc_wrap(pyobj(PyImport_ImportModule(name.c_str())))
m = pyobj(pyobj.exc_wrap(PyImport_ImportModule(name.c_str())), steal=True)
_PY_MODULE_CACHE[name] = m
return m
@ -312,17 +307,13 @@ class pyobj:
ensure_initialized()
PyRun_SimpleString(code.c_str())
def get(self, T: type) -> T:
return T.__from_py__(self)
def _main_module() -> pyobj:
m = PyImport_AddModule("__main__".c_str())
return pyobj(m)
return pyobj(pyobj.exc_wrap(PyImport_AddModule("__main__".c_str())))
def _repr_mimebundle_(self, bundle=Set[str]()) -> Dict[str, str]:
fn = pyobj._main_module()._getattr("__codon_repr__")
assert fn.p != cobj(), "cannot find python.__codon_repr__"
mime, txt = fn.__call__(self).get(Tuple[str, str])
mime, txt = Tuple[str, str].__from_py__(fn.__call__(self).p)
return {mime: txt}
@ -333,114 +324,115 @@ def none():
# Type conversions
def py(x) -> pyobj:
return x.__to_py__()
def get(x: pyobj, T: type) -> T:
return T.__from_py__(x)
@extend
class int:
def __to_py__(self) -> pyobj:
return pyobj.exc_wrap(pyobj(PyLong_FromLong(self)))
def __to_py__(self) -> cobj:
return pyobj.exc_wrap(PyLong_FromLong(self))
def __from_py__(i: pyobj) -> int:
return pyobj.exc_wrap(PyLong_AsLong(i.p))
def __from_py__(i: cobj) -> int:
return pyobj.exc_wrap(PyLong_AsLong(i))
@extend
class float:
def __to_py__(self) -> pyobj:
return pyobj.exc_wrap(pyobj(PyFloat_FromDouble(self)))
def __to_py__(self) -> cobj:
return pyobj.exc_wrap(PyFloat_FromDouble(self))
def __from_py__(d: pyobj) -> float:
return pyobj.exc_wrap(PyFloat_AsDouble(d.p))
def __from_py__(d: cobj) -> float:
return pyobj.exc_wrap(PyFloat_AsDouble(d))
@extend
class bool:
def __to_py__(self) -> pyobj:
return pyobj.exc_wrap(pyobj(PyBool_FromLong(int(self))))
def __to_py__(self) -> cobj:
return pyobj.exc_wrap(PyBool_FromLong(int(self)))
def __from_py__(b: pyobj) -> bool:
return pyobj.exc_wrap(PyObject_IsTrue(b.p)) != 0
def __from_py__(b: cobj) -> bool:
return pyobj.exc_wrap(PyObject_IsTrue(b)) != 0
@extend
class byte:
def __to_py__(self) -> pyobj:
def __to_py__(self) -> cobj:
return str.__to_py__(str(__ptr__(self), 1))
def __from_py__(c: pyobj) -> byte:
return str.__from_py__(c).p[0]
def __from_py__(c: cobj) -> byte:
return str.__from_py__(c).ptr[0]
@extend
class str:
def __to_py__(self) -> pyobj:
return pyobj.exc_wrap(
pyobj(PyUnicode_DecodeFSDefaultAndSize(self.ptr, self.len))
)
def __to_py__(self) -> cobj:
return pyobj.exc_wrap(PyUnicode_DecodeFSDefaultAndSize(self.ptr, self.len))
def __from_py__(s: pyobj) -> str:
r = s.to_str("strict")
pyobj.exc_check()
return r
def __from_py__(s: cobj) -> str:
return pyobj.exc_wrap(pyobj.to_str(s, "strict"))
@extend
class List:
def __to_py__(self) -> pyobj:
def __to_py__(self) -> cobj:
pylist = PyList_New(len(self))
pyobj.exc_check()
idx = 0
for a in self:
PyList_SetItem(pylist, idx, py(a).p)
PyList_SetItem(pylist, idx, a.__to_py__())
pyobj.exc_check()
idx += 1
return pyobj(pylist)
return pylist
def __from_py__(v: pyobj) -> List[T]:
n = v.__len__()
def __from_py__(v: cobj) -> List[T]:
n = pyobj.exc_wrap(PyObject_Length(v))
t = List[T](n)
for i in range(n):
elem = pyobj(PyList_GetItem(v.p, i))
elem = PyList_GetItem(v, i)
pyobj.exc_check()
t.append(get(elem, T))
t.append(T.__from_py__(elem))
return t
@extend
class Dict:
def __to_py__(self) -> pyobj:
def __to_py__(self) -> cobj:
pydict = PyDict_New()
pyobj.exc_check()
for k, v in self.items():
PyDict_SetItem(pydict, py(k).p, py(v).p)
PyDict_SetItem(pydict, k.__to_py__(), v.__to_py__())
pyobj.exc_check()
return pyobj(pydict)
return pydict
def __from_py__(d: pyobj) -> Dict[K, V]:
def __from_py__(d: cobj) -> Dict[K, V]:
b = dict[K, V]()
pos = 0
k_ptr = cobj()
v_ptr = cobj()
while PyDict_Next(d.p, __ptr__(pos), __ptr__(k_ptr), __ptr__(v_ptr)):
while PyDict_Next(d, __ptr__(pos), __ptr__(k_ptr), __ptr__(v_ptr)):
pyobj.exc_check()
k = get(pyobj(k_ptr), K)
v = get(pyobj(v_ptr), V)
k = K.__from_py__(k_ptr)
v = V.__from_py__(v_ptr)
b[k] = v
return b
@extend
class Set:
def __to_py__(self) -> pyobj:
def __to_py__(self) -> cobj:
pyset = PySet_New(cobj())
pyobj.exc_check()
for a in self:
PySet_Add(pyset, py(a).p)
PySet_Add(pyset, a.__to_py__())
pyobj.exc_check()
return pyobj(pyset)
return pyset
def __from_py__(s: cobj) -> Set[K]:
b = set[K]()
s_iter = pyobj.exc_wrap(PyObject_GetIter(s))
while True:
k_ptr = pyobj.exc_wrap(PyIter_Next(s_iter))
if not k_ptr:
break
k = K.__from_py__(k_ptr)
pyobj.decref(k_ptr)
b.add(k)
pyobj.decref(s_iter)
return b

View File

@ -1 +1,3 @@
_py_init()
from internal.python import ensure_initialized
ensure_initialized()

View File

@ -23,7 +23,7 @@ def test_conversions():
{'ACGT'},
[['abc'], ['1.1', '2.2'], list[str]()]
)
assert T.__from_py__(t) == ({'a': 3.14, 'b': 2.123}, (222, 3.14))
assert T.__from_py__(t.p) == ({'a': 3.14, 'b': 2.123}, (222, 3.14))
test_conversions()
@test
@ -44,3 +44,23 @@ def test_pyargs():
assert str(mymodule.print_args_var(1, z=5, b=2)) == "a=1, b=2, c=1, args=(), kwargs={'z': 5}"
assert str(mymodule.print_args_var(1, *(1,2,3,4,5), z=5)) == "a=1, b=1, c=2, args=(3, 4, 5), kwargs={'z': 5}"
test_pyargs()
@test
def test_roundtrip(x: T, T: type):
assert T.__from_py__(x.__to_py__()) == x
test_roundtrip(42)
test_roundtrip(3.14)
test_roundtrip(True)
test_roundtrip(False)
test_roundtrip(byte(99))
test_roundtrip('hello world')
test_roundtrip('')
test_roundtrip(List[int]())
test_roundtrip([11, 22, 33])
test_roundtrip(Set[int]())
test_roundtrip({11, 22, 33})
test_roundtrip(Dict[str,int]())
test_roundtrip({'aa': 11, 'bb': 22, 'cc': 33})
test_roundtrip((11, 1.1, '11', [1, 1], {1}, {1: 1}))
test_roundtrip(())