From 12e8fe76660b2c203c134a8e9e1c3a25693b23f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ibrahim=20Numanagi=C4=87?= Date: Thu, 31 Mar 2022 01:22:26 -0700 Subject: [PATCH] @codon Python decorator and Python interop fixes (#19) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Codon decorator * Move to extra/cython, add error handling * Small fixes * CR * CR * Fix cython CI * Fix cython CI v2 * Fix cython CI v3 * Fix cython CI v4 * Fix cython CI v5 * Fix cython CI v6 * Fix cython CI v7 * Fix cython CI v8 * Fix cython CI v9 * Fix cython CI v10 * Fix cython CI v11 * CR * Fix CI * Fix CI * Fix CI * Fix CI * Fix CI Co-authored-by: Ishak Numanagić --- .github/actions/build-manylinux/entrypoint.sh | 11 + .github/workflows/ci.yml | 12 +- .gitignore | 2 + codon/app/main.cpp | 2 +- codon/compiler/jit.cpp | 11 +- codon/compiler/jit.h | 26 +- codon/parser/visitors/simplify/simplify.cpp | 2 +- extra/jupyter/jupyter.cpp | 2 +- extra/python/README.md | 14 + extra/python/pyproject.toml | 2 + extra/python/setup.py | 57 ++++ extra/python/src/__init__.py | 3 + extra/python/src/decorator.py | 174 +++++++++++ extra/python/src/jit.pxd | 16 + extra/python/src/jit.pyx | 31 ++ stdlib/internal/dlopen.codon | 2 +- stdlib/internal/python.codon | 275 ++++++++++-------- test/python/cython_jit.py | 87 ++++++ 18 files changed, 602 insertions(+), 127 deletions(-) create mode 100644 extra/python/README.md create mode 100644 extra/python/pyproject.toml create mode 100644 extra/python/setup.py create mode 100644 extra/python/src/__init__.py create mode 100644 extra/python/src/decorator.py create mode 100644 extra/python/src/jit.pxd create mode 100644 extra/python/src/jit.pyx create mode 100644 test/python/cython_jit.py diff --git a/.github/actions/build-manylinux/entrypoint.sh b/.github/actions/build-manylinux/entrypoint.sh index e8495ae6..4a3a69df 100755 --- a/.github/actions/build-manylinux/entrypoint.sh +++ b/.github/actions/build-manylinux/entrypoint.sh @@ -26,11 +26,22 @@ export LLVM_DIR=$(llvm/bin/llvm-config --cmakedir) -DCMAKE_CXX_COMPILER=${CXX}) cmake --build build --config Release -- VERBOSE=1 +# build cython +export PATH=$PATH:$(pwd)/llvm/bin +export LD_LIBRARY_PATH=$(pwd)/build:$LD_LIBRARY_PATH +export CODON_INCLUDE_DIR=$(pwd)/build/include +export CODON_LIB_DIR=$(pwd)/build +python3 -m pip install cython +python3 -m pip install -v extra/python + # test +export CODON_PATH=$(pwd)/stdlib ln -s build/libcodonrt.so . build/codon_test build/codon run test/core/helloworld.codon build/codon run test/core/exit.codon || if [[ $? -ne 42 ]]; then false; fi +export PYTHONPATH=$(pwd):$PYTHONPATH +python3 test/python/cython_jit.py # package export CODON_BUILD_ARCHIVE=codon-$(uname -s | awk '{print tolower($0)}')-$(uname -m).tar.gz diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c22050b..5805b3d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,6 +151,14 @@ jobs: CC: clang CXX: clang++ + - name: Build Cython + run: | + python -m pip install -v extra/python + env: + CC: clang + CXX: clang++ + PYTHONPATH: ./test/python + - name: Test run: | echo $CODON_PYTHON @@ -158,9 +166,11 @@ jobs: build/codon_test build/codon run test/core/helloworld.codon build/codon run test/core/exit.codon || if [[ $? -ne 42 ]]; then false; fi + python test/python/cython_jit.py env: CODON_PATH: ./stdlib - PYTHONPATH: ./test/python + PYTHONPATH: .:./test/python + LD_LIBRARY_PATH: ./build - name: Build Documentation run: | diff --git a/.gitignore b/.gitignore index 3bc55219..72101f09 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ *.pyc build/ build_*/ +extra/python/src/jit.cpp extra/jupyter/build/ # Packages # @@ -29,6 +30,7 @@ extra/jupyter/build/ *.rar *.tar *.zip +**/**.egg-info # Logs and databases # ###################### diff --git a/codon/app/main.cpp b/codon/app/main.cpp index 0387a8b1..d3dcb128 100644 --- a/codon/app/main.cpp +++ b/codon/app/main.cpp @@ -189,7 +189,7 @@ int runMode(const std::vector &args) { namespace { std::string jitExec(codon::jit::JIT *jit, const std::string &code) { - auto result = jit->exec(code); + auto result = jit->execute(code); if (auto err = result.takeError()) { std::string output; llvm::handleAllErrors( diff --git a/codon/compiler/jit.cpp b/codon/compiler/jit.cpp index 881e5a82..0a70cb03 100644 --- a/codon/compiler/jit.cpp +++ b/codon/compiler/jit.cpp @@ -100,7 +100,7 @@ llvm::Expected JIT::run(const ir::Func *input) { return getCapturedOutput(); } -llvm::Expected JIT::exec(const std::string &code) { +llvm::Expected JIT::execute(const std::string &code) { auto *cache = compiler->getCache(); ast::StmtPtr node = ast::parseCode(cache, JIT_FILENAME, code, /*startLine=*/0); @@ -160,5 +160,14 @@ llvm::Expected JIT::exec(const std::string &code) { } } +JITResult JIT::executeSafe(const std::string &code) { + auto result = this->execute(code); + if (auto err = result.takeError()) { + auto errorInfo = llvm::toString(std::move(err)); + return JITResult::error(errorInfo); + } + return JITResult::success(result.get()); +} + } // namespace jit } // namespace codon diff --git a/codon/compiler/jit.h b/codon/compiler/jit.h index ab3e42a7..61c92889 100644 --- a/codon/compiler/jit.h +++ b/codon/compiler/jit.h @@ -15,6 +15,29 @@ namespace codon { namespace jit { +struct JITResult { + std::string data; + bool isError; + + JITResult(): + data(""), isError(false) {} + + JITResult(const std::string &data, bool isError): + data(data), isError(isError) {} + + operator bool() { + return !this->isError; + } + + static JITResult success(const std::string &output) { + return JITResult(output, false); + } + + static JITResult error(const std::string &errorInfo) { + return JITResult(errorInfo, true); + } +}; + class JIT { private: std::unique_ptr compiler; @@ -29,7 +52,8 @@ public: llvm::Error init(); llvm::Expected run(const ir::Func *input); - llvm::Expected exec(const std::string &code); + llvm::Expected execute(const std::string &code); + JITResult executeSafe(const std::string &code); }; } // namespace jit diff --git a/codon/parser/visitors/simplify/simplify.cpp b/codon/parser/visitors/simplify/simplify.cpp index 6c149a94..07f068e1 100644 --- a/codon/parser/visitors/simplify/simplify.cpp +++ b/codon/parser/visitors/simplify/simplify.cpp @@ -98,7 +98,7 @@ SimplifyVisitor::apply(Cache *cache, const StmtPtr &node, const std::string &fil stdlib->isStdlibLoading = true; stdlib->moduleName = {ImportFile::STDLIB, stdlibPath->path, "__init__"}; auto baseTypeCode = - "@__internal__\n@tuple\nclass pyobj:\n p: Ptr[byte]\n" + "@__internal__\nclass pyobj:\n p: Ptr[byte]\n" "@__internal__\n@tuple\nclass str:\n ptr: Ptr[byte]\n len: int\n"; SimplifyVisitor(stdlib, preamble) .transform(parseCode(stdlib->cache, stdlibPath->path, baseTypeCode)); diff --git a/extra/jupyter/jupyter.cpp b/extra/jupyter/jupyter.cpp index eb068f28..f3ec268f 100644 --- a/extra/jupyter/jupyter.cpp +++ b/extra/jupyter/jupyter.cpp @@ -33,7 +33,7 @@ nl::json CodonJupyter::execute_request_impl(int execution_counter, const string bool silent, bool store_history, nl::json user_expressions, bool allow_stdin) { - auto result = jit->exec(code); + auto result = jit->execute(code); string failed; llvm::handleAllErrors( result.takeError(), diff --git a/extra/python/README.md b/extra/python/README.md new file mode 100644 index 00000000..8c2878cb --- /dev/null +++ b/extra/python/README.md @@ -0,0 +1,14 @@ +To install: + +```bash +$ pip install extra/python +``` + +To use: + +```python +from codon import codon, JitError + +@codon +def ... +``` diff --git a/extra/python/pyproject.toml b/extra/python/pyproject.toml new file mode 100644 index 00000000..c753eee1 --- /dev/null +++ b/extra/python/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["cython", "setuptools", "wheel"] diff --git a/extra/python/setup.py b/extra/python/setup.py new file mode 100644 index 00000000..e658a9c9 --- /dev/null +++ b/extra/python/setup.py @@ -0,0 +1,57 @@ +import os +import subprocess + +from Cython.Distutils import build_ext +from setuptools import setup +from setuptools.extension import Extension + + +def exists(executable): + ps = subprocess.run(["which", executable], stdout=subprocess.PIPE) + return ps.returncode == 0 + + +def get_output(*args): + ps = subprocess.run(args, stdout=subprocess.PIPE) + return ps.stdout.decode("utf8").strip() + + +from_root = lambda relpath: os.path.realpath(f"{os.getcwd()}/../../{relpath}") + +llvm_config: str +llvm_config_candidates = ["llvm-config-12", "llvm-config", from_root("llvm/bin/llvm-config")] +for candidate in llvm_config_candidates: + if exists(candidate): + llvm_config = candidate + break +else: + raise FileNotFoundError("Cannot find llvm-config; is llvm installed?") + +llvm_include_dir = get_output(llvm_config, "--includedir") +llvm_lib_dir = get_output(llvm_config, "--libdir") + +codon_include_dir = os.environ.get("CODON_INCLUDE_DIR", from_root("build/include")) +codon_lib_dir = os.environ.get("CODON_LIB_DIR", from_root("build")) + +print(f" {llvm_include_dir}, {llvm_lib_dir}") +print(f" {codon_include_dir}, {codon_lib_dir}") + +jit_extension = Extension( + "codon_jit", + sources=["src/jit.pyx"], + libraries=["codonc", "codonrt"], + language="c++", + extra_compile_args=["-w", "-std=c++17"], + extra_link_args=[f"-Wl,-rpath,{codon_lib_dir}"], + include_dirs=[llvm_include_dir, codon_include_dir], + library_dirs=[llvm_lib_dir, codon_lib_dir], +) + +setup( + name="codon", + version="0.1.0", + cmdclass={"build_ext": build_ext}, + ext_modules=[jit_extension], + packages=["codon"], + package_dir={"codon": "src"} +) diff --git a/extra/python/src/__init__.py b/extra/python/src/__init__.py new file mode 100644 index 00000000..30589239 --- /dev/null +++ b/extra/python/src/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["codon", "JitError"] + +from .decorator import codon, JitError diff --git a/extra/python/src/decorator.py b/extra/python/src/decorator.py new file mode 100644 index 00000000..2a668c66 --- /dev/null +++ b/extra/python/src/decorator.py @@ -0,0 +1,174 @@ +import ctypes +import inspect +import importlib +import importlib.util +import sys + +from typing import List, Tuple + +sys.setdlopenflags(sys.getdlopenflags() | ctypes.RTLD_GLOBAL) + +from codon_jit import Jit, JitError + + +separator = "__" + + +# codon wrapper stubs + + +def _wrapper_stub_init(): + from internal.python import ( + pyobj, + ensure_initialized, + Py_None, + PyImport_AddModule, + PyObject_GetAttrString, + PyObject_SetAttrString, + PyTuple_GetItem, + ) + + ensure_initialized(True) + + module = PyImport_AddModule("__codon_interop__".c_str()) + + +def _wrapper_stub_header(): + argt = PyObject_GetAttrString(module, "__codon_args__".c_str()) + + +def _wrapper_stub_footer_ret(): + PyObject_SetAttrString(module, "__codon_ret__".c_str(), ret.p) + + +def _wrapper_stub_footer_void(): + pyobj.incref(Py_None) + PyObject_SetAttrString(module, "__codon_ret__".c_str(), Py_None) + + +# helpers + + +def _reset_jit(): + global jit + jit = Jit() + lines = inspect.getsourcelines(_wrapper_stub_init)[0][1:] + jit.execute("".join([l[4:] for l in lines])) + + return jit + + +def _init(): + spec = importlib.machinery.ModuleSpec("__codon_interop__", None) + module = importlib.util.module_from_spec(spec) + exec("__codon_args__ = ()\n__codon_ret__ = None", module.__dict__) + sys.modules["__codon_interop__"] = module + exec("import __codon_interop__") + + return _reset_jit(), module + + +jit, module = _init() + + +def _obj_to_str(obj) -> str: + if inspect.isclass(obj): + lines = inspect.getsourcelines(obj)[0] + extra_spaces = lines[0].find("class") + obj_str = "".join(l[extra_spaces:] for l in lines) + elif callable(obj): + lines = inspect.getsourcelines(obj)[0] + extra_spaces = lines[0].find("@") + obj_str = "".join(l[extra_spaces:] for l in lines[1:]) + else: + raise TypeError(f"Function or class expected, got {type(obj).__name__}.") + return obj_str.replace("_@par", "@par") + + +def _obj_name(obj) -> str: + if inspect.isclass(obj) or callable(obj): + return obj.__name__ + else: + raise TypeError(f"Function or class expected, got {type(obj).__name__}.") + + +def _obj_name_full(obj) -> str: + obj_name = _obj_name(obj) + obj_name_stack = [obj_name] + frame = inspect.currentframe() + while frame.f_code.co_name != "codon": + frame = frame.f_back + frame = frame.f_back + while frame: + if frame.f_code.co_name == "": + obj_name_stack += [frame.f_globals["__name__"]] + break + else: + obj_name_stack += [frame.f_code.co_name] + frame = frame.f_back + return obj_name, separator.join(reversed(obj_name_stack)) + + +def _parse_decorated(obj): + return _obj_name(obj), _obj_to_str(obj) + + +def _get_type_info(obj) -> Tuple[List[str], str]: + sgn = inspect.signature(obj) + par = [p.annotation for p in sgn.parameters.values()] + ret = sgn.return_annotation + return par, ret + + +def _type_str(typ) -> str: + if typ in (int, float, str, bool): + return typ.__name__ + obj_str = str(typ) + return obj_str[7:] if obj_str.startswith("typing.") else obj_str + + +def _build_wrapper(obj, obj_name) -> str: + arg_types, ret_type = _get_type_info(obj) + arg_count = len(arg_types) + wrap_name = f"{obj_name}{separator}wrapped" + wrap = [f"def {wrap_name}():\n"] + wrap += inspect.getsourcelines(_wrapper_stub_header)[0][1:] + wrap += [ + f" arg_{i} = {_type_str(arg_types[i])}.__from_py__(pyobj(PyTuple_GetItem(argt, {i})))\n" + for i in range(arg_count) + ] + args = ", ".join([f"arg_{i}" for i in range(arg_count)]) + if ret_type != inspect._empty: + wrap += [f" ret = {obj_name}({args}).__to_py__()\n"] + wrap += inspect.getsourcelines(_wrapper_stub_footer_ret)[0][1:] + else: + wrap += [f" {obj_name}({args})\n"] + wrap += inspect.getsourcelines(_wrapper_stub_footer_void)[0][1:] + return wrap_name, "".join(wrap) + + +# decorator + + +def codon(obj): + try: + obj_name, obj_str = _parse_decorated(obj) + jit.execute(obj_str) + + wrap_name, wrap_str = _build_wrapper(obj, obj_name) + jit.execute(wrap_str) + except JitError as e: + _reset_jit() + raise + + def wrapped(*args, **kwargs): + try: + module.__codon_args__ = (*args, *kwargs.values()) + stdout = jit.execute(f"{wrap_name}()") + print(stdout, end="") + return module.__codon_ret__ + except JitError as e: + _reset_jit() + raise + + return wrapped diff --git a/extra/python/src/jit.pxd b/extra/python/src/jit.pxd new file mode 100644 index 00000000..21c608cb --- /dev/null +++ b/extra/python/src/jit.pxd @@ -0,0 +1,16 @@ +from libcpp.string cimport string + + +cdef extern from "llvm/Support/Error.h" namespace "llvm": + cdef cppclass Error + + +cdef extern from "codon/compiler/jit.h" namespace "codon::jit": + cdef cppclass JITResult: + string data + bint operator bool() + + cdef cppclass JIT: + JIT(string) + Error init() + JITResult executeSafe(string) diff --git a/extra/python/src/jit.pyx b/extra/python/src/jit.pyx new file mode 100644 index 00000000..aa2546a8 --- /dev/null +++ b/extra/python/src/jit.pyx @@ -0,0 +1,31 @@ +# distutils: language=c++ +# cython: language_level=3 +# cython: c_string_type=unicode +# cython: c_string_encoding=ascii + +from cython.operator import dereference as dref +from libcpp.string cimport string + +from src.jit cimport JIT, JITResult + + +class JitError(Exception): + pass + + +cdef class Jit: + cdef JIT* jit + + def __cinit__(self): + self.jit = new JIT(b"codon jit") + dref(self.jit).init() + + def __dealloc__(self): + del self.jit + + def execute(self, code: str) -> str: + result = dref(self.jit).executeSafe(code) + if result: + return result.data + else: + raise JitError(result.data) diff --git a/stdlib/internal/dlopen.codon b/stdlib/internal/dlopen.codon index b7cf65b2..ecd59ba9 100644 --- a/stdlib/internal/dlopen.codon +++ b/stdlib/internal/dlopen.codon @@ -31,7 +31,7 @@ def dlerror() -> str: def dlopen(name: str, flag: int = RTLD_NOW | RTLD_GLOBAL) -> cobj: - h = c_dlopen(cobj(0) if name == "" else name.c_str(), flag) + h = c_dlopen(cobj() if name == "" else name.c_str(), flag) if h == cobj(): raise CError(dlerror()) return h diff --git a/stdlib/internal/python.codon b/stdlib/internal/python.codon index 6ef03a3c..91f9f908 100644 --- a/stdlib/internal/python.codon +++ b/stdlib/internal/python.codon @@ -1,44 +1,47 @@ # (c) 2022 Exaloop Inc. All rights reserved. import os + from internal.dlopen import * -PyUnicode_AsEncodedString = Function[[cobj, cobj, cobj], cobj](cobj()) -PyBytes_AsString = Function[[cobj], cobj](cobj()) -PyErr_Fetch = Function[[Ptr[cobj], Ptr[cobj], Ptr[cobj]], void](cobj()) -PyObject_GetAttrString = Function[[cobj, cobj], cobj](cobj()) -PyObject_GetAttr = Function[[cobj, cobj], cobj](cobj()) -PyObject_Str = Function[[cobj], cobj](cobj()) -PyRun_SimpleString = Function[[cobj], void](cobj()) -Py_IncRef = Function[[cobj], void](cobj()) Py_DecRef = Function[[cobj], void](cobj()) -PyObject_Call = Function[[cobj, cobj, cobj], cobj](cobj()) -PyObject_SetAttrString = Function[[cobj, cobj, cobj], cobj](cobj()) -PyObject_Length = Function[[cobj], int](cobj()) +Py_IncRef = Function[[cobj], void](cobj()) Py_Initialize = Function[[], void](cobj()) -PyImport_ImportModule = Function[[cobj], cobj](cobj()) -PyLong_FromLong = Function[[int], cobj](cobj()) -PyLong_AsLong = Function[[cobj], int](cobj()) -PyFloat_FromDouble = Function[[float], cobj](cobj()) -PyFloat_AsDouble = Function[[cobj], float](cobj()) +Py_None = cobj() PyBool_FromLong = Function[[int], cobj](cobj()) -PyObject_IsTrue = Function[[cobj], int](cobj()) -PyUnicode_DecodeFSDefaultAndSize = Function[[cobj, int], cobj](cobj()) -PyTuple_New = Function[[int], cobj](cobj()) -PyTuple_SetItem = Function[[cobj, int, cobj], void](cobj()) -PyTuple_GetItem = Function[[cobj, int], cobj](cobj()) +PyBytes_AsString = Function[[cobj], cobj](cobj()) +PyDict_New = Function[[], cobj](cobj()) +PyDict_Next = Function[[cobj, Ptr[int], Ptr[cobj], Ptr[cobj]], int](cobj()) +PyDict_SetItem = Function[[cobj, cobj, cobj], cobj](cobj()) +PyErr_Fetch = Function[[Ptr[cobj], Ptr[cobj], Ptr[cobj]], void](cobj()) +PyFloat_AsDouble = Function[[cobj], float](cobj()) +PyFloat_FromDouble = Function[[float], cobj](cobj()) +PyImport_AddModule = Function[[cobj], cobj](cobj()) +PyImport_ImportModule = Function[[cobj], cobj](cobj()) +PyIter_Next = Function[[cobj], cobj](cobj()) +PyList_GetItem = Function[[cobj, int], cobj](cobj()) PyList_New = Function[[int], cobj](cobj()) PyList_SetItem = Function[[cobj, int, cobj], cobj](cobj()) -PyList_GetItem = Function[[cobj, int], cobj](cobj()) -PySet_New = Function[[cobj], cobj](cobj()) -PySet_Add = Function[[cobj, cobj], cobj](cobj()) -PyDict_New = Function[[], cobj](cobj()) -PyDict_SetItem = Function[[cobj, cobj, cobj], cobj](cobj()) -PyDict_Next = Function[[cobj, Ptr[int], Ptr[cobj], Ptr[cobj]], int](cobj()) +PyLong_AsLong = Function[[cobj], int](cobj()) +PyLong_FromLong = Function[[int], cobj](cobj()) +PyObject_Call = Function[[cobj, cobj, cobj], cobj](cobj()) +PyObject_GetAttr = Function[[cobj, cobj], cobj](cobj()) +PyObject_GetAttrString = Function[[cobj, cobj], cobj](cobj()) PyObject_GetIter = Function[[cobj], cobj](cobj()) -PyIter_Next = Function[[cobj], cobj](cobj()) PyObject_HasAttrString = Function[[cobj, cobj], int](cobj()) -PyImport_AddModule = Function[[cobj], cobj](cobj()) +PyObject_IsTrue = Function[[cobj], int](cobj()) +PyObject_Length = Function[[cobj], int](cobj()) +PyObject_SetAttrString = Function[[cobj, cobj, cobj], cobj](cobj()) +PyObject_Str = Function[[cobj], cobj](cobj()) +PyRun_SimpleString = Function[[cobj], void](cobj()) +PySet_Add = Function[[cobj, cobj], cobj](cobj()) +PySet_New = Function[[cobj], cobj](cobj()) +PyTuple_GetItem = Function[[cobj, int], cobj](cobj()) +PyTuple_New = Function[[int], cobj](cobj()) +PyTuple_SetItem = Function[[cobj, int, cobj], void](cobj()) +PyUnicode_AsEncodedString = Function[[cobj, cobj, cobj], cobj](cobj()) +PyUnicode_DecodeFSDefaultAndSize = Function[[cobj, int], cobj](cobj()) +PyUnicode_FromString = Function[[cobj], cobj](cobj()) _PY_MODULE_CACHE = Dict[str, pyobj]() @@ -77,102 +80,125 @@ def __codon_repr__(fig): _PY_INITIALIZED = False -def init(): +def init_dl_handles(py_handle: cobj): + global Py_DecRef + global Py_IncRef + global Py_Initialize + global Py_None + global PyBool_FromLong + global PyBytes_AsString + global PyDict_New + global PyDict_Next + global PyDict_SetItem + global PyErr_Fetch + global PyFloat_AsDouble + global PyFloat_FromDouble + global PyImport_AddModule + global PyImport_ImportModule + global PyIter_Next + global PyList_GetItem + global PyList_New + global PyList_SetItem + global PyLong_AsLong + global PyLong_FromLong + global PyObject_Call + global PyObject_GetAttr + global PyObject_GetAttrString + global PyObject_GetIter + global PyObject_HasAttrString + global PyObject_IsTrue + global PyObject_Length + global PyObject_SetAttrString + global PyObject_Str + global PyRun_SimpleString + global PySet_Add + global PySet_New + global PyTuple_GetItem + global PyTuple_New + global PyTuple_SetItem + global PyUnicode_AsEncodedString + global PyUnicode_DecodeFSDefaultAndSize + global PyUnicode_FromString + Py_DecRef = dlsym(py_handle, "Py_DecRef") + Py_IncRef = dlsym(py_handle, "Py_IncRef") + Py_Initialize = dlsym(py_handle, "Py_Initialize") + Py_None = dlsym(py_handle, "_Py_NoneStruct") + PyBool_FromLong = dlsym(py_handle, "PyBool_FromLong") + PyBytes_AsString = dlsym(py_handle, "PyBytes_AsString") + PyDict_New = dlsym(py_handle, "PyDict_New") + PyDict_Next = dlsym(py_handle, "PyDict_Next") + PyDict_SetItem = dlsym(py_handle, "PyDict_SetItem") + PyErr_Fetch = dlsym(py_handle, "PyErr_Fetch") + PyFloat_AsDouble = dlsym(py_handle, "PyFloat_AsDouble") + PyFloat_FromDouble = dlsym(py_handle, "PyFloat_FromDouble") + PyImport_AddModule = dlsym(py_handle, "PyImport_AddModule") + PyImport_ImportModule = dlsym(py_handle, "PyImport_ImportModule") + PyIter_Next = dlsym(py_handle, "PyIter_Next") + PyList_GetItem = dlsym(py_handle, "PyList_GetItem") + PyList_New = dlsym(py_handle, "PyList_New") + PyList_SetItem = dlsym(py_handle, "PyList_SetItem") + PyLong_AsLong = dlsym(py_handle, "PyLong_AsLong") + PyLong_FromLong = dlsym(py_handle, "PyLong_FromLong") + PyObject_Call = dlsym(py_handle, "PyObject_Call") + PyObject_GetAttr = dlsym(py_handle, "PyObject_GetAttr") + PyObject_GetAttrString = dlsym(py_handle, "PyObject_GetAttrString") + PyObject_GetIter = dlsym(py_handle, "PyObject_GetIter") + PyObject_HasAttrString = dlsym(py_handle, "PyObject_HasAttrString") + PyObject_IsTrue = dlsym(py_handle, "PyObject_IsTrue") + PyObject_Length = dlsym(py_handle, "PyObject_Length") + PyObject_SetAttrString = dlsym(py_handle, "PyObject_SetAttrString") + PyObject_Str = dlsym(py_handle, "PyObject_Str") + PyRun_SimpleString = dlsym(py_handle, "PyRun_SimpleString") + PySet_Add = dlsym(py_handle, "PySet_Add") + PySet_New = dlsym(py_handle, "PySet_New") + PyTuple_GetItem = dlsym(py_handle, "PyTuple_GetItem") + PyTuple_New = dlsym(py_handle, "PyTuple_New") + PyTuple_SetItem = dlsym(py_handle, "PyTuple_SetItem") + PyUnicode_AsEncodedString = dlsym(py_handle, "PyUnicode_AsEncodedString") + PyUnicode_DecodeFSDefaultAndSize = dlsym(py_handle, "PyUnicode_DecodeFSDefaultAndSize") + PyUnicode_FromString = dlsym(py_handle, "PyUnicode_FromString") + + +def init(python_loaded: bool = False): global _PY_INITIALIZED if _PY_INITIALIZED: return - LD = os.getenv("CODON_PYTHON", default=f"libpython.{dlext()}") - hnd = dlopen(LD, RTLD_LOCAL | RTLD_NOW) + py_handle: cobj + if python_loaded: + py_handle = dlopen("", RTLD_LOCAL | RTLD_NOW) + else: + LD = os.getenv("CODON_PYTHON", default="libpython." + dlext()) + py_handle = dlopen(LD, RTLD_LOCAL | RTLD_NOW) - global PyUnicode_AsEncodedString - PyUnicode_AsEncodedString = dlsym(hnd, "PyUnicode_AsEncodedString") - global PyBytes_AsString - PyBytes_AsString = dlsym(hnd, "PyBytes_AsString") - global PyErr_Fetch - PyErr_Fetch = dlsym(hnd, "PyErr_Fetch") - global PyObject_GetAttrString - PyObject_GetAttrString = dlsym(hnd, "PyObject_GetAttrString") - global PyObject_GetAttr - PyObject_GetAttr = dlsym(hnd, "PyObject_GetAttr") - global PyObject_Str - PyObject_Str = dlsym(hnd, "PyObject_Str") - global PyRun_SimpleString - PyRun_SimpleString = dlsym(hnd, "PyRun_SimpleString") - global Py_IncRef - Py_IncRef = dlsym(hnd, "Py_IncRef") - global Py_DecRef - Py_DecRef = dlsym(hnd, "Py_DecRef") - global PyObject_Call - PyObject_Call = dlsym(hnd, "PyObject_Call") - global PyObject_SetAttrString - PyObject_SetAttrString = dlsym(hnd, "PyObject_SetAttrString") - global PyObject_Length - PyObject_Length = dlsym(hnd, "PyObject_Length") - global Py_Initialize - Py_Initialize = dlsym(hnd, "Py_Initialize") - global PyImport_ImportModule - PyImport_ImportModule = dlsym(hnd, "PyImport_ImportModule") - global PyLong_FromLong - PyLong_FromLong = dlsym(hnd, "PyLong_FromLong") - global PyLong_AsLong - PyLong_AsLong = dlsym(hnd, "PyLong_AsLong") - global PyFloat_FromDouble - PyFloat_FromDouble = dlsym(hnd, "PyFloat_FromDouble") - global PyFloat_AsDouble - PyFloat_AsDouble = dlsym(hnd, "PyFloat_AsDouble") - global PyBool_FromLong - PyBool_FromLong = dlsym(hnd, "PyBool_FromLong") - global PyObject_IsTrue - PyObject_IsTrue = dlsym(hnd, "PyObject_IsTrue") - global PyUnicode_DecodeFSDefaultAndSize - PyUnicode_DecodeFSDefaultAndSize = dlsym(hnd, "PyUnicode_DecodeFSDefaultAndSize") - global PyTuple_New - PyTuple_New = dlsym(hnd, "PyTuple_New") - global PyTuple_SetItem - PyTuple_SetItem = dlsym(hnd, "PyTuple_SetItem") - global PyTuple_GetItem - PyTuple_GetItem = dlsym(hnd, "PyTuple_GetItem") - global PyList_New - PyList_New = dlsym(hnd, "PyList_New") - global PyList_SetItem - PyList_SetItem = dlsym(hnd, "PyList_SetItem") - global PyList_GetItem - PyList_GetItem = dlsym(hnd, "PyList_GetItem") - global PySet_New - PySet_New = dlsym(hnd, "PySet_New") - global PySet_Add - PySet_Add = dlsym(hnd, "PySet_Add") - global PyDict_New - PyDict_New = dlsym(hnd, "PyDict_New") - global PyDict_SetItem - PyDict_SetItem = dlsym(hnd, "PyDict_SetItem") - global PyDict_Next - PyDict_Next = dlsym(hnd, "PyDict_Next") - global PyObject_GetIter - PyObject_GetIter = dlsym(hnd, "PyObject_GetIter") - global PyIter_Next - PyIter_Next = dlsym(hnd, "PyIter_Next") - global PyObject_HasAttrString - PyObject_HasAttrString = dlsym(hnd, "PyObject_HasAttrString") - global PyImport_AddModule - PyImport_AddModule = dlsym(hnd, "PyImport_AddModule") + init_dl_handles(py_handle) + + if not python_loaded: + Py_Initialize() - Py_Initialize() PyRun_SimpleString(_PY_INIT.c_str()) _PY_INITIALIZED = True -def ensure_initialized(): +def ensure_initialized(python_loaded: bool = False): if not _PY_INITIALIZED: - init() - # raise ValueError("Python not initialized; make sure to 'import python'") + init(python_loaded) @extend class pyobj: - def __new__(p: Ptr[byte]) -> pyobj: - return (p,) + @__internal__ + def __new__() -> pyobj: + pass + + def __raw__(self) -> Ptr[byte]: + return __internal__.class_raw(self) + + def __init__(self, p: Ptr[byte]): + self.p = p + + def __del__(self): + self.decref() def _getattr(self, name: str) -> pyobj: return pyobj.exc_wrap(pyobj(PyObject_GetAttrString(self.p, name.c_str()))) @@ -203,8 +229,7 @@ class pyobj: raise ValueError("Python object is not iterable") while i := PyIter_Next(it): yield pyobj(pyobj.exc_wrap(i)) - pyobj(i).decref() - pyobj(it).decref() + pyobj.decref(it) pyobj.exc_check() def to_str(self, errors: str, empty: str = "") -> str: @@ -212,7 +237,7 @@ class pyobj: if obj == cobj(): return empty bts = PyBytes_AsString(obj) - pyobj(obj).decref() + pyobj.decref(obj) return str.from_ptr(bts) def exc_check(): @@ -221,14 +246,12 @@ class pyobj: if ptype != cobj(): py_msg = PyObject_Str(pvalue) if pvalue != cobj() else pvalue msg = pyobj(py_msg).to_str("ignore", "") - typ = pyobj.to_str( - pyobj(PyObject_GetAttrString(ptype, "__name__".c_str())), "ignore" - ) + typ = pyobj(PyObject_GetAttrString(ptype, "__name__".c_str())).to_str("ignore") - pyobj(ptype).decref() - pyobj(pvalue).decref() - pyobj(ptraceback).decref() - pyobj(py_msg).decref() + pyobj.decref(ptype) + pyobj.decref(pvalue) + pyobj.decref(ptraceback) + pyobj.decref(py_msg) raise PyError(msg, typ) @@ -239,9 +262,21 @@ class pyobj: def incref(self): Py_IncRef(self.p) + def incref(obj: pyobj): + Py_IncRef(obj.p) + + def incref(ptr: Ptr[byte]): + Py_IncRef(ptr) + def decref(self): Py_DecRef(self.p) + def decref(obj: pyobj): + Py_DecRef(obj.p) + + def decref(ptr: Ptr[byte]): + Py_DecRef(ptr) + def __call__(self, *args, **kwargs) -> pyobj: names = iter(kwargs.__dict__()) kws = dict[str, pyobj]() diff --git a/test/python/cython_jit.py b/test/python/cython_jit.py new file mode 100644 index 00000000..ede06865 --- /dev/null +++ b/test/python/cython_jit.py @@ -0,0 +1,87 @@ +import sys +from io import StringIO +from typing import Dict, List, Tuple + +from codon import codon, JitError + + +# test stdout + + +def test_stdout(): + @codon + def run(): + print("hello world!") + + try: + output = StringIO() + sys.stdout = output + run() + assert output.getvalue() == "hello world!\n" + finally: + sys.stdout = sys.__stdout__ + + +test_stdout() + + +# test error handling + + +def test_error_handling(): + @codon + def run() -> int: + return "not int" + + try: + r = run() + except JitError: + assert True + except BaseException: + assert False + else: + assert False + + +test_error_handling() + + +# test type validity + + +def test_return_type(): + @codon + def run() -> Tuple[int, str, float, List[int], Dict[str, int]]: + return (1, "str", 2.45, [1, 2, 3], {"a": 1, "b": 2}) + + r = run() + assert type(r) == tuple + assert type(r[0]) == int + assert type(r[1]) == str + assert type(r[2]) == float + assert type(r[3]) == list + assert len(r[3]) == 3 + assert type(r[3][0]) == int + assert type(r[4]) == dict + assert len(r[4].items()) == 2 + assert type(next(iter(r[4].keys()))) == str + assert type(next(iter(r[4].values()))) == int + + +test_return_type() + + +def test_param_types(): + @codon + def run(a: int, b: Tuple[int, int], c: List[int], d: Dict[str, int]) -> int: + s = 0 + for v in [a, *b, *c, *d.values()]: + s += v + return s + + r = run(1, (2, 3), [4, 5, 6], dict(a=7, b=8, c=9)) + assert type(r) == int + assert r == 45 + + +test_param_types()