"""
Build a c-extension module on-the-fly in tests.
See build_and_import_extensions for usage hints
"""
import os
import pathlib
import subprocess
import sys
import sysconfig
import textwrap
__all__ = ['build_and_import_extension', 'compile_extension_module']
def build_and_import_extension(
modname, functions, *, prologue="", build_dir=None,
include_dirs=[], more_init=""):
"""
Build and imports a c-extension module `modname` from a list of function
fragments `functions`.
Parameters
----------
functions : list of fragments
Each fragment is a sequence of func_name, calling convention, snippet.
prologue : string
Code to precede the rest, usually extra ``#include`` or ``#define``
macros.
build_dir : pathlib.Path
Where to build the module, usually a temporary directory
include_dirs : list
Extra directories to find include files when compiling
more_init : string
Code to appear in the module PyMODINIT_FUNC
Returns
-------
out: module
The module will have been loaded and is ready for use
Examples
--------
>>> functions = [("test_bytes", "METH_O", \"\"\"
if ( !PyBytesCheck(args)) {
Py_RETURN_FALSE;
}
Py_RETURN_TRUE;
\"\"\")]
>>> mod = build_and_import_extension("testme", functions)
>>> assert not mod.test_bytes('abc')
>>> assert mod.test_bytes(b'abc')
"""
body = prologue + _make_methods(functions, modname)
init = """
PyObject *mod = PyModule_Create(&moduledef);
#ifdef Py_GIL_DISABLED
PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED);
#endif
"""
if not build_dir:
build_dir = pathlib.Path('.')
if more_init:
init += """#define INITERROR return NULL
"""
init += more_init
init += "\nreturn mod;"
source_string = _make_source(modname, init, body)
try:
mod_so = compile_extension_module(
modname, build_dir, include_dirs, source_string)
except Exception as e:
# shorten the exception chain
raise RuntimeError(f"could not compile in {build_dir}:") from e
import importlib.util
spec = importlib.util.spec_from_file_location(modname, mod_so)
foo = importlib.util.module_from_spec(spec)
spec.loader.exec_module(foo)
return foo
def compile_extension_module(
name, builddir, include_dirs,
source_string, libraries=[], library_dirs=[]):
"""
Build an extension module and return the filename of the resulting
native code file.
Parameters
----------
name : string
name of the module, possibly including dots if it is a module inside a
package.
builddir : pathlib.Path
Where to build the module, usually a temporary directory
include_dirs : list
Extra directories to find include files when compiling
libraries : list
Libraries to link into the extension module
library_dirs: list
Where to find the libraries, ``-L`` passed to the linker
"""
modname = name.split('.')[-1]
dirname = builddir / name
dirname.mkdir(exist_ok=True)
cfile = _convert_str_to_file(source_string, dirname)
include_dirs = include_dirs + [sysconfig.get_config_var('INCLUDEPY')]
return _c_compile(
cfile, outputfilename=dirname / modname,
include_dirs=include_dirs, libraries=[], library_dirs=[],
)
def _convert_str_to_file(source, dirname):
"""Helper function to create a file ``source.c`` in `dirname` that contains
the string in `source`. Returns the file name
"""
filename = dirname / 'source.c'
with filename.open('w') as f:
f.write(str(source))
return filename
def _make_methods(functions, modname):
""" Turns the name, signature, code in functions into complete functions
and lists them in a methods_table. Then turns the methods_table into a
``PyMethodDef`` structure and returns the resulting code fragment ready
for compilation
"""
methods_table = []
codes = []
for funcname, flags, code in functions:
cfuncname = "%s_%s" % (modname, funcname)
if 'METH_KEYWORDS' in flags:
signature = '(PyObject *self, PyObject *args, PyObject *kwargs)'
else:
signature = '(PyObject *self, PyObject *args)'
methods_table.append(
"{\"%s\", (PyCFunction)%s, %s}," % (funcname, cfuncname, flags))
func_code = """
static PyObject* {cfuncname}{signature}
{{
{code}
}}
""".format(cfuncname=cfuncname, signature=signature, code=code)
codes.append(func_code)
body = "\n".join(codes) + """
static PyMethodDef methods[] = {
%(methods)s
{ NULL }
};
static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
"%(modname)s", /* m_name */
NULL, /* m_doc */
-1, /* m_size */
methods, /* m_methods */
};
""" % dict(methods='\n'.join(methods_table), modname=modname)
return body
def _make_source(name, init, body):
""" Combines the code fragments into source code ready to be compiled
"""
code = """
#include <Python.h>
%(body)s
PyMODINIT_FUNC
PyInit_%(name)s(void) {
%(init)s
}
""" % dict(
name=name, init=init, body=body,
)
return code
def _c_compile(cfile, outputfilename, include_dirs=[], libraries=[],
library_dirs=[]):
if sys.platform == 'win32':
compile_extra = ["/we4013"]
link_extra = ["/LIBPATH:" + os.path.join(sys.base_prefix, 'libs')]
elif sys.platform.startswith('linux'):
compile_extra = [
"-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
link_extra = []
else:
compile_extra = link_extra = []
pass
if sys.platform == 'win32':
link_extra = link_extra + ['/DEBUG'] # generate .pdb file
if sys.platform == 'darwin':
# support Fink & Darwinports
for s in ('/sw/', '/opt/local/'):
if (s + 'include' not in include_dirs
and os.path.exists(s + 'include')):
include_dirs.append(s + 'include')
if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'):
library_dirs.append(s + 'lib')
outputfilename = outputfilename.with_suffix(get_so_suffix())
build(
cfile, outputfilename,
compile_extra, link_extra,
include_dirs, libraries, library_dirs)
return outputfilename
def build(cfile, outputfilename, compile_extra, link_extra,
include_dirs, libraries, library_dirs):
"use meson to build"
build_dir = cfile.parent / "build"
os.makedirs(build_dir, exist_ok=True)
so_name = outputfilename.parts[-1]
with open(cfile.parent / "meson.build", "wt") as fid:
includes = ['-I' + d for d in include_dirs]
link_dirs = ['-L' + d for d in library_dirs]
fid.write(textwrap.dedent(f"""\
project('foo', 'c')
shared_module('{so_name}', '{cfile.parts[-1]}',
c_args: {includes} + {compile_extra},
link_args: {link_dirs} + {link_extra},
link_with: {libraries},
name_prefix: '',
name_suffix: 'dummy',
)
"""))
if sys.platform == "win32":
subprocess.check_call(["meson", "setup",
"--buildtype=release",
"--vsenv", ".."],
cwd=build_dir,
)
else:
subprocess.check_call(["meson", "setup", "--vsenv", ".."],
cwd=build_dir
)
subprocess.check_call(["meson", "compile"], cwd=build_dir)
os.rename(str(build_dir / so_name) + ".dummy", cfile.parent / so_name)
def get_so_suffix():
ret = sysconfig.get_config_var('EXT_SUFFIX')
assert ret
return ret