diff --git a/codepy/bpl.py b/codepy/bpl.py index 1e09c9f..bf39830 100644 --- a/codepy/bpl.py +++ b/codepy/bpl.py @@ -58,7 +58,7 @@ def expose_vector_type(self, name, py_name=None): if py_name is None: py_name = name - from cgen import (Block, Typedef, Line, Statement, Value) + from cgen import Block, Line, Statement, Typedef, Value self.init_body.append( Block([ @@ -134,8 +134,7 @@ def generate(self): module line-by-line. """ - from cgen import Block, Module, Include, Line, Define, \ - PrivateNamespace + from cgen import Block, Define, Include, Line, Module, PrivateNamespace body = [] diff --git a/codepy/cuda.py b/codepy/cuda.py index c9b7b0d..58de7d3 100644 --- a/codepy/cuda.py +++ b/codepy/cuda.py @@ -1,4 +1,6 @@ import cgen + + """Convenience interface for using CodePy with CUDA""" @@ -70,8 +72,7 @@ def compile(self, host_toolchain, nvcc_toolchain, host_code = "{}\n".format(self.boost_module.generate()) device_code = "{}\n".format(self.generate()) - from codepy.jit import compile_from_string - from codepy.jit import link_extension + from codepy.jit import compile_from_string, link_extension local_host_kwargs = kwargs.copy() local_host_kwargs.update(host_kwargs) diff --git a/codepy/elementwise.py b/codepy/elementwise.py index 3401b3b..1d882c1 100644 --- a/codepy/elementwise.py +++ b/codepy/elementwise.py @@ -4,10 +4,11 @@ __copyright__ = "Copyright (C) 2009 Andreas Kloeckner" -from pytools import memoize import numpy from cgen import POD, Value, dtype_to_ctype +from pytools import memoize + class Argument: def __init__(self, dtype, name): @@ -46,11 +47,11 @@ def struct_char(self): def get_elwise_module_descriptor(arguments, operation, name="kernel"): - from codepy.bpl import BoostPythonModule + from cgen import ( + POD, Block, For, FunctionBody, FunctionDeclaration, Include, Initializer, + Line, Statement, Struct, Value) - from cgen import FunctionBody, FunctionDeclaration, \ - Value, POD, Struct, For, Initializer, Include, Statement, \ - Line, Block + from codepy.bpl import BoostPythonModule S = Statement # noqa: N806 diff --git a/codepy/jit.py b/codepy/jit.py index c1fcb87..136713a 100644 --- a/codepy/jit.py +++ b/codepy/jit.py @@ -26,15 +26,18 @@ THE SOFTWARE. """ +import logging +from dataclasses import dataclass +from typing import List, NamedTuple + from codepy import CompileError -from pytools import Record -import logging + logger = logging.getLogger(__name__) def _erase_dir(dir): - from os import listdir, unlink, rmdir + from os import listdir, rmdir, unlink from os.path import join for name in listdir(dir): unlink(join(dir, name)) @@ -211,8 +214,16 @@ class _InvalidInfoFile(RuntimeError): pass -class _SourceInfo(Record): - pass +class _Dependency(NamedTuple): + name: str + mtime: int + md5: str + + +@dataclass(frozen=True) +class _SourceInfo: + dependencies: List[NamedTuple] + source_name: str def compile_from_string(toolchain, name, source_string, @@ -308,10 +319,9 @@ def get_file_md5sum(fname): return checksum.hexdigest() def get_dep_structure(source_paths): - deps = list(toolchain.get_dependencies(source_paths)) - deps.sort() - return [(dep, os.stat(dep).st_mtime, get_file_md5sum(dep)) for dep in deps - if dep not in source_paths] + deps = toolchain.get_dependencies(source_paths) + return [_Dependency(dep, os.stat(dep).st_mtime, get_file_md5sum(dep)) + for dep in sorted(deps) if dep not in source_paths] def write_source(name): for i, source in enumerate(source_string): @@ -479,6 +489,8 @@ def link_extension(toolchain, objects, mod_name, cache_dir=None, from pytools import MovedFunctionDeprecationWrapper # noqa: E402 + from codepy.toolchain import guess_toolchain as _gtc # noqa: E402 + guess_toolchain = MovedFunctionDeprecationWrapper(_gtc) diff --git a/codepy/libraries.py b/codepy/libraries.py index 9723cff..125408b 100644 --- a/codepy/libraries.py +++ b/codepy/libraries.py @@ -10,8 +10,8 @@ def search_on_path(filenames): """Find file on system path.""" # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52224 - from os.path import exists, join, abspath - from os import pathsep, environ + from os import environ, pathsep + from os.path import abspath, exists, join search_path = environ["PATH"] diff --git a/codepy/toolchain.py b/codepy/toolchain.py index c4f8d56..624d7a8 100644 --- a/codepy/toolchain.py +++ b/codepy/toolchain.py @@ -1,6 +1,5 @@ """Toolchains for Just-in-time Python extension compilation.""" - __copyright__ = """ "Copyright (C) 2008,9 Andreas Kloeckner, Bryan Catanzaro """ @@ -25,28 +24,40 @@ THE SOFTWARE. """ +from abc import ABC, abstractmethod +from dataclasses import dataclass, replace +from typing import AbstractSet, Any, List from codepy import CompileError -from pytools import Record -class Toolchain(Record): +@dataclass(frozen=True) +class Toolchain(ABC): """Abstract base class for tools used to link dynamic Python modules.""" - def __init__(self, *args, **kwargs): - if "features" not in kwargs: - kwargs["features"] = set() - Record.__init__(self, *args, **kwargs) + #: A list of directories where libraries are found. + library_dirs: List[str] + #: A list of libraries used. + libraries: List[str] + #: A list of directories from which to include header files. + include_dirs: List[str] + + features: AbstractSet[str] + + def copy(self, **kwargs: Any) -> "Toolchain": + from warnings import warn + warn(f"'{type(self).__name__}.copy' is deprecated. This is now a " + "dataclass and should be used with the standard 'replace'.", + DeprecationWarning, stacklevel=2) + + return replace(self, **kwargs) + @abstractmethod def get_version(self): """Return a string describing the exact version of the tools (compilers etc.) involved in this toolchain. - - Implemented by subclasses. """ - raise NotImplementedError - def abi_id(self): """Return a picklable Python object that describes the ABI (Python version, compiler versions, etc.) against which a Python module is compiled. @@ -80,50 +91,38 @@ def add_library(self, feature, include_dirs, library_dirs, libraries): self.libraries = libraries + self.libraries + @abstractmethod def get_dependencies(self, source_files): - """Return a list of header files referred to by *source_files. - - Implemented by subclasses. - """ - - raise NotImplementedError + """Return a list of header files referred to by *source_files*.""" + @abstractmethod def build_extension(self, ext_file, source_files, debug=False): """Create the extension file *ext_file* from *source_files* by invoking the toolchain. Raise :exc:`~codepy.jit.CompileError` in case of error. If *debug* is True, print the commands executed. - - Implemented by subclasses. """ - raise NotImplementedError - + @abstractmethod def build_object(self, obj_file, source_files, debug=False): """Build a compiled object *obj_file* from *source_files* by invoking the toolchain. Raise :exc:`CompileError` in case of error. If *debug* is True, print the commands executed. - - Implemented by subclasses. """ - raise NotImplementedError - + @abstractmethod def link_extension(self, ext_file, object_files, debug=False): """Create the extension file *ext_file* from *object_files* by invoking the toolchain. Raise :exc:`CompileError` in case of error. If *debug* is True, print the commands executed. - - Implemented by subclasses. """ - raise NotImplementedError - + @abstractmethod def with_optimization_level(self, level, **extra): """Return a new Toolchain object with the optimization level set to `level` , on the scale defined by the gcc -O option. @@ -133,16 +132,36 @@ def with_optimization_level(self, level, **extra): simply ignore it. Level may also be "debug" to specify a debug build. - - Implemented by subclasses. """ - raise NotImplementedError - # {{{ gcc-like tool chain +@dataclass(frozen=True) class GCCLikeToolchain(Toolchain): + #: Path to the C compiler. + cc: str + #: Path to linker. + ld: str + + #: A list of flags to pass to the C compiler. + cflags: List[str] + #: A list of linker flags. + ldflags: List[str] + + #: A list of defines to pass to the C compiler. + defines: List[str] + # A list of variables to undefine. + undefines: List[str] + + #: Extension for shared library generated by the compiler. + so_ext: str + #: Extension of the object file generated by the compiler. + o_ext: str + + def _cmdline(self, source_files, object=False): + raise NotImplementedError + def get_version(self): result, stdout, stderr = call_capture_output([self.cc, "--version"]) if result != 0: @@ -233,6 +252,7 @@ def link_extension(self, ext_file, object_files, debug=False): # {{{ gcc toolchain +@dataclass(frozen=True) class GCCToolchain(GCCLikeToolchain): def get_version_tuple(self): ver = self.get_version() @@ -288,13 +308,14 @@ def remove_prefix(flags, prefix): if level >= 2 and self.get_version_tuple() >= (4, 3): oflags.extend(["-march=native", "-mtune=native", ]) - return self.copy(cflags=cflags + oflags) + return replace(self, cflags=cflags + oflags) # }}} # {{{ nvcc +@dataclass(frozen=True) class NVCCToolchain(GCCLikeToolchain): def get_version_tuple(self): ver = self.get_version() @@ -358,6 +379,9 @@ def build_object(self, ext_file, source_files, debug=False): file=sys.stderr) raise CompileError("module compilation failed") + def with_optimization_level(self, level, **extra): + raise NotImplementedError + # }}} @@ -412,13 +436,15 @@ def _guess_toolchain_kwargs_from_python_config(): "o_ext": object_suffix, "defines": defines, "undefines": undefines, + "features": set(), } def call_capture_output(*args): - from pytools.prefork import call_capture_output import sys + from pytools.prefork import call_capture_output + encoding = sys.getdefaultencoding() result, stdout, stderr = call_capture_output(*args) return result, stdout.decode(encoding), stderr.decode(encoding) @@ -446,7 +472,20 @@ def guess_toolchain(): if sys.maxsize == 0x7fffffff: kwargs["cflags"].extend(["-arch", "i386"]) - return GCCToolchain(**kwargs) + return GCCToolchain( + cc=kwargs["cc"], + ld=kwargs["ld"], + library_dirs=kwargs["library_dirs"], + libraries=kwargs["libraries"], + include_dirs=kwargs["include_dirs"], + cflags=kwargs["cflags"], + ldflags=kwargs["ldflags"], + defines=kwargs["defines"], + undefines=kwargs["undefines"], + so_ext=kwargs["so_ext"], + o_ext=kwargs["o_ext"], + features=set(), + ) else: raise ToolchainGuessError( "Unable to determine compiler. Tried running " @@ -469,7 +508,6 @@ def guess_nvcc_toolchain(): "undefines": gcc_kwargs["undefines"], } kwargs.setdefault("undefines", []).append("__BLOCKS__") - kwargs["cc"] = "nvcc" return NVCCToolchain(**kwargs) diff --git a/examples/demo.py b/examples/demo.py index 966c7eb..0923b4a 100644 --- a/examples/demo.py +++ b/examples/demo.py @@ -1,5 +1,8 @@ import cgen as c + from codepy.bpl import BoostPythonModule + + mod = BoostPythonModule() mod.add_function( @@ -9,6 +12,8 @@ )) from codepy.toolchain import guess_toolchain + + cmod = mod.compile(guess_toolchain()) print(cmod.greet()) diff --git a/examples/demo_plain.py b/examples/demo_plain.py index babde52..80f0ce1 100644 --- a/examples/demo_plain.py +++ b/examples/demo_plain.py @@ -16,12 +16,18 @@ """ from codepy.toolchain import guess_toolchain + + toolchain = guess_toolchain() from codepy.libraries import add_boost_python + + add_boost_python(toolchain) from codepy.jit import extension_from_string + + cmod = extension_from_string(toolchain, "module", MODULE_CODE) print(cmod.greet()) diff --git a/examples/nvcc-test.py b/examples/nvcc-test.py index 28e6bfa..1ebf7d1 100644 --- a/examples/nvcc-test.py +++ b/examples/nvcc-test.py @@ -1,7 +1,9 @@ import cgen as c +from cgen.cuda import CudaGlobal + from codepy.bpl import BoostPythonModule from codepy.cuda import CudaModule -from cgen.cuda import CudaGlobal + # This file tests the ability to use compile and link CUDA code into the # Python interpreter. Running this test requires PyCUDA @@ -11,9 +13,11 @@ # The host module should include a function which is callable from Python host_mod = BoostPythonModule() +import math # Are we on a 32 or 64 bit platform? import sys -import math + + bitness = math.log(sys.maxsize) + 1 ptr_sz_uint_conv = "K" if bitness > 32 else "I" @@ -99,16 +103,17 @@ import codepy.jit import codepy.toolchain + gcc_toolchain = codepy.toolchain.guess_toolchain() nvcc_toolchain = codepy.toolchain.guess_nvcc_toolchain() module = cuda_mod.compile(gcc_toolchain, nvcc_toolchain, debug=True) +import numpy as np import pycuda.autoinit import pycuda.driver +import pycuda.gpuarray -import pycuda.gpuarray -import numpy as np length = 25 constant_value = 2 # This is a strange way to create a GPUArray, but is meant to illustrate diff --git a/setup.cfg b/setup.cfg index a1a0478..6b47f35 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,3 +7,12 @@ docstring-quotes = """ multiline-quotes = """ # enable-flake8-bugbear +# enable-flake8-isort + +[isort] +known_firstparty=pytools +known_local_folder=codepy +line_length = 85 +lines_after_imports = 2 +combine_as_imports = True +multi_line_output = 4 diff --git a/setup.py b/setup.py index f78ef4d..6da6b6f 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -from setuptools import setup, find_packages +from setuptools import find_packages, setup + setup( name="codepy", diff --git a/test/test_identical_symbols.py b/test/test_identical_symbols.py index 20de29f..98f1678 100644 --- a/test/test_identical_symbols.py +++ b/test/test_identical_symbols.py @@ -2,8 +2,9 @@ def make_greet_mod(greeting): - from cgen import FunctionBody, FunctionDeclaration, Block, \ - Const, Pointer, Value, Statement + from cgen import ( + Block, Const, FunctionBody, FunctionDeclaration, Pointer, Statement, Value) + from codepy.bpl import BoostPythonModule mod = BoostPythonModule() diff --git a/test/test_two_stage_compile.py b/test/test_two_stage_compile.py index 2500894..0a18e41 100644 --- a/test/test_two_stage_compile.py +++ b/test/test_two_stage_compile.py @@ -1,7 +1,8 @@ -from codepy.toolchain import guess_toolchain -from codepy.jit import compile_from_string from ctypes import CDLL +from codepy.jit import compile_from_string +from codepy.toolchain import guess_toolchain + def test(): toolchain = guess_toolchain()