代码拉取完成，页面将自动刷新
#! /usr/bin/env python3
'''
Developer CLI: building (meson), tests, benchmark, etc.
This file contains tasks definitions for doit (https://pydoit.org).
And also a CLI interface using click (https://click.palletsprojects.com).
The CLI is ideal for project contributors while,
doit interface is better suited for authoring the development tasks.
REQUIREMENTS:
--------------
- see environment.yml: doit, pydevtool, click, rich-click
# USAGE:
## 1 - click API
Commands can added using default Click API. i.e.
```
@cli.command()
@click.argument('extra_argv', nargs=-1)
@click.pass_obj
def python(ctx_obj, extra_argv):
"""Start a Python shell with PYTHONPATH set"""
```
## 2 - class based Click command definition
`CliGroup` provides an alternative class based API to create Click commands.
Just use the `cls_cmd` decorator. And define a `run()` method
```
@cli.cls_cmd('test')
class Test():
"""Run tests"""
@classmethod
def run(cls):
print('Running tests...')
```
- Command may make use a Click.Group context defining a `ctx` class attribute
- Command options are also define as class attributes
```
@cli.cls_cmd('test')
class Test():
"""Run tests"""
ctx = CONTEXT
verbose = Option(
['--verbose', '-v'], default=False, is_flag=True, help="verbosity")
@classmethod
def run(cls, **kwargs): # kwargs contains options from class and CONTEXT
print('Running tests...')
```
## 3 - class based interface can be run as a doit task by subclassing from Task
- Extra doit task metadata can be defined as class attribute TASK_META.
- `run()` method will be used as python-action by task
```
@cli.cls_cmd('test')
class Test(Task): # Task base class, doit will create a task
"""Run tests"""
ctx = CONTEXT
TASK_META = {
'task_dep': ['build'],
}
@classmethod
def run(cls, **kwargs):
pass
```
## 4 - doit tasks with cmd-action "shell" or dynamic metadata
Define method `task_meta()` instead of `run()`:
```
@cli.cls_cmd('refguide-check')
class RefguideCheck(Task):
@classmethod
def task_meta(cls, **kwargs):
return {
```
'''
import os
import subprocess
import sys
import warnings
import shutil
import json
import datetime
import time
import importlib
import importlib.util
import errno
import contextlib
from sysconfig import get_path
import math
import traceback
from concurrent.futures.process import _MAX_WINDOWS_WORKERS
from pathlib import Path
from collections import namedtuple
from types import ModuleType as new_module
from dataclasses import dataclass
import click
from click import Option, Argument
from doit.cmd_base import ModuleTaskLoader
from doit.reporter import ZeroReporter
from doit.exceptions import TaskError
from doit.api import run_tasks
from pydevtool.cli import UnifiedContext, CliGroup, Task
from rich.console import Console
from rich.panel import Panel
from rich.theme import Theme
from rich_click import rich_click
DOIT_CONFIG = {
'verbosity': 2,
'minversion': '0.36.0',
}
console_theme = Theme({
"cmd": "italic gray50",
})
if sys.platform == 'win32':
class EMOJI:
cmd = ">"
else:
class EMOJI:
cmd = ":computer:"
rich_click.STYLE_ERRORS_SUGGESTION = "yellow italic"
rich_click.SHOW_ARGUMENTS = True
rich_click.GROUP_ARGUMENTS_OPTIONS = False
rich_click.SHOW_METAVARS_COLUMN = True
rich_click.USE_MARKDOWN = True
rich_click.OPTION_GROUPS = {
"dev.py": [
{
"name": "Options",
"options": [
"--help", "--build-dir", "--no-build", "--install-prefix"],
},
],
"dev.py test": [
{
"name": "Options",
"options": ["--help", "--verbose", "--parallel", "--coverage",
"--durations"],
},
{
"name": "Options: test selection",
"options": ["--submodule", "--tests", "--mode"],
},
],
}
rich_click.COMMAND_GROUPS = {
"dev.py": [
{
"name": "build & testing",
"commands": ["build", "test"],
},
{
"name": "static checkers",
"commands": ["lint", "mypy"],
},
{
"name": "environments",
"commands": ["shell", "python", "ipython", "show_PYTHONPATH"],
},
{
"name": "documentation",
"commands": ["doc", "refguide-check"],
},
{
"name": "release",
"commands": ["notes", "authors"],
},
{
"name": "benchmarking",
"commands": ["bench"],
},
]
}
class ErrorOnlyReporter(ZeroReporter):
desc = """Report errors only"""
def runtime_error(self, msg):
console = Console()
console.print("[red bold] msg")
def add_failure(self, task, fail_info):
console = Console()
if isinstance(fail_info, TaskError):
console.print(f'[red]Task Error - {task.name}'
f' => {fail_info.message}')
if fail_info.traceback:
console.print(Panel(
"".join(fail_info.traceback),
title=f"{task.name}",
subtitle=fail_info.message,
border_style="red",
))
CONTEXT = UnifiedContext({
'build_dir': Option(
['--build-dir'], metavar='BUILD_DIR',
default='build', show_default=True,
help=':wrench: Relative path to the build directory.'),
'no_build': Option(
["--no-build", "-n"], default=False, is_flag=True,
help=(":wrench: Do not build the project"
" (note event python only modification require build).")),
'install_prefix': Option(
['--install-prefix'], default=None, metavar='INSTALL_DIR',
help=(":wrench: Relative path to the install directory."
" Default is <build-dir>-install.")),
})
def run_doit_task(tasks):
"""
:param tasks: (dict) task_name -> {options}
"""
loader = ModuleTaskLoader(globals())
doit_config = {
'verbosity': 2,
'reporter': ErrorOnlyReporter,
}
return run_tasks(loader, tasks, extra_config={'GLOBAL': doit_config})
class CLI(CliGroup):
context = CONTEXT
run_doit_task = run_doit_task
@click.group(cls=CLI)
@click.pass_context
def cli(ctx, **kwargs):
"""Developer Tool for SciPy
\bCommands that require a built/installed instance are marked with :wrench:.
\b**python dev.py --build-dir my-build test -s stats**
"""
CLI.update_context(ctx, kwargs)
PROJECT_MODULE = "scipy"
PROJECT_ROOT_FILES = ['scipy', 'LICENSE.txt', 'meson.build']
@dataclass
class Dirs:
"""
root:
Directory where scr, build config and tools are located
(and this file)
build:
Directory where build output files (i.e. *.o) are saved
install:
Directory where .so from build and .py from src are put together.
site:
Directory where the built SciPy version was installed.
This is a custom prefix, followed by a relative path matching
the one the system would use for the site-packages of the active
Python interpreter.
"""
# all paths are absolute
root: Path
build: Path
installed: Path
site: Path # <install>/lib/python<version>/site-packages
def __init__(self, args=None):
""":params args: object like Context(build_dir, install_prefix)"""
self.root = Path(__file__).parent.absolute()
if not args:
return
self.build = Path(args.build_dir).resolve()
if args.install_prefix:
self.installed = Path(args.install_prefix).resolve()
else:
self.installed = self.build.parent / (self.build.stem + "-install")
if sys.platform == 'win32' and sys.version_info < (3, 10):
# Work around a pathlib bug; these must be absolute paths
self.build = Path(os.path.abspath(self.build))
self.installed = Path(os.path.abspath(self.installed))
# relative path for site-package with py version
# i.e. 'lib/python3.10/site-packages'
self.site = self.get_site_packages()
def add_sys_path(self):
"""Add site dir to sys.path / PYTHONPATH"""
site_dir = str(self.site)
sys.path.insert(0, site_dir)
os.environ['PYTHONPATH'] = \
os.pathsep.join((site_dir, os.environ.get('PYTHONPATH', '')))
def get_site_packages(self):
"""
Depending on whether we have debian python or not,
return dist_packages path or site_packages path.
"""
if sys.version_info >= (3, 12):
plat_path = Path(get_path('platlib'))
else:
# distutils is required to infer meson install path
# for python < 3.12 in debian patched python
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
from distutils import dist
from distutils.command.install import INSTALL_SCHEMES
if 'deb_system' in INSTALL_SCHEMES:
# debian patched python in use
install_cmd = dist.Distribution().get_command_obj('install')
install_cmd.select_scheme('deb_system')
install_cmd.finalize_options()
plat_path = Path(install_cmd.install_platlib)
else:
plat_path = Path(get_path('platlib'))
return self.installed / plat_path.relative_to(sys.exec_prefix)
@contextlib.contextmanager
def working_dir(new_dir):
current_dir = os.getcwd()
try:
os.chdir(new_dir)
yield
finally:
os.chdir(current_dir)
def import_module_from_path(mod_name, mod_path):
"""Import module with name `mod_name` from file path `mod_path`"""
spec = importlib.util.spec_from_file_location(mod_name, mod_path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def get_test_runner(project_module):
"""
get Test Runner from locally installed/built project
"""
__import__(project_module)
# scipy._lib._testutils:PytestTester
test = sys.modules[project_module].test
version = sys.modules[project_module].__version__
mod_path = sys.modules[project_module].__file__
mod_path = os.path.abspath(os.path.join(os.path.dirname(mod_path)))
return test, version, mod_path
############
@cli.cls_cmd('build')
class Build(Task):
""":wrench: Build & install package on path.
\b
```shell-session
Examples:
$ python dev.py build --asan ;
ASAN_OPTIONS=detect_leaks=0:symbolize=1:strict_init_order=true
LD_PRELOAD=$(gcc --print-file-name=libasan.so)
python dev.py test -v -t
./scipy/ndimage/tests/test_morphology.py -- -s
```
"""
ctx = CONTEXT
werror = Option(
['--werror'], default=False, is_flag=True,
help="Treat warnings as errors")
gcov = Option(
['--gcov'], default=False, is_flag=True,
help="enable C code coverage via gcov (requires GCC)."
"gcov output goes to build/**/*.gc*")
asan = Option(
['--asan'], default=False, is_flag=True,
help=("Build and run with AddressSanitizer support. "
"Note: the build system doesn't check whether "
"the project is already compiled with ASan. "
"If not, you need to do a clean build (delete "
"build and build-install directories)."))
debug = Option(
['--debug', '-d'], default=False, is_flag=True, help="Debug build")
parallel = Option(
['--parallel', '-j'], default=None, metavar='N_JOBS',
help=("Number of parallel jobs for building. "
"This defaults to the number of available physical CPU cores"))
setup_args = Option(
['--setup-args', '-C'], default=[], multiple=True,
help=("Pass along one or more arguments to `meson setup` "
"Repeat the `-C` in case of multiple arguments."))
show_build_log = Option(
['--show-build-log'], default=False, is_flag=True,
help="Show build output rather than using a log file")
with_scipy_openblas = Option(
['--with-scipy-openblas'], default=False, is_flag=True,
help=("If set, use the `scipy-openblas32` wheel installed into the "
"current environment as the BLAS/LAPACK to build against."))
@classmethod
def setup_build(cls, dirs, args):
"""
Setting up meson-build
"""
for fn in PROJECT_ROOT_FILES:
if not (dirs.root / fn).exists():
print("To build the project, run dev.py in "
"git checkout or unpacked source")
sys.exit(1)
env = dict(os.environ)
cmd = ["meson", "setup", dirs.build, "--prefix", dirs.installed]
build_dir = dirs.build
run_dir = Path()
if build_dir.exists() and not (build_dir / 'meson-info').exists():
if list(build_dir.iterdir()):
raise RuntimeError("Can't build into non-empty directory "
f"'{build_dir.absolute()}'")
if sys.platform == "cygwin":
# Cygwin only has netlib lapack, but can link against
# OpenBLAS rather than netlib blas at runtime. There is
# no libopenblas-devel to enable linking against
# openblas-specific functions or OpenBLAS Lapack
cmd.extend(["-Dlapack=lapack", "-Dblas=blas"])
build_options_file = (
build_dir / "meson-info" / "intro-buildoptions.json")
if build_options_file.exists():
with open(build_options_file) as f:
build_options = json.load(f)
installdir = None
for option in build_options:
if option["name"] == "prefix":
installdir = option["value"]
break
if installdir != str(dirs.installed):
run_dir = build_dir
cmd = ["meson", "setup", "--reconfigure",
"--prefix", str(dirs.installed)]
else:
return
if args.werror:
cmd += ["--werror"]
if args.gcov:
cmd += ['-Db_coverage=true']
if args.asan:
cmd += ['-Db_sanitize=address,undefined']
if args.setup_args:
cmd += [str(arg) for arg in args.setup_args]
if args.with_scipy_openblas:
cls.configure_scipy_openblas()
env['PKG_CONFIG_PATH'] = os.pathsep.join([
os.path.join(os.getcwd(), '.openblas'),
env.get('PKG_CONFIG_PATH', '')
])
# Setting up meson build
cmd_str = ' '.join([str(p) for p in cmd])
cls.console.print(f"{EMOJI.cmd} [cmd] {cmd_str}")
ret = subprocess.call(cmd, env=env, cwd=run_dir)
if ret == 0:
print("Meson build setup OK")
else:
print("Meson build setup failed!")
sys.exit(1)
return env
@classmethod
def build_project(cls, dirs, args, env):
"""
Build a dev version of the project.
"""
cmd = ["ninja", "-C", str(dirs.build)]
if args.parallel is None:
# Use number of physical cores rather than ninja's default of 2N+2,
# to avoid out of memory issues (see gh-17941 and gh-18443)
n_cores = cpu_count(only_physical_cores=True)
cmd += [f"-j{n_cores}"]
else:
cmd += ["-j", str(args.parallel)]
# Building with ninja-backend
cmd_str = ' '.join([str(p) for p in cmd])
cls.console.print(f"{EMOJI.cmd} [cmd] {cmd_str}")
ret = subprocess.call(cmd, env=env, cwd=dirs.root)
if ret == 0:
print("Build OK")
else:
print("Build failed!")
sys.exit(1)
@classmethod
def install_project(cls, dirs, args):
"""
Installs the project after building.
"""
if dirs.installed.exists():
non_empty = len(os.listdir(dirs.installed))
if non_empty and not dirs.site.exists():
raise RuntimeError("Can't install in non-empty directory: "
f"'{dirs.installed}'")
cmd = ["meson", "install", "-C", args.build_dir, "--only-changed"]
log_filename = dirs.root / 'meson-install.log'
start_time = datetime.datetime.now()
cmd_str = ' '.join([str(p) for p in cmd])
cls.console.print(f"{EMOJI.cmd} [cmd] {cmd_str}")
if args.show_build_log:
ret = subprocess.call(cmd, cwd=dirs.root)
else:
print("Installing, see meson-install.log...")
with open(log_filename, 'w') as log:
p = subprocess.Popen(cmd, stdout=log, stderr=log,
cwd=dirs.root)
try:
# Wait for it to finish, and print something to indicate the
# process is alive, but only if the log file has grown (to
# allow continuous integration environments kill a hanging
# process accurately if it produces no output)
last_blip = time.time()
last_log_size = os.stat(log_filename).st_size
while p.poll() is None:
time.sleep(0.5)
if time.time() - last_blip > 60:
log_size = os.stat(log_filename).st_size
if log_size > last_log_size:
elapsed = datetime.datetime.now() - start_time
print(f" ... installation in progress ({elapsed} "
"elapsed)")
last_blip = time.time()
last_log_size = log_size
ret = p.wait()
except: # noqa: E722
p.terminate()
raise
elapsed = datetime.datetime.now() - start_time
if ret != 0:
if not args.show_build_log:
with open(log_filename) as f:
print(f.read())
print(f"Installation failed! ({elapsed} elapsed)")
sys.exit(1)
# ignore everything in the install directory.
with open(dirs.installed / ".gitignore", "w") as f:
f.write("*")
if sys.platform == "cygwin":
rebase_cmd = ["/usr/bin/rebase", "--database", "--oblivious"]
rebase_cmd.extend(Path(dirs.installed).glob("**/*.dll"))
subprocess.check_call(rebase_cmd)
print("Installation OK")
return
@classmethod
def configure_scipy_openblas(self, blas_variant='32'):
"""Create .openblas/scipy-openblas.pc and scipy/_distributor_init_local.py
Requires a pre-installed scipy-openblas32 wheel from PyPI.
"""
basedir = os.getcwd()
openblas_dir = os.path.join(basedir, ".openblas")
pkg_config_fname = os.path.join(openblas_dir, "scipy-openblas.pc")
if os.path.exists(pkg_config_fname):
return None
module_name = f"scipy_openblas{blas_variant}"
try:
openblas = importlib.import_module(module_name)
except ModuleNotFoundError:
raise RuntimeError(f"Importing '{module_name}' failed. "
"Make sure it is installed and reachable "
"by the current Python executable. You can "
f"install it via 'pip install {module_name}'.")
local = os.path.join(basedir, "scipy", "_distributor_init_local.py")
with open(local, "w", encoding="utf8") as fid:
fid.write(f"import {module_name}\n")
os.makedirs(openblas_dir, exist_ok=True)
with open(pkg_config_fname, "w", encoding="utf8") as fid:
fid.write(openblas.get_pkg_config().replace("\\", "/"))
@classmethod
def run(cls, add_path=False, **kwargs):
kwargs.update(cls.ctx.get(kwargs))
Args = namedtuple('Args', [k for k in kwargs.keys()])
args = Args(**kwargs)
cls.console = Console(theme=console_theme)
dirs = Dirs(args)
if args.no_build:
print("Skipping build")
else:
env = cls.setup_build(dirs, args)
cls.build_project(dirs, args, env)
cls.install_project(dirs, args)
# add site to sys.path
if add_path:
dirs.add_sys_path()
@cli.cls_cmd('test')
class Test(Task):
""":wrench: Run tests.
\b
```python
Examples:
$ python dev.py test -s {SAMPLE_SUBMODULE}
$ python dev.py test -t scipy.optimize.tests.test_minimize_constrained
$ python dev.py test -s cluster -m full --durations 20
$ python dev.py test -s stats -- --tb=line # `--` passes next args to pytest
$ python dev.py test -b numpy -b pytorch -s cluster
```
"""
ctx = CONTEXT
verbose = Option(
['--verbose', '-v'], default=False, is_flag=True,
help="more verbosity")
# removed doctests as currently not supported by _lib/_testutils.py
# doctests = Option(['--doctests'], default=False)
coverage = Option(
['--coverage', '-c'], default=False, is_flag=True,
help=("report coverage of project code. "
"HTML output goes under build/coverage"))
durations = Option(
['--durations', '-d'], default=None, metavar="NUM_TESTS",
help="Show timing for the given number of slowest tests"
)
submodule = Option(
['--submodule', '-s'], default=None, metavar='MODULE_NAME',
help="Submodule whose tests to run (cluster, constants, ...)")
tests = Option(
['--tests', '-t'], default=None, multiple=True, metavar='TESTS',
help='Specify tests to run')
mode = Option(
['--mode', '-m'], default='fast', metavar='MODE', show_default=True,
help=("'fast', 'full', or something that could be passed to "
"`pytest -m` as a marker expression"))
parallel = Option(
['--parallel', '-j'], default=1, metavar='N_JOBS',
help="Number of parallel jobs for testing"
)
array_api_backend = Option(
['--array-api-backend', '-b'], default=None, metavar='ARRAY_BACKEND',
multiple=True,
help=(
"Array API backend ('all', 'numpy', 'pytorch', 'cupy', 'numpy.array_api')."
)
)
# Argument can't have `help=`; used to consume all of `-- arg1 arg2 arg3`
pytest_args = Argument(
['pytest_args'], nargs=-1, metavar='PYTEST-ARGS', required=False
)
TASK_META = {
'task_dep': ['build'],
}
@classmethod
def scipy_tests(cls, args, pytest_args):
dirs = Dirs(args)
dirs.add_sys_path()
print(f"SciPy from development installed path at: {dirs.site}")
# FIXME: support pos-args with doit
extra_argv = pytest_args[:] if pytest_args else []
if extra_argv and extra_argv[0] == '--':
extra_argv = extra_argv[1:]
if args.coverage:
dst_dir = dirs.root / args.build_dir / 'coverage'
fn = dst_dir / 'coverage_html.js'
if dst_dir.is_dir() and fn.is_file():
shutil.rmtree(dst_dir)
extra_argv += ['--cov-report=html:' + str(dst_dir)]
shutil.copyfile(dirs.root / '.coveragerc',
dirs.site / '.coveragerc')
if args.durations:
extra_argv += ['--durations', args.durations]
# convert options to test selection
if args.submodule:
tests = [PROJECT_MODULE + "." + args.submodule]
elif args.tests:
tests = args.tests
else:
tests = None
if len(args.array_api_backend) != 0:
os.environ['SCIPY_ARRAY_API'] = json.dumps(list(args.array_api_backend))
runner, version, mod_path = get_test_runner(PROJECT_MODULE)
# FIXME: changing CWD is not a good practice
with working_dir(dirs.site):
print("Running tests for {} version:{}, installed at:{}".format(
PROJECT_MODULE, version, mod_path))
# runner verbosity - convert bool to int
verbose = int(args.verbose) + 1
result = runner( # scipy._lib._testutils:PytestTester
args.mode,
verbose=verbose,
extra_argv=extra_argv,
doctests=False,
coverage=args.coverage,
tests=tests,
parallel=args.parallel)
return result
@classmethod
def run(cls, pytest_args, **kwargs):
"""run unit-tests"""
kwargs.update(cls.ctx.get())
Args = namedtuple('Args', [k for k in kwargs.keys()])
args = Args(**kwargs)
return cls.scipy_tests(args, pytest_args)
@cli.cls_cmd('bench')
class Bench(Task):
""":wrench: Run benchmarks.
\b
```python
Examples:
$ python dev.py bench -t integrate.SolveBVP
$ python dev.py bench -t linalg.Norm
$ python dev.py bench --compare main
```
"""
ctx = CONTEXT
TASK_META = {
'task_dep': ['build'],
}
submodule = Option(
['--submodule', '-s'], default=None, metavar='SUBMODULE',
help="Submodule whose tests to run (cluster, constants, ...)")
tests = Option(
['--tests', '-t'], default=None, multiple=True,
metavar='TESTS', help='Specify tests to run')
compare = Option(
['--compare', '-c'], default=None, metavar='COMPARE', multiple=True,
help=(
"Compare benchmark results of current HEAD to BEFORE. "
"Use an additional --bench COMMIT to override HEAD with COMMIT. "
"Note that you need to commit your changes first!"))
@staticmethod
def run_asv(dirs, cmd):
EXTRA_PATH = ['/usr/lib/ccache', '/usr/lib/f90cache',
'/usr/local/lib/ccache', '/usr/local/lib/f90cache']
bench_dir = dirs.root / 'benchmarks'
sys.path.insert(0, str(bench_dir))
# Always use ccache, if installed
env = dict(os.environ)
env['PATH'] = os.pathsep.join(EXTRA_PATH +
env.get('PATH', '').split(os.pathsep))
# Control BLAS/LAPACK threads
env['OPENBLAS_NUM_THREADS'] = '1'
env['MKL_NUM_THREADS'] = '1'
# Limit memory usage
from benchmarks.common import set_mem_rlimit
try:
set_mem_rlimit()
except (ImportError, RuntimeError):
pass
try:
return subprocess.call(cmd, env=env, cwd=bench_dir)
except OSError as err:
if err.errno == errno.ENOENT:
cmd_str = " ".join(cmd)
print(f"Error when running '{cmd_str}': {err}\n")
print("You need to install Airspeed Velocity "
"(https://airspeed-velocity.github.io/asv/)")
print("to run Scipy benchmarks")
return 1
raise
@classmethod
def scipy_bench(cls, args):
dirs = Dirs(args)
dirs.add_sys_path()
print(f"SciPy from development installed path at: {dirs.site}")
with working_dir(dirs.site):
runner, version, mod_path = get_test_runner(PROJECT_MODULE)
extra_argv = []
if args.tests:
extra_argv.append(args.tests)
if args.submodule:
extra_argv.append([args.submodule])
bench_args = []
for a in extra_argv:
bench_args.extend(['--bench', ' '.join(str(x) for x in a)])
if not args.compare:
print(f"Running benchmarks for Scipy version {version} at {mod_path}")
cmd = ['asv', 'run', '--dry-run', '--show-stderr',
'--python=same', '--quick'] + bench_args
retval = cls.run_asv(dirs, cmd)
sys.exit(retval)
else:
if len(args.compare) == 1:
commit_a = args.compare[0]
commit_b = 'HEAD'
elif len(args.compare) == 2:
commit_a, commit_b = args.compare
else:
print("Too many commits to compare benchmarks for")
# Check for uncommitted files
if commit_b == 'HEAD':
r1 = subprocess.call(['git', 'diff-index', '--quiet',
'--cached', 'HEAD'])
r2 = subprocess.call(['git', 'diff-files', '--quiet'])
if r1 != 0 or r2 != 0:
print("*" * 80)
print("WARNING: you have uncommitted changes --- "
"these will NOT be benchmarked!")
print("*" * 80)
# Fix commit ids (HEAD is local to current repo)
p = subprocess.Popen(['git', 'rev-parse', commit_b],
stdout=subprocess.PIPE)
out, err = p.communicate()
commit_b = out.strip()
p = subprocess.Popen(['git', 'rev-parse', commit_a],
stdout=subprocess.PIPE)
out, err = p.communicate()
commit_a = out.strip()
cmd_compare = [
'asv', 'continuous', '--show-stderr', '--factor', '1.05',
'--quick', commit_a, commit_b
] + bench_args
cls.run_asv(dirs, cmd_compare)
sys.exit(1)
@classmethod
def run(cls, **kwargs):
"""run benchmark"""
kwargs.update(cls.ctx.get())
Args = namedtuple('Args', [k for k in kwargs.keys()])
args = Args(**kwargs)
cls.scipy_bench(args)
###################
# linters
def emit_cmdstr(cmd):
"""Print the command that's being run to stdout
Note: cannot use this in the below tasks (yet), because as is these command
strings are always echoed to the console, even if the command isn't run
(but for example the `build` command is run).
"""
console = Console(theme=console_theme)
# The [cmd] square brackets controls the font styling, typically in italics
# to differentiate it from other stdout content
console.print(f"{EMOJI.cmd} [cmd] {cmd}")
def task_lint():
# Lint just the diff since branching off of main using a
# stricter configuration.
# emit_cmdstr(os.path.join('tools', 'lint.py') + ' --diff-against main')
return {
'basename': 'lint',
'actions': [str(Dirs().root / 'tools' / 'lint.py') +
' --diff-against=main'],
'doc': 'Lint only files modified since last commit (stricter rules)',
}
def task_unicode_check():
# emit_cmdstr(os.path.join('tools', 'unicode-check.py'))
return {
'basename': 'unicode-check',
'actions': [str(Dirs().root / 'tools' / 'unicode-check.py')],
'doc': 'Check for disallowed Unicode characters in the SciPy Python '
'and Cython source code.',
}
def task_check_test_name():
# emit_cmdstr(os.path.join('tools', 'check_test_name.py'))
return {
"basename": "check-testname",
"actions": [str(Dirs().root / "tools" / "check_test_name.py")],
"doc": "Check tests are correctly named so that pytest runs them."
}
@cli.cls_cmd('lint')
class Lint:
""":dash: Run linter on modified files and check for
disallowed Unicode characters and possibly-invalid test names."""
def run():
run_doit_task({
'lint': {},
'unicode-check': {},
'check-testname': {},
})
@cli.cls_cmd('mypy')
class Mypy(Task):
""":wrench: Run mypy on the codebase."""
ctx = CONTEXT
TASK_META = {
'task_dep': ['build'],
}
@classmethod
def run(cls, **kwargs):
kwargs.update(cls.ctx.get())
Args = namedtuple('Args', [k for k in kwargs.keys()])
args = Args(**kwargs)
dirs = Dirs(args)
try:
import mypy.api
except ImportError as e:
raise RuntimeError(
"Mypy not found. Please install it by running "
"pip install -r mypy_requirements.txt from the repo root"
) from e
config = dirs.root / "mypy.ini"
check_path = PROJECT_MODULE
with working_dir(dirs.site):
# By default mypy won't color the output since it isn't being
# invoked from a tty.
os.environ['MYPY_FORCE_COLOR'] = '1'
# Change to the site directory to make sure mypy doesn't pick
# up any type stubs in the source tree.
emit_cmdstr(f"mypy.api.run --config-file {config} {check_path}")
report, errors, status = mypy.api.run([
"--config-file",
str(config),
check_path,
])
print(report, end='')
print(errors, end='', file=sys.stderr)
return status == 0
##########################################
# DOC
@cli.cls_cmd('doc')
class Doc(Task):
""":wrench: Build documentation.
TARGETS: Sphinx build targets [default: 'html']
Running `python dev.py doc -j8 html` is equivalent to:
1. Execute build command (skip by passing the global `-n` option).
2. Set the PYTHONPATH environment variable
(query with `python dev.py -n show_PYTHONPATH`).
3. Run make on `doc/Makefile`, i.e.: `make -C doc -j8 TARGETS`
To remove all generated documentation do: `python dev.py -n doc clean`
"""
ctx = CONTEXT
args = Argument(['args'], nargs=-1, metavar='TARGETS', required=False)
list_targets = Option(
['--list-targets', '-t'], default=False, is_flag=True,
help='List doc targets',
)
parallel = Option(
['--parallel', '-j'], default=1, metavar='N_JOBS',
help="Number of parallel jobs"
)
no_cache = Option(
['--no-cache'], default=False, is_flag=True,
help="Forces a full rebuild of the docs. Note that this may be " + \
"needed in order to make docstring changes in C/Cython files " + \
"show up."
)
@classmethod
def task_meta(cls, list_targets, parallel, no_cache, args, **kwargs):
if list_targets: # list MAKE targets, remove default target
task_dep = []
targets = ''
else:
task_dep = ['build']
targets = ' '.join(args) if args else 'html'
kwargs.update(cls.ctx.get())
Args = namedtuple('Args', [k for k in kwargs.keys()])
build_args = Args(**kwargs)
dirs = Dirs(build_args)
make_params = [f'PYTHON="{sys.executable}"']
if parallel or no_cache:
sphinxopts = ""
if parallel:
sphinxopts += f"-j{parallel} "
if no_cache:
sphinxopts += "-E"
make_params.append(f'SPHINXOPTS="{sphinxopts}"')
return {
'actions': [
# move to doc/ so local scipy does not get imported
(f'cd doc; env PYTHONPATH="{dirs.site}" '
f'make {" ".join(make_params)} {targets}'),
],
'task_dep': task_dep,
'io': {'capture': False},
}
@cli.cls_cmd('refguide-check')
class RefguideCheck(Task):
""":wrench: Run refguide check."""
ctx = CONTEXT
submodule = Option(
['--submodule', '-s'], default=None, metavar='SUBMODULE',
help="Submodule whose tests to run (cluster, constants, ...)")
verbose = Option(
['--verbose', '-v'], default=False, is_flag=True, help="verbosity")
@classmethod
def task_meta(cls, **kwargs):
kwargs.update(cls.ctx.get())
Args = namedtuple('Args', [k for k in kwargs.keys()])
args = Args(**kwargs)
dirs = Dirs(args)
cmd = [f'{sys.executable}',
str(dirs.root / 'tools' / 'refguide_check.py'),
'--doctests']
if args.verbose:
cmd += ['-vvv']
if args.submodule:
cmd += [args.submodule]
cmd_str = ' '.join(cmd)
return {
'actions': [f'env PYTHONPATH={dirs.site} {cmd_str}'],
'task_dep': ['build'],
'io': {'capture': False},
}
##########################################
# ENVS
@cli.cls_cmd('python')
class Python:
""":wrench: Start a Python shell with PYTHONPATH set.
ARGS: Arguments passed to the Python interpreter.
If not set, an interactive shell is launched.
Running `python dev.py shell my_script.py` is equivalent to:
1. Execute build command (skip by passing the global `-n` option).
2. Set the PYTHONPATH environment variable
(query with `python dev.py -n show_PYTHONPATH`).
3. Run interpreter: `python my_script.py`
"""
ctx = CONTEXT
pythonpath = Option(
['--pythonpath', '-p'], metavar='PYTHONPATH', default=None,
help='Paths to prepend to PYTHONPATH')
extra_argv = Argument(
['extra_argv'], nargs=-1, metavar='ARGS', required=False)
@classmethod
def _setup(cls, pythonpath, **kwargs):
vals = Build.opt_defaults()
vals.update(kwargs)
Build.run(add_path=True, **vals)
if pythonpath:
for p in reversed(pythonpath.split(os.pathsep)):
sys.path.insert(0, p)
@classmethod
def run(cls, pythonpath, extra_argv=None, **kwargs):
cls._setup(pythonpath, **kwargs)
if extra_argv:
# Don't use subprocess, since we don't want to include the
# current path in PYTHONPATH.
sys.argv = extra_argv
with open(extra_argv[0]) as f:
script = f.read()
sys.modules['__main__'] = new_module('__main__')
ns = dict(__name__='__main__', __file__=extra_argv[0])
exec(script, ns)
else:
import code
code.interact()
@cli.cls_cmd('ipython')
class Ipython(Python):
""":wrench: Start IPython shell with PYTHONPATH set.
Running `python dev.py ipython` is equivalent to:
1. Execute build command (skip by passing the global `-n` option).
2. Set the PYTHONPATH environment variable
(query with `python dev.py -n show_PYTHONPATH`).
3. Run the `ipython` interpreter.
"""
ctx = CONTEXT
pythonpath = Python.pythonpath
@classmethod
def run(cls, pythonpath, **kwargs):
cls._setup(pythonpath, **kwargs)
import IPython
IPython.embed(user_ns={})
@cli.cls_cmd('shell')
class Shell(Python):
""":wrench: Start Unix shell with PYTHONPATH set.
Running `python dev.py shell` is equivalent to:
1. Execute build command (skip by passing the global `-n` option).
2. Open a new shell.
3. Set the PYTHONPATH environment variable in shell
(query with `python dev.py -n show_PYTHONPATH`).
"""
ctx = CONTEXT
pythonpath = Python.pythonpath
extra_argv = Python.extra_argv
@classmethod
def run(cls, pythonpath, extra_argv, **kwargs):
cls._setup(pythonpath, **kwargs)
shell = os.environ.get('SHELL', 'sh')
click.echo(f"Spawning a Unix shell '{shell}' ...")
os.execv(shell, [shell] + list(extra_argv))
sys.exit(1)
@cli.cls_cmd('show_PYTHONPATH')
class ShowDirs(Python):
""":information: Show value of the PYTHONPATH environment variable used in
this script.
PYTHONPATH sets the default search path for module files for the
interpreter. Here, it includes the path to the local SciPy build
(typically `.../build-install/lib/python3.10/site-packages`).
Use the global option `-n` to skip the building step, e.g.:
`python dev.py -n show_PYTHONPATH`
"""
ctx = CONTEXT
pythonpath = Python.pythonpath
extra_argv = Python.extra_argv
@classmethod
def run(cls, pythonpath, extra_argv, **kwargs):
cls._setup(pythonpath, **kwargs)
py_path = os.environ.get('PYTHONPATH', '')
click.echo(f"PYTHONPATH={py_path}")
@cli.command()
@click.argument('version_args', nargs=2)
@click.pass_obj
def notes(ctx_obj, version_args):
""":ledger: Release notes and log generation.
\b
```python
Example:
$ python dev.py notes v1.7.0 v1.8.0
```
"""
if version_args:
sys.argv = version_args
log_start = sys.argv[0]
log_end = sys.argv[1]
cmd = f"python tools/write_release_and_log.py {log_start} {log_end}"
click.echo(cmd)
try:
subprocess.run([cmd], check=True, shell=True)
except subprocess.CalledProcessError:
print('Error caught: Incorrect log start or log end version')
@cli.command()
@click.argument('revision_args', nargs=2)
@click.pass_obj
def authors(ctx_obj, revision_args):
""":ledger: Generate list of authors who contributed within revision
interval.
\b
```python
Example:
$ python dev.py authors v1.7.0 v1.8.0
```
"""
if revision_args:
sys.argv = revision_args
start_revision = sys.argv[0]
end_revision = sys.argv[1]
cmd = f"python tools/authors.py {start_revision}..{end_revision}"
click.echo(cmd)
try:
subprocess.run([cmd], check=True, shell=True)
except subprocess.CalledProcessError:
print('Error caught: Incorrect revision start or revision end')
# The following CPU core count functions were taken from loky/backend/context.py
# See https://github.com/joblib/loky
# Cache for the number of physical cores to avoid repeating subprocess calls.
# It should not change during the lifetime of the program.
physical_cores_cache = None
def cpu_count(only_physical_cores=False):
"""Return the number of CPUs the current process can use.
The returned number of CPUs accounts for:
* the number of CPUs in the system, as given by
``multiprocessing.cpu_count``;
* the CPU affinity settings of the current process
(available on some Unix systems);
* Cgroup CPU bandwidth limit (available on Linux only, typically
set by docker and similar container orchestration systems);
* the value of the LOKY_MAX_CPU_COUNT environment variable if defined.
and is given as the minimum of these constraints.
If ``only_physical_cores`` is True, return the number of physical cores
instead of the number of logical cores (hyperthreading / SMT). Note that
this option is not enforced if the number of usable cores is controlled in
any other way such as: process affinity, Cgroup restricted CPU bandwidth
or the LOKY_MAX_CPU_COUNT environment variable. If the number of physical
cores is not found, return the number of logical cores.
Note that on Windows, the returned number of CPUs cannot exceed 61 (or 60 for
Python < 3.10), see:
https://bugs.python.org/issue26903.
It is also always larger or equal to 1.
"""
# Note: os.cpu_count() is allowed to return None in its docstring
os_cpu_count = os.cpu_count() or 1
if sys.platform == "win32":
# On Windows, attempting to use more than 61 CPUs would result in a
# OS-level error. See https://bugs.python.org/issue26903. According to
# https://learn.microsoft.com/en-us/windows/win32/procthread/processor-groups
# it might be possible to go beyond with a lot of extra work but this
# does not look easy.
os_cpu_count = min(os_cpu_count, _MAX_WINDOWS_WORKERS)
cpu_count_user = _cpu_count_user(os_cpu_count)
aggregate_cpu_count = max(min(os_cpu_count, cpu_count_user), 1)
if not only_physical_cores:
return aggregate_cpu_count
if cpu_count_user < os_cpu_count:
# Respect user setting
return max(cpu_count_user, 1)
cpu_count_physical, exception = _count_physical_cores()
if cpu_count_physical != "not found":
return cpu_count_physical
# Fallback to default behavior
if exception is not None:
# warns only the first time
warnings.warn(
"Could not find the number of physical cores for the "
f"following reason:\n{exception}\n"
"Returning the number of logical cores instead. You can "
"silence this warning by setting LOKY_MAX_CPU_COUNT to "
"the number of cores you want to use."
)
traceback.print_tb(exception.__traceback__)
return aggregate_cpu_count
def _cpu_count_cgroup(os_cpu_count):
# Cgroup CPU bandwidth limit available in Linux since 2.6 kernel
cpu_max_fname = "/sys/fs/cgroup/cpu.max"
cfs_quota_fname = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"
cfs_period_fname = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"
if os.path.exists(cpu_max_fname):
# cgroup v2
# https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html
with open(cpu_max_fname) as fh:
cpu_quota_us, cpu_period_us = fh.read().strip().split()
elif os.path.exists(cfs_quota_fname) and os.path.exists(cfs_period_fname):
# cgroup v1
# https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html#management
with open(cfs_quota_fname) as fh:
cpu_quota_us = fh.read().strip()
with open(cfs_period_fname) as fh:
cpu_period_us = fh.read().strip()
else:
# No Cgroup CPU bandwidth limit (e.g. non-Linux platform)
cpu_quota_us = "max"
cpu_period_us = 100_000 # unused, for consistency with default values
if cpu_quota_us == "max":
# No active Cgroup quota on a Cgroup-capable platform
return os_cpu_count
else:
cpu_quota_us = int(cpu_quota_us)
cpu_period_us = int(cpu_period_us)
if cpu_quota_us > 0 and cpu_period_us > 0:
return math.ceil(cpu_quota_us / cpu_period_us)
else: # pragma: no cover
# Setting a negative cpu_quota_us value is a valid way to disable
# cgroup CPU bandwidth limits
return os_cpu_count
def _cpu_count_affinity(os_cpu_count):
# Number of available CPUs given affinity settings
if hasattr(os, "sched_getaffinity"):
try:
return len(os.sched_getaffinity(0))
except NotImplementedError:
pass
# On PyPy and possibly other platforms, os.sched_getaffinity does not exist
# or raises NotImplementedError, let's try with the psutil if installed.
try:
import psutil
p = psutil.Process()
if hasattr(p, "cpu_affinity"):
return len(p.cpu_affinity())
except ImportError: # pragma: no cover
if (
sys.platform == "linux"
and os.environ.get("LOKY_MAX_CPU_COUNT") is None
):
# PyPy does not implement os.sched_getaffinity on Linux which
# can cause severe oversubscription problems. Better warn the
# user in this particularly pathological case which can wreck
# havoc, typically on CI workers.
warnings.warn(
"Failed to inspect CPU affinity constraints on this system. "
"Please install psutil or explicitly set LOKY_MAX_CPU_COUNT."
)
# This can happen for platforms that do not implement any kind of CPU
# infinity such as macOS-based platforms.
return os_cpu_count
def _cpu_count_user(os_cpu_count):
"""Number of user defined available CPUs"""
cpu_count_affinity = _cpu_count_affinity(os_cpu_count)
cpu_count_cgroup = _cpu_count_cgroup(os_cpu_count)
# User defined soft-limit passed as a loky specific environment variable.
cpu_count_loky = int(os.environ.get("LOKY_MAX_CPU_COUNT", os_cpu_count))
return min(cpu_count_affinity, cpu_count_cgroup, cpu_count_loky)
def _count_physical_cores():
"""Return a tuple (number of physical cores, exception)
If the number of physical cores is found, exception is set to None.
If it has not been found, return ("not found", exception).
The number of physical cores is cached to avoid repeating subprocess calls.
"""
exception = None
# First check if the value is cached
global physical_cores_cache
if physical_cores_cache is not None:
return physical_cores_cache, exception
# Not cached yet, find it
try:
if sys.platform == "linux":
cpu_info = subprocess.run(
"lscpu --parse=core".split(), capture_output=True, text=True
)
cpu_info = cpu_info.stdout.splitlines()
cpu_info = {line for line in cpu_info if not line.startswith("#")}
cpu_count_physical = len(cpu_info)
elif sys.platform == "win32":
cpu_info = subprocess.run(
"wmic CPU Get NumberOfCores /Format:csv".split(),
capture_output=True,
text=True,
)
cpu_info = cpu_info.stdout.splitlines()
cpu_info = [
l.split(",")[1]
for l in cpu_info
if (l and l != "Node,NumberOfCores")
]
cpu_count_physical = sum(map(int, cpu_info))
elif sys.platform == "darwin":
cpu_info = subprocess.run(
"sysctl -n hw.physicalcpu".split(),
capture_output=True,
text=True,
)
cpu_info = cpu_info.stdout
cpu_count_physical = int(cpu_info)
else:
raise NotImplementedError(f"unsupported platform: {sys.platform}")
# if cpu_count_physical < 1, we did not find a valid value
if cpu_count_physical < 1:
raise ValueError(f"found {cpu_count_physical} physical cores < 1")
except Exception as e:
exception = e
cpu_count_physical = "not found"
# Put the result in cache
physical_cores_cache = cpu_count_physical
return cpu_count_physical, exception
if __name__ == '__main__':
cli()
此处可能存在不合适展示的内容，页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容，可点击提交进行申诉，我们将尽快为您处理。