From b9e297ad9145148d2d2783e6ff31506c21be9ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Mon, 6 Apr 2020 11:01:36 +0200 Subject: [PATCH] make: Always use --jobs and support no argument (max jobs) The jobs logic is factored out into `options.py` and used consistently in all places where a `make` (build) is called. It's not used for `make install`. Scripts can now be called with `--jobs` without argument, which will use the max amount of CPU threads reported by the OS. Also adds `-j` alias for `--jobs`. --- android.py | 4 ++-- bcl.py | 13 +++++++------ cmd_utils.py | 4 +++- desktop.py | 4 ++-- ios.py | 4 ++-- llvm.py | 32 ++++++++++++++------------------ options.py | 6 ++++++ reference_assemblies.py | 10 ++++++---- wasm.py | 4 ++-- 9 files changed, 44 insertions(+), 37 deletions(-) diff --git a/android.py b/android.py index 0c2ad55..b538010 100755 --- a/android.py +++ b/android.py @@ -460,8 +460,8 @@ def make(opts: AndroidOpts, product: str, target: str): build_dir = path_join(opts.configure_dir, '%s-%s-%s' % (product, target, opts.configuration)) - make_args = ['-j', opts.jobs, '-C', build_dir] - make_args += ['V=1'] if opts.verbose_make else [] + make_args = make_default_args(opts) + make_args += ['-C', build_dir] run_command('make', args=make_args, name='make') run_command('make', args=['-C', '%s/mono' % build_dir, 'install'], name='make install mono') diff --git a/bcl.py b/bcl.py index 3e3238a..5a3cc1d 100755 --- a/bcl.py +++ b/bcl.py @@ -64,8 +64,8 @@ def make_bcl(opts: BclOpts): build_dir = path_join(opts.configure_dir, 'bcl') - make_args = ['-j', opts.jobs, '-C', build_dir, '-C', 'mono'] - make_args += ['V=1'] if opts.verbose_make else [] + make_args = make_default_args(opts) + make_args += ['-C', build_dir, '-C', 'mono'] run_command('make', args=make_args, name='make bcl') @@ -96,8 +96,8 @@ def make_product(opts: BclOpts, product: str): mkdir_p(install_dir) - make_args = ['-C', build_dir, '-C', 'runtime', 'all-mcs', 'build_profiles=%s' % ' '.join(profiles)] - make_args += ['V=1'] if opts.verbose_make else [] + make_args = make_default_args(opts) + make_args += ['-C', build_dir, '-C', 'runtime', 'all-mcs', 'build_profiles=%s' % ' '.join(profiles)] if product == 'desktop-win32': make_args += ['PROFILE_PLATFORM=win32'] # Requires patch: 'bcl-profile-platform-override.diff' @@ -105,8 +105,9 @@ def make_product(opts: BclOpts, product: str): run_command('make', args=make_args, name='make profiles') if opts.tests and len(test_profiles) > 0: - test_make_args = ['-C', build_dir, '-C', 'runtime', 'test', 'xunit-test', 'test_profiles=%s' % ' '.join(test_profiles)] - test_make_args += ['V=1'] if opts.verbose_make else [] + test_make_args = make_default_args(opts) + test_make_args += ['-C', build_dir, '-C', 'runtime', 'test', 'xunit-test', 'test_profiles=%s' % ' '.join(test_profiles)] + run_command('make', args=test_make_args, name='make tests') # Copy the bcl profiles to the output directory diff --git a/cmd_utils.py b/cmd_utils.py index 2e2ef64..1bdaeda 100644 --- a/cmd_utils.py +++ b/cmd_utils.py @@ -44,7 +44,9 @@ def add_base_arguments(parser, default_help): mono_sources_default = os.environ.get('MONO_SOURCE_ROOT', '') parser.add_argument('--verbose-make', action='store_true', default=False, help=default_help) - parser.add_argument('--jobs', default='1', help=default_help) + # --jobs supports not passing an argument, in which case the 'const' is used, + # which is the number of CPU cores on the host system. + parser.add_argument('--jobs', '-j', nargs='?', const=str(os.cpu_count()), default='1', help=default_help) parser.add_argument('--configure-dir', default=path_join(home, 'mono-configs'), help=default_help) parser.add_argument('--install-dir', default=path_join(home, 'mono-installs'), help=default_help) diff --git a/desktop.py b/desktop.py index 80dcd06..1f6cd4f 100755 --- a/desktop.py +++ b/desktop.py @@ -167,8 +167,8 @@ def configure(opts: DesktopOpts, product: str, target_platform: str, target: str def make(opts: DesktopOpts, product: str, target_platform: str, target: str): build_dir = path_join(opts.configure_dir, '%s-%s-%s' % (product, target, opts.configuration)) - make_args = ['-j', opts.jobs, '-C', build_dir] - make_args += ['V=1'] if opts.verbose_make else [] + make_args = make_default_args(opts) + make_args += ['-C', build_dir] run_command('make', args=make_args, name='make') run_command('make', args=['-C', '%s/mono' % build_dir, 'install'], name='make install mono') diff --git a/ios.py b/ios.py index e457ee4..d67bd9a 100755 --- a/ios.py +++ b/ios.py @@ -424,8 +424,8 @@ def make(opts: iOSOpts, product: str, target: str): build_dir = path_join(opts.configure_dir, '%s-%s-%s' % (product, target, opts.configuration)) - make_args = ['-C', build_dir] - make_args += ['V=1'] if opts.verbose_make else [] + make_args = make_default_args(opts) + make_args += ['-C', build_dir] run_command('make', args=make_args, name='make') run_command('make', args=['-C', '%s/mono' % build_dir, 'install'], name='make install mono') diff --git a/llvm.py b/llvm.py index 03b0127..47e228f 100755 --- a/llvm.py +++ b/llvm.py @@ -66,22 +66,19 @@ def make(opts: BaseOpts, target: str): CMAKE_ARGS += [os.environ.get('llvm-%s_CMAKE_ARGS' % target, '')] - make_args = [ - '-j', opts.jobs, - '-C', '%s/llvm' % opts.mono_source_root, - '-f', 'build.mk', 'install-llvm', - 'LLVM_BUILD=%s' % build_dir, - 'LLVM_PREFIX=%s' % install_dir, - 'LLVM_CMAKE_ARGS=%s' % ' '.join([a for a in CMAKE_ARGS if a]) - ] - - make_args += ['V=1'] if opts.verbose_make else [] - # IMPORTANT: We must specify the jobs count for this Makefile. # The Makefile itself runs Make as well with the '-j' option, which tells it to spawn as many jobs as possible. # This can result in errors like 'posix_spawn failed: Resource temporarily unavailable' on macOS due to the process limit. # The job count seems to be inherited from the parent Make process, so that fixes the issue. - make_args += ['-j', opts.jobs] + # Note: This is handle automatically in make_default_args. + make_args = make_default_args(opts) + make_args += [ + '-C', '%s/llvm' % opts.mono_source_root, + '-f', 'build.mk', 'install-llvm', + 'LLVM_BUILD=%s' % build_dir, + 'LLVM_PREFIX=%s' % install_dir, + 'LLVM_CMAKE_ARGS=%s' % ' '.join([a for a in CMAKE_ARGS if a]) + ] if not find_executable('cmake') and not 'CMAKE' in os.environ: print('WARNING: Cannot find CMake. Required by the llvm Makefile.') @@ -98,15 +95,14 @@ def clean(opts: BaseOpts, target: str): rm_rf(stamp_file) - make_args = [ - '-C', '%s/llvm' % opts.mono_source_root, + make_args = make_default_args(opts) + make_args += [ + '-C', '%s/llvm' % opts.mono_source_root, '-f', 'build.mk', 'clean-llvm', - 'LLVM_BUILD=%s' % build_dir, - 'LLVM_PREFIX=%s' % install_dir + 'LLVM_BUILD=%s' % build_dir, + 'LLVM_PREFIX=%s' % install_dir ] - make_args += ['V=1'] if opts.verbose_make else [] - run_command('make', args=make_args, name='make clean') diff --git a/options.py b/options.py index 7c0424d..b4f3624 100755 --- a/options.py +++ b/options.py @@ -111,3 +111,9 @@ def desktop_opts_from_args(args): **vars(runtime_opts_from_args(args)), with_llvm = args.with_llvm ) + + +def make_default_args(opts: BaseOpts): + make_args = ['-j%s' % opts.jobs] + make_args += ['V=1'] if opts.verbose_make else [] + return make_args diff --git a/reference_assemblies.py b/reference_assemblies.py index 2669b73..9d2b774 100755 --- a/reference_assemblies.py +++ b/reference_assemblies.py @@ -13,8 +13,9 @@ def build(opts: BaseOpts): mkdir_p(install_dir) - make_args = ['-C', build_dir, 'build-reference-assemblies'] - make_args += ['V=1'] if opts.verbose_make else [] + make_args = make_default_args(opts) + make_args += ['-C', build_dir, 'build-reference-assemblies'] + run_command('make', args=make_args, name='make build-reference-assemblies') @@ -24,8 +25,9 @@ def install(opts: BaseOpts): mkdir_p(install_dir) - make_args = ['-C', build_dir, 'install-local', 'DESTDIR=%s' % install_dir, 'prefix=/'] - make_args += ['V=1'] if opts.verbose_make else [] + make_args = make_default_args(opts) + make_args += ['-C', build_dir, 'install-local', 'DESTDIR=%s' % install_dir, 'prefix=/'] + run_command('make', args=make_args, name='make install-local') diff --git a/wasm.py b/wasm.py index 1f9fe21..139d527 100755 --- a/wasm.py +++ b/wasm.py @@ -128,8 +128,8 @@ def make(opts: RuntimeOpts, product: str, target: str): build_dir = path_join(opts.configure_dir, '%s-%s-%s' % (product, target, opts.configuration)) install_dir = path_join(opts.install_dir, '%s-%s-%s' % (product, target, opts.configuration)) - make_args = ['-j', opts.jobs, '-C', build_dir] - make_args += ['V=1'] if opts.verbose_make else [] + make_args = make_default_args(opts) + make_args += ['-C', build_dir] make_env = os.environ.copy() make_env['PATH'] = emsdk_root + ':' + make_env['PATH']