| """ |
| This is a script for launching PyTorch inference on Intel(R) Xeon(R) Scalable Processors with optimal configurations. |
| |
| Single instance inference, multi-instance inference are enabled. |
| |
| Note: term "instance" here doesn't refer to a cloud instance. This script is executed as a single process. It invokes |
| multiple "instances" which are formed from multiple threads for each. "instance" is kind of group of threads in this |
| context. |
| |
| Illustrated as below: |
| |
| :: |
| |
| +-----------------------------+----------------------+-------+ |
| | process | thread | core | |
| +=============================+======================+=======+ |
| | torch.backends.xeon.run_cpu | instance 0: thread 0 | 0 | |
| | | thread 1 | 1 | |
| | +----------------------+-------+ |
| | | instance 1: thread 0 | 2 | |
| | | thread 1 | 3 | |
| | +----------------------+-------+ |
| | | ... | ... | |
| | +----------------------+-------+ |
| | | instance N: thread 0 | M | |
| | | thread 1 | M+1 | |
| +-----------------------------+----------------------+-------+ |
| |
| To get the peak performance on Intel(R) Xeon(R) Scalable Processors, the script optimizes the configuration of thread and memory |
| management. For thread management, the script configures thread affinity and the preload of Intel OMP library. |
| For memory management, it configures NUMA binding and preload optimized memory allocation library (e.g. tcmalloc, jemalloc). |
| |
| Environment variables that will be set by this script: |
| |
| +------------------+-------------------------------------------------------------------------------------------------+ |
| | Environ Variable | Value | |
| +==================+=================================================================================================+ |
| | LD_PRELOAD | Depending on knobs you set, <lib>/libiomp5.so, <lib>/libjemalloc.so, <lib>/libtcmalloc.so might | |
| | | be appended to LD_PRELOAD. | |
| +------------------+-------------------------------------------------------------------------------------------------+ |
| | KMP_AFFINITY | If libiomp5.so is preloaded, KMP_AFFINITY could be set to "granularity=fine,compact,1,0". | |
| +------------------+-------------------------------------------------------------------------------------------------+ |
| | KMP_BLOCKTIME | If libiomp5.so is preloaded, KMP_BLOCKTIME is set to "1". | |
| +------------------+-------------------------------------------------------------------------------------------------+ |
| | OMP_NUM_THREADS | value of ncores_per_instance | |
| +------------------+-------------------------------------------------------------------------------------------------+ |
| | MALLOC_CONF | If libjemalloc.so is preloaded, MALLOC_CONF will be set to | |
| | | "oversize_threshold:1,background_thread:true,metadata_thp:auto". | |
| +------------------+-------------------------------------------------------------------------------------------------+ |
| |
| *Note*: This script respects environment variables set preliminarily. I.e. If you set the environment variables |
| mentioned above before running the script, the script will not overwrite the values in the script. |
| |
| How to use this module: |
| ~~~~~~~~~~~~~~~~~~~~~~~ |
| |
| Single instance inference |
| ------------------------- |
| |
| 1. Run single-instance inference on a single node with all CPU nodes. |
| |
| :: |
| |
| python -m torch.backends.xeon.run_cpu --throughput-mode script.py args |
| |
| 2. Run single-instance inference on a single CPU node. |
| |
| :: |
| |
| python -m torch.backends.xeon.run_cpu --node-id 1 script.py args |
| |
| Multi-instance inference |
| ------------------------ |
| |
| 1. Multi-instance |
| By default this tool runs one process per node. If you want to set the instance numbers and core per instance, |
| --ninstances and --ncores-per-instance should be set. |
| |
| :: |
| |
| python -m torch.backends.xeon.run_cpu -- python_script args |
| |
| eg: on an Intel(R) Xeon(R) Scalable Processor with 14 instance, 4 cores per instance |
| |
| :: |
| |
| python -m torch.backends.xeon.run_cpu --ninstances 14 --ncores-per-instance 4 python_script args |
| |
| 2. Run single-instance inference among multiple instances. |
| By default, runs all ninstances. If you want to independently run a single instance among ninstances, specify rank. |
| |
| eg: run 0th instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance (i.e., numactl -C 0-27) |
| |
| :: |
| |
| python -m torch.backends.xeon.run_cpu --ninstances 2 --rank 0 python_script args |
| |
| eg: run 1st instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance (i.e., numactl -C 28-55) |
| |
| :: |
| |
| python -m torch.backends.xeon.run_cpu --ninstances 2 --rank 1 python_script args |
| |
| eg: run 0th instance on an Intel(R) Xeon(R) Scalable Processor with 2 instance, 2 cores per instance, |
| first four cores (i.e., numactl -C 0-1) |
| |
| :: |
| |
| python -m torch.backends.xeon.run_cpu --core-list "0, 1, 2, 3" --ninstances 2 --ncores-per-instance 2 |
| --rank 0 python_script args |
| |
| 3. To look up what optional arguments this module offers: |
| |
| :: |
| |
| python -m torch.backends.xeon.run_cpu --help |
| |
| Memory allocator |
| ---------------- |
| |
| "--enable-tcmalloc" and "--enable-jemalloc" can be used to enable different memory allcator. |
| |
| """ |
| |
| import glob |
| import logging |
| import os |
| import platform |
| import re |
| import subprocess |
| import sys |
| from argparse import ArgumentParser, RawTextHelpFormatter, REMAINDER |
| from os.path import expanduser |
| from typing import Dict, List |
| |
| from torch.distributed.elastic.multiprocessing import start_processes, Std |
| |
| format_str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" |
| logging.basicConfig(level=logging.INFO, format=format_str) |
| logger = logging.getLogger(__name__) |
| |
| |
| class _CPUinfo: |
| """Get CPU information, such as cores list and NUMA information.""" |
| |
| def __init__(self, test_input=""): |
| self.cpuinfo = [] |
| if platform.system() in ["Windows", "Darwin"]: |
| raise RuntimeError(f"{platform.system()} is not supported!!!") |
| elif platform.system() == "Linux": |
| # Sample output of: `lscpu --parse=CPU,Core,Socket,Node` |
| # |
| # # The following is the parsable format, which can be fed to other |
| # # programs. Each different item in every column has an unique ID |
| # # starting from zero. |
| # # CPU,Core,Socket,Node |
| # 0,0,0,0 |
| # 1,1,0,0 |
| # ... |
| if test_input == "": |
| lscpu_cmd = ["lscpu", "--parse=CPU,Core,Socket,Node"] |
| lscpu_info = subprocess.check_output( |
| lscpu_cmd, universal_newlines=True |
| ).split("\n") |
| else: |
| lscpu_info = test_input.split("\n") |
| |
| # Get information about cpu, core, socket and node |
| for line in lscpu_info: |
| pattern = r"^([\d]+,[\d]+,[\d]+,[\d]?)" |
| regex_out = re.search(pattern, line) |
| if regex_out: |
| self.cpuinfo.append(regex_out.group(1).strip().split(",")) |
| |
| # physical cores := core column in lscpu output |
| # logical cores := cPU column in lscpu output |
| self.node_nums = int(max([line[3] for line in self.cpuinfo])) + 1 |
| self.node_physical_cores: List[List[int]] = [] # node_id is index |
| self.node_logical_cores: List[List[int]] = [] # node_id is index |
| self.physical_core_node_map = {} # physical core to numa node id |
| self.logical_core_node_map = {} # logical core to numa node id |
| |
| for node_id in range(self.node_nums): |
| cur_node_physical_core = [] |
| cur_node_logical_core = [] |
| for cpuinfo in self.cpuinfo: |
| nid = cpuinfo[3] if cpuinfo[3] != "" else "0" |
| if node_id == int(nid): |
| if int(cpuinfo[1]) not in cur_node_physical_core: |
| cur_node_physical_core.append(int(cpuinfo[1])) |
| self.physical_core_node_map[int(cpuinfo[1])] = int(node_id) |
| cur_node_logical_core.append(int(cpuinfo[0])) |
| self.logical_core_node_map[int(cpuinfo[0])] = int(node_id) |
| self.node_physical_cores.append(cur_node_physical_core) |
| self.node_logical_cores.append(cur_node_logical_core) |
| |
| def _physical_core_nums(self): |
| return len(self.node_physical_cores) * len(self.node_physical_cores[0]) |
| |
| def _logical_core_nums(self): |
| return len(self.node_logical_cores) * len(self.node_logical_cores[0]) |
| |
| def get_node_physical_cores(self, node_id): |
| if node_id < 0 or node_id > self.node_nums - 1: |
| raise ValueError( |
| f"Invalid node id: {node_id}. Valid node ids: {list(range(len(self.node_physical_cores)))}" |
| ) |
| return self.node_physical_cores[node_id] |
| |
| def get_node_logical_cores(self, node_id): |
| if node_id < 0 or node_id > self.node_nums - 1: |
| raise ValueError( |
| f"Invalid node id: {node_id}. Valid node ids: {list(range(len(self.node_physical_cores)))}" |
| ) |
| return self.node_logical_cores[node_id] |
| |
| def get_all_physical_cores(self): |
| all_cores = [] |
| for cores in self.node_physical_cores: |
| all_cores.extend(cores) |
| return all_cores |
| |
| def get_all_logical_cores(self): |
| all_cores = [] |
| for cores in self.node_logical_cores: |
| all_cores.extend(cores) |
| return all_cores |
| |
| def numa_aware_check(self, core_list): |
| """ |
| Check whether all cores in core_list are in the same NUMA node. |
| |
| Cross NUMA will reduce performance. |
| We strongly advice to not use cores on different nodes. |
| """ |
| cores_numa_map = self.logical_core_node_map |
| numa_ids = [] |
| for core in core_list: |
| numa_id = cores_numa_map[core] |
| if numa_id not in numa_ids: |
| numa_ids.append(numa_id) |
| if len(numa_ids) > 1: |
| logger.warning( |
| "Numa Aware: cores:%s on different NUMA nodes:%s. To avoid \ |
| this behavior, please use --ncores-per-instance knob to make sure number of cores is divisible by --ncores-per-\ |
| instance. Alternatively, please use --skip-cross-node-cores knob.", |
| str(core_list), |
| str(numa_ids), |
| ) |
| if len(numa_ids) == 0: |
| raise RuntimeError( |
| "invalid number of NUMA nodes; please make sure numa_ids >= 1" |
| ) |
| return numa_ids |
| |
| |
| class _Launcher: |
| r"""Class for launcher.""" |
| |
| msg_lib_notfound = f"Unable to find the {{0}} library file lib{{1}}.so in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib \ |
| or /.local/lib/ or /usr/local/lib/ or /usr/local/lib64/ or /usr/lib or /usr/lib64 or \ |
| {expanduser('~')}/.local/lib/ so the LD_PRELOAD environment variable will not be set." |
| |
| def __init__(self): |
| self.cpuinfo = _CPUinfo() |
| |
| def add_lib_preload(self, lib_type): |
| """Enable TCMalloc/JeMalloc/intel OpenMP.""" |
| library_paths = [] |
| if "CONDA_PREFIX" in os.environ: |
| library_paths.append(f"{os.environ['CONDA_PREFIX']}/lib") |
| if "VIRTUAL_ENV" in os.environ: |
| library_paths.append(f"{os.environ['VIRTUAL_ENV']}/lib") |
| |
| library_paths += [ |
| f"{expanduser('~')}/.local/lib", |
| "/usr/local/lib", |
| "/usr/local/lib64", |
| "/usr/lib", |
| "/usr/lib64", |
| ] |
| |
| lib_find = False |
| lib_set = False |
| for item in os.getenv("LD_PRELOAD", "").split(":"): |
| if item.endswith(f"lib{lib_type}.so"): |
| lib_set = True |
| break |
| if not lib_set: |
| for lib_path in library_paths: |
| library_file = os.path.join(lib_path, f"lib{lib_type}.so") |
| matches = glob.glob(library_file) |
| if len(matches) > 0: |
| ld_preloads = [f"{matches[0]}", os.getenv("LD_PRELOAD", "")] |
| os.environ["LD_PRELOAD"] = os.pathsep.join( |
| [p.strip(os.pathsep) for p in ld_preloads if p] |
| ) |
| lib_find = True |
| break |
| return lib_set or lib_find |
| |
| def is_numactl_available(self): |
| numactl_available = False |
| try: |
| cmd = ["numactl", "-C", "0", "-m", "0", "hostname"] |
| r = subprocess.run( |
| cmd, |
| env=os.environ, |
| stdout=subprocess.DEVNULL, |
| stderr=subprocess.DEVNULL, |
| check=False, |
| ) |
| if r.returncode == 0: |
| numactl_available = True |
| except Exception: |
| pass |
| return numactl_available |
| |
| def set_memory_allocator( |
| self, enable_tcmalloc=True, enable_jemalloc=False, use_default_allocator=False |
| ): |
| """ |
| Enable TCMalloc/JeMalloc with LD_PRELOAD and set configuration for JeMalloc. |
| |
| By default, PTMalloc will be used for PyTorch, but TCMalloc and JeMalloc can get better |
| memory reuse and reduce page fault to improve performance. |
| """ |
| if enable_tcmalloc and enable_jemalloc: |
| raise RuntimeError( |
| "Unable to enable TCMalloc and JEMalloc at the same time." |
| ) |
| |
| if enable_tcmalloc: |
| find_tc = self.add_lib_preload(lib_type="tcmalloc") |
| if not find_tc: |
| msg = f'{self.msg_lib_notfound} you can use "conda install -c conda-forge gperftools" to install {{0}}' |
| logger.warning(msg.format("TCmalloc", "tcmalloc")) # noqa: G001 |
| else: |
| logger.info("Use TCMalloc memory allocator") |
| |
| elif enable_jemalloc: |
| find_je = self.add_lib_preload(lib_type="jemalloc") |
| if not find_je: |
| msg = f'{self.msg_lib_notfound} you can use "conda install -c conda-forge jemalloc" to install {{0}}' |
| logger.warning(msg.format("Jemalloc", "jemalloc")) # noqa: G001 |
| else: |
| logger.info("Use JeMalloc memory allocator") |
| self.set_env( |
| "MALLOC_CONF", |
| "oversize_threshold:1,background_thread:true,metadata_thp:auto", |
| ) |
| |
| elif use_default_allocator: |
| pass |
| |
| else: |
| find_tc = self.add_lib_preload(lib_type="tcmalloc") |
| if find_tc: |
| logger.info("Use TCMalloc memory allocator") |
| return |
| find_je = self.add_lib_preload(lib_type="jemalloc") |
| if find_je: |
| logger.info("Use JeMalloc memory allocator") |
| return |
| logger.warning( |
| """Neither TCMalloc nor JeMalloc is found in $CONDA_PREFIX/lib or $VIRTUAL_ENV/lib |
| or /.local/lib/ or /usr/local/lib/ or /usr/local/lib64/ or /usr/lib or /usr/lib64 or |
| %s/.local/lib/ so the LD_PRELOAD environment variable will not be set. |
| This may drop the performance""", |
| expanduser("~"), |
| ) |
| |
| def log_env_var(self, env_var_name=""): |
| if env_var_name in os.environ: |
| logger.info("%s=%s", env_var_name, os.environ[env_var_name]) |
| |
| def set_env(self, env_name, env_value): |
| if not env_value: |
| logger.warning("%s is None", env_name) |
| if env_name not in os.environ: |
| os.environ[env_name] = env_value |
| elif os.environ[env_name] != env_value: |
| logger.warning( |
| "Overriding value with the one set in environment variable: %s. \ |
| Value applied: %s. Value ignored: %s", |
| env_name, |
| os.environ[env_name], |
| env_value, |
| ) |
| self.log_env_var(env_name) |
| |
| # set_kmp_affinity is used to control whether to set KMP_AFFINITY or not. |
| # In scenario that use all cores on all nodes, including logical cores, setting KMP_AFFINITY disables logical cores. |
| # In this case, KMP_AFFINITY should not be set. |
| def set_multi_thread_and_allocator( |
| self, |
| ncores_per_instance, |
| disable_iomp=False, |
| set_kmp_affinity=True, |
| enable_tcmalloc=True, |
| enable_jemalloc=False, |
| use_default_allocator=False, |
| ): |
| """ |
| Set multi-thread configuration and enable Intel openMP and TCMalloc/JeMalloc. |
| |
| By default, GNU openMP and PTMalloc are used in PyTorch. but Intel openMP and TCMalloc/JeMalloc are better alternatives |
| to get performance benefit. |
| """ |
| self.set_memory_allocator( |
| enable_tcmalloc, enable_jemalloc, use_default_allocator |
| ) |
| self.set_env("OMP_NUM_THREADS", str(ncores_per_instance)) |
| if not disable_iomp: |
| find_iomp = self.add_lib_preload(lib_type="iomp5") |
| if not find_iomp: |
| msg = f'{self.msg_lib_notfound} you can use "conda install mkl" to install {{0}}' |
| logger.warning(msg.format("iomp", "iomp5")) # noqa: G001 |
| else: |
| logger.info("Using Intel OpenMP") |
| if set_kmp_affinity: |
| self.set_env("KMP_AFFINITY", "granularity=fine,compact,1,0") |
| self.set_env("KMP_BLOCKTIME", "1") |
| self.log_env_var("LD_PRELOAD") |
| |
| r""" |
| Launcher for single instance and multi-instance |
| """ |
| |
| def launch(self, args): |
| cores = [] |
| set_kmp_affinity = True |
| enable_taskset = False |
| if args.core_list: # user specify what cores will be used by params |
| cores = [int(x) for x in args.core_list.split(",")] |
| if args.ncores_per_instance == -1: |
| raise RuntimeError( |
| 'please specify the "--ncores-per-instance" if you have pass the --core-list params' |
| ) |
| elif ( |
| args.ninstances > 1 |
| and args.ncores_per_instance * args.ninstances < len(cores) |
| ): |
| logger.warning( |
| "only first %s cores will be used, \ |
| but you specify %s cores in core_list", |
| args.ncores_per_instance * args.ninstances, |
| len(cores), |
| ) |
| else: |
| args.ninstances = len(cores) // args.ncores_per_instance |
| |
| else: |
| if args.use_logical_core: |
| if args.node_id != -1: |
| cores = self.cpuinfo.get_node_logical_cores(args.node_id) |
| else: |
| cores = self.cpuinfo.get_all_logical_cores() |
| # When using all cores on all nodes, including logical cores, |
| # setting KMP_AFFINITY disables logical cores. Thus, KMP_AFFINITY should not be set. |
| set_kmp_affinity = False |
| else: |
| if args.node_id != -1: |
| cores = self.cpuinfo.get_node_physical_cores(args.node_id) |
| else: |
| cores = self.cpuinfo.get_all_physical_cores() |
| if ( |
| not args.multi_instance |
| and args.ninstances == -1 |
| and args.ncores_per_instance == -1 |
| ): |
| args.ninstances = 1 |
| args.ncores_per_instance = len(cores) |
| elif ( |
| args.multi_instance |
| and args.ninstances == -1 |
| and args.ncores_per_instance == -1 |
| ): |
| args.throughput_mode = True |
| elif args.ncores_per_instance == -1 and args.ninstances != -1: |
| if args.ninstances > len(cores): |
| raise RuntimeError( |
| f"there are {len(cores)} total cores but you specify {args.ninstances} ninstances; \ |
| please make sure ninstances <= total_cores)" |
| ) |
| else: |
| args.ncores_per_instance = len(cores) // args.ninstances |
| elif args.ncores_per_instance != -1 and args.ninstances == -1: |
| if not args.skip_cross_node_cores: |
| args.ninstances = len(cores) // args.ncores_per_instance |
| else: |
| ncore_per_node = len(self.cpuinfo.node_physical_cores[0]) |
| num_leftover_cores = ncore_per_node % args.ncores_per_instance |
| if args.ncores_per_instance > ncore_per_node: |
| # too many ncores_per_instance to skip cross-node cores |
| logger.warning( |
| "there are %s core(s) per socket, but you specify %s ncores_per_instance and \ |
| skip_cross_node_cores. Please make sure --ncores-per-instance < core(s) per \ |
| socket", |
| ncore_per_node, |
| args.ncores_per_instance, |
| ) |
| sys.exit(-1) |
| elif num_leftover_cores == 0: |
| # aren't any cross-node cores |
| logger.info( |
| "--skip-cross-node-cores is set, but there are no cross-node cores." |
| ) |
| args.ninstances = len(cores) // args.ncores_per_instance |
| else: |
| # skip cross-node cores |
| if args.ninstances != -1: |
| logger.warning( |
| "--skip-cross-node-cores is exclusive to --ninstances. --ninstances \ |
| won't take effect even if it is set explicitly." |
| ) |
| |
| i = 1 |
| leftover_cores = set() |
| while ncore_per_node * i <= len(cores): |
| leftover_cores.update( |
| cores[ |
| ncore_per_node * i |
| - num_leftover_cores : ncore_per_node * i |
| ] |
| ) |
| i += 1 |
| cores = list(set(cores) - leftover_cores) |
| assert len(cores) % args.ncores_per_instance == 0 |
| args.ninstances = len(cores) // args.ncores_per_instance |
| else: |
| if args.ninstances * args.ncores_per_instance > len(cores): |
| raise RuntimeError( |
| "Please make sure ninstances * ncores_per_instance <= total_cores" |
| ) |
| if args.latency_mode: |
| logger.warning( |
| "--latency-mode is exclusive to --ninstances, --ncores-per-instance, --node-id and \ |
| --use-logical-core. They won't take effect even they are set explicitly." |
| ) |
| args.ncores_per_instance = 4 |
| cores = self.cpuinfo.get_all_physical_cores() |
| args.ninstances = len(cores) // args.ncores_per_instance |
| |
| if args.throughput_mode: |
| logger.warning( |
| "--throughput-mode is exclusive to --ninstances, --ncores-per-instance, --node-id and \ |
| --use-logical-core. They won't take effect even they are set explicitly." |
| ) |
| args.ninstances = self.cpuinfo.node_nums |
| cores = self.cpuinfo.get_all_physical_cores() |
| args.ncores_per_instance = len(cores) // args.ninstances |
| |
| if args.ninstances > 1 and args.rank != -1: |
| logger.info( |
| "assigning %s cores for instance %s", |
| args.ncores_per_instance, |
| args.rank, |
| ) |
| |
| if not args.disable_numactl: |
| numactl_available = self.is_numactl_available() |
| if not numactl_available: |
| if not args.disable_taskset: |
| logger.warning( |
| "Core binding with numactl is not available. Disabling numactl and using taskset instead. \ |
| This may affect performance in multi-socket system; please use numactl if memory binding is needed." |
| ) |
| args.disable_numactl = True |
| enable_taskset = True |
| else: |
| logger.warning( |
| "Core binding with numactl is not available, and --disable_taskset is set. \ |
| Please unset --disable_taskset to use taskset instead of numactl." |
| ) |
| sys.exit(-1) |
| |
| if not args.disable_taskset: |
| enable_taskset = True |
| |
| self.set_multi_thread_and_allocator( |
| args.ncores_per_instance, |
| args.disable_iomp, |
| set_kmp_affinity, |
| args.enable_tcmalloc, |
| args.enable_jemalloc, |
| args.use_default_allocator, |
| ) |
| entrypoint = "" |
| launch_args = {} |
| launch_envs: Dict[int, Dict] = {} |
| launch_tee = {} |
| for i in range(args.ninstances): |
| cmd = [] |
| cur_process_cores = "" |
| if not args.disable_numactl or enable_taskset: |
| if not args.disable_numactl: |
| cmd = ["numactl"] |
| elif enable_taskset: |
| cmd = ["taskset"] |
| cores = sorted(cores) |
| if ( |
| args.rank == -1 |
| ): # sequentially assign ncores_per_instance to ninstances |
| core_list = cores[ |
| i |
| * args.ncores_per_instance : (i + 1) |
| * args.ncores_per_instance |
| ] |
| else: # assign ncores_per_instance from rank |
| core_list = cores[ |
| args.rank |
| * args.ncores_per_instance : (args.rank + 1) |
| * args.ncores_per_instance |
| ] |
| |
| core_ranges: List[Dict] = [] |
| for core in core_list: |
| if len(core_ranges) == 0: |
| range_elem = {"start": core, "end": core} |
| core_ranges.append(range_elem) |
| else: |
| if core - core_ranges[-1]["end"] == 1: |
| core_ranges[-1]["end"] = core |
| else: |
| range_elem = {"start": core, "end": core} |
| core_ranges.append(range_elem) |
| for r in core_ranges: |
| cur_process_cores = f"{cur_process_cores}{r['start']}-{r['end']}," |
| cur_process_cores = cur_process_cores[:-1] |
| if not args.disable_numactl: |
| numa_params = f"-C {cur_process_cores} " |
| numa_ids = ",".join( |
| [ |
| str(numa_id) |
| for numa_id in self.cpuinfo.numa_aware_check(core_list) |
| ] |
| ) |
| numa_params += f"-m {numa_ids}" |
| cmd.extend(numa_params.split()) |
| elif enable_taskset: |
| taskset_params = f"-c {cur_process_cores} " |
| cmd.extend(taskset_params.split()) |
| with_python = not args.no_python |
| if with_python: |
| cmd.append(sys.executable) |
| cmd.append("-u") |
| if args.module: |
| cmd.append("-m") |
| cmd.append(args.program) |
| cmd.extend(args.program_args) |
| cmd_s = " ".join(cmd) |
| logger.info(cmd_s) |
| if entrypoint == "": |
| entrypoint = cmd[0] |
| del cmd[0] |
| launch_args[i] = tuple(cmd) |
| launch_envs[i] = {} |
| launch_tee[i] = Std.ALL |
| |
| if args.rank != -1: # launches single instance, rank, only |
| break |
| |
| ctx = start_processes( |
| name=args.log_file_prefix, |
| entrypoint=entrypoint, |
| args=launch_args, |
| envs=launch_envs, |
| log_dir=args.log_path, |
| tee=launch_tee, |
| ) |
| ctx.wait() |
| |
| |
| def _add_memory_allocator_params(parser): |
| group = parser.add_argument_group("Memory Allocator Parameters") |
| # allocator control |
| group.add_argument( |
| "--enable-tcmalloc", |
| "--enable_tcmalloc", |
| action="store_true", |
| default=False, |
| help="Enable tcmalloc allocator", |
| ) |
| group.add_argument( |
| "--enable-jemalloc", |
| "--enable_jemalloc", |
| action="store_true", |
| default=False, |
| help="Enable jemalloc allocator", |
| ) |
| group.add_argument( |
| "--use-default-allocator", |
| "--use_default_allocator", |
| action="store_true", |
| default=False, |
| help="Use default memory allocator", |
| ) |
| |
| |
| def _add_multi_instance_params(parser): |
| group = parser.add_argument_group("Multi-instance Parameters") |
| # multi-instance control |
| group.add_argument( |
| "--ncores-per-instance", |
| "--ncores_per_instance", |
| metavar="\b", |
| default=-1, |
| type=int, |
| help="Cores per instance", |
| ) |
| group.add_argument( |
| "--ninstances", |
| metavar="\b", |
| default=-1, |
| type=int, |
| help="For multi-instance, you should give the cores number you used for per instance.", |
| ) |
| group.add_argument( |
| "--skip-cross-node-cores", |
| "--skip_cross_node_cores", |
| action="store_true", |
| default=False, |
| help="If specified --ncores-per-instance, skips cross-node cores.", |
| ) |
| group.add_argument( |
| "--rank", |
| metavar="\b", |
| default="-1", |
| type=int, |
| help="Specify instance index to assign ncores_per_instance for rank; \ |
| otherwise ncores_per_instance will be assigned sequentially to ninstances. Please refer to \ |
| https://github.com/intel/intel-extension-for-pytorch/blob/master/docs/tutorials/performance_tuning/launch_script.md", |
| ) |
| group.add_argument( |
| "--latency-mode", |
| "--latency_mode", |
| action="store_true", |
| default=False, |
| help="By default 4 core per instance and use all physical cores", |
| ) |
| group.add_argument( |
| "--throughput-mode", |
| "--throughput_mode", |
| action="store_true", |
| default=False, |
| help="By default one instance per node and use all physical cores", |
| ) |
| group.add_argument( |
| "--node-id", |
| "--node_id", |
| metavar="\b", |
| default=-1, |
| type=int, |
| help="node id for multi-instance, by default all nodes will be used", |
| ) |
| group.add_argument( |
| "--use-logical-core", |
| "--use_logical_core", |
| action="store_true", |
| default=False, |
| help="Whether only use physical cores", |
| ) |
| group.add_argument( |
| "--disable-numactl", |
| "--disable_numactl", |
| action="store_true", |
| default=False, |
| help="Disable numactl", |
| ) |
| group.add_argument( |
| "--disable-taskset", |
| "--disable_taskset", |
| action="store_true", |
| default=False, |
| help="Disable taskset", |
| ) |
| group.add_argument( |
| "--core-list", |
| "--core_list", |
| metavar="\b", |
| default=None, |
| type=str, |
| help='Specify the core list as "core_id, core_id, ....", otherwise, all the cores will be used.', |
| ) |
| group.add_argument( |
| "--log-path", |
| "--log_path", |
| metavar="\b", |
| default="", |
| type=str, |
| help="The log file directory. Default path is " |
| ", which means disable logging to files.", |
| ) |
| group.add_argument( |
| "--log-file-prefix", |
| "--log_file_prefix", |
| metavar="\b", |
| default="run", |
| type=str, |
| help="log file prefix", |
| ) |
| |
| |
| def _add_kmp_iomp_params(parser): |
| group = parser.add_argument_group("IOMP Parameters") |
| group.add_argument( |
| "--disable-iomp", |
| "--disable_iomp", |
| action="store_true", |
| default=False, |
| help="By default, we use Intel OpenMP and libiomp5.so will be add to LD_PRELOAD", |
| ) |
| |
| |
| def create_args(parser=None): |
| """ |
| Parse the command line options. |
| |
| @retval ArgumentParser |
| """ |
| parser.add_argument( |
| "--multi-instance", |
| "--multi_instance", |
| action="store_true", |
| default=False, |
| help="Enable multi-instance, by default one instance per node", |
| ) |
| |
| parser.add_argument( |
| "-m", |
| "--module", |
| default=False, |
| action="store_true", |
| help="Changes each process to interpret the launch script " |
| "as a python module, executing with the same behavior as" |
| '"python -m".', |
| ) |
| |
| parser.add_argument( |
| "--no-python", |
| "--no_python", |
| default=False, |
| action="store_true", |
| help='Do not prepend the --program script with "python" - just exec ' |
| "it directly. Useful when the script is not a Python script.", |
| ) |
| |
| _add_memory_allocator_params(parser) |
| _add_kmp_iomp_params(parser) |
| |
| _add_multi_instance_params(parser) |
| # positional |
| parser.add_argument( |
| "program", |
| type=str, |
| help="The full path to the program/script to be launched. " |
| "followed by all the arguments for the script", |
| ) |
| |
| # rest from the training program |
| parser.add_argument("program_args", nargs=REMAINDER) |
| |
| |
| def main(args): |
| env_before = set(os.environ.keys()) |
| if platform.system() in ["Windows", "Darwin"]: |
| raise RuntimeError(f"{platform.system()} is not supported!!!") |
| |
| if args.log_path: |
| os.makedirs(args.log_path, exist_ok=True) |
| else: |
| args.log_path = os.devnull |
| |
| if args.latency_mode and args.throughput_mode: |
| raise RuntimeError( |
| "Either args.latency_mode or args.throughput_mode should be set" |
| ) |
| |
| if not args.no_python and not args.program.endswith(".py"): |
| raise RuntimeError( |
| 'For non Python script, you should use "--no-python" parameter.' |
| ) |
| |
| # Verify LD_PRELOAD |
| if "LD_PRELOAD" in os.environ: |
| lst_valid = [] |
| tmp_ldpreload = os.environ["LD_PRELOAD"] |
| for item in tmp_ldpreload.split(":"): |
| matches = glob.glob(item) |
| if len(matches) > 0: |
| lst_valid.append(item) |
| else: |
| logger.warning("%s doesn't exist. Removing it from LD_PRELOAD.", item) |
| if len(lst_valid) > 0: |
| os.environ["LD_PRELOAD"] = ":".join(lst_valid) |
| else: |
| os.environ["LD_PRELOAD"] = "" |
| |
| launcher = _Launcher() |
| launcher.launch(args) |
| for x in sorted(set(os.environ.keys()) - env_before): |
| logger.debug("%s=%s", x, os.environ[x]) |
| |
| |
| if __name__ == "__main__": |
| parser = ArgumentParser( |
| description="This is a script for launching PyTorch inference on Intel(R) Xeon(R) Scalable " |
| "Processors with optimal configurations. Single instance inference, " |
| "multi-instance inference are enable. To get the peak performance on Intel(R) " |
| "Xeon(R) Scalable Processors, the script optimizes the configuration " |
| "of thread and memory management. For thread management, the script configures thread " |
| "affinity and the preload of Intel OMP library. For memory management, it configures " |
| "NUMA binding and preload optimized memory allocation library (e.g. tcmalloc, jemalloc) " |
| "\n################################# Basic usage ############################# \n" |
| "\n 1. single instance\n" |
| "\n >>> python -m torch.backends.xeon.run_cpu python_script args \n" |
| "\n2. multi-instance \n" |
| "\n >>> python -m torch.backends.xeon.run_cpu --ninstances xxx " |
| "--ncores-per-instance xx python_script args\n" |
| "\n############################################################################# \n", |
| formatter_class=RawTextHelpFormatter, |
| ) |
| create_args(parser) |
| args = parser.parse_args() |
| main(args) |
