\;jFaddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z ddl m Z ddlmZddlmZmZmZddlmZmZmZdd lmZgZGd d Zd Zd ZdZdZdZ dZ!dZ"dZ#GddZ$ddZ%y)N)core) get_device)_get_trainers_numget_cluster_and_pod)use_paddlecloud)get_cluster_from_args) DeviceModeblock_windows_and_macos check_backend)_prepare_trainer_env_print_argumentsget_host_name_ip) set_flagsceZdZdZy)ParallelEnvArgscXd|_d|_d|_d|_d|_d|_y)NT)cluster_node_ipsnode_ipr started_port print_configselected_devices)selfs aG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/distributed/spawn.py__init__zParallelEnvArgs.__init__/s; $  $!! !%N)__name__ __module__ __qualname__rrrrr.s%rrcgd}gd}|D]6}||vs||vrtjd|zt*td|zy)N) start_methodipsgpusxpusrbackend)rrrrrzThe config option (%s) of `paddle.distributed.spawn` is deprecated. Please use the latest config options stated in the `spawn` API documentation.zFThe config option (%s) of `paddle.distributed.spawn` is not supported.)warningswarnDeprecationWarning ValueError)optionssupported_optionsdeprecated_optionskeys r_options_valid_checkr.Gsi ' '(( d' !\rc>t}d|vrtjSd|vrtjSd|vrt j S|tj vr'tj|jddStd|d)Ngpuxpucpu:rI`paddle.distributed.spawn` does not support parallel training on device `` now.) rrget_cuda_device_countget_xpu_device_countmultiprocessing cpu_countget_available_custom_deviceget_custom_device_countsplit RuntimeErrordevices r_get_default_nprocsr@hs \F ))++ &((** &((** 4335 5++FLL,=a,@AAWX^W__e f  rct}d|vryd|vryd|vry|tjvrytd|d ) Nr0ncclr1bkclr2glooxcclr4r5)rrr:r=r>s r_get_default_backendrFxsW \F  & & 4335 5WX^W__e f  rcd}|jdDcgc]}|j}}t|dk(r|d}|St\}}|Scc}w)N,r)r<striplenr)r"rxnode_ips_s r _get_node_iprOs]G#&99S>2>a >H2 8}1+ N&' 7 N 3sAc  d|vs|ddk(r t|d<t|dt|dg}t}|j dd|_|j *|j dd|_|j d|_|ddk(r|j dd|_|j |j dd|_tjd d}||d k(r4ttjDcgc] }t|}}n|jd }|j ct||krtd t||fzd j!td |Dcgc]}t||c}|_n|j jd }t||k7rt#dt||fz|D]0}||vst#dj%|d j!|n|ddk(r|j dd|_|j |j dd|_tjdd}||d k(r4ttj&Dcgc] }t|}}n|jd }|j ct||krtdt||fzd j!td |Dcgc]}t||c}|_n|j jd }t||k7rt#dt||fz|D]0}||vst#dj%|d j!|n|ddk(rt)j*dd|_d|_|j |_|j ddJdt|j jd dksJdt1dk(sJd|ddk(rd|_tj2d } tjd| d d}||d k(r5ttj4| Dcgc] }t|}}n|jd }t||krtd!t||| fzd j!td |Dcgc]}t||c}|_|j d"d|_|j6t9|j |_|j d#d|_|j dd|_|j<t=|_|ddk(r4t?td |} tA|tBjD| \} } ntG|\} } | jHD]!} |jKtM| | |d#|j d$d%|_'|jNr tQ||Scc}wcc}wcc}wcc}wcc}wcc}w)&Nr%autor"rz 127.0.0.1rBr#rCUDA_VISIBLE_DEVICESrHzthe number of visible devices(%d) is less than the number of spawn processes(%d), please ensure that the correct `nprocs` argument is passed or the environment variable `CUDA_VISIBLE_DEVICES` is correctly configured.rzThe number of selected devices(%s) is not equal to the number of spawn processes(%d), please ensure that the correct `nprocs` and `gpus` arguments are passed.zCThe selected gpu card {} cannot found in CUDA_VISIBLE_DEVICES ({}).rCr$XPU_VISIBLE_DEVICESzthe number of visible devices(%d) is less than the number of spawn processes(%d), please ensure that the correct `nprocs` argument is passed or the environment variable `XPU_VISIBLE_DEVICES` is correctly configured.zThe number of selected devices(%s) is not equal to the number of spawn processes(%d), please ensure that the correct `nprocs` and `xpus` arguments are passed.zBThe selected xpu card {} cannot found in XPU_VISIBLE_DEVICES ({}).rDzYour model will be trained under CPUONLY mode by using GLOO,because CPUPlace is specified manually or your installed PaddlePaddle only support CPU Device.Trz.CPUONLY spawn doesn't support use paddle cloudrIzJCPUONLY spawn only support single trainer, that is len(ips)=1, but got %s.z+CPUONLY spawn doesn't support multi-trainerrEFLAGS_selected_szthe number of visible devices(%d) is less than the number of spawn processes(%d), please ensure that the correct `nprocs` argument is passed or the environment variable `FLAGS_selected_%ss` is correctly configured.rrrF))rFr r rgetrrosgetenvrangerr6strr<rKr=joinr)formatr7r&r'paddle_cpuonlyr"rget_all_custom_device_typer;rrOrrlistrr CPUrtrainersappendr rr )nprocsr*processes_env_listargs env_devicesrLenv_devices_listselected_device_listcard_idcustom_device_namedevices_per_procclusterpodtrainers r_get_subprocess_env_listrps 79#5#?13 ')$%GI./  D$KKt4D $ ' ,> E  ($/D !yV# ' FD 9  ($+KK0BD$ID !ii 6=  +"3 %d&@&@&B C  C1A C   +005   (#$v-"F+,f5 6%(HH38F3CD3Ca%a()3CD%D !$(#8#8#>#>s#C '(F2 H/0&9: 0"22$55;V#SXX.>%?60  v % ' FD 9  ($+KK0BD$ID !ii 5t<  +"3 %d&?&?&A B  B1A B   +005   (#$v-"E+,f5 6%(HH38F3CD3Ca%a()3CD%D !$(#8#8#>#>s#C '(F2 H/0&9: 0"22$44:F#SXX.>%?50  v %  m # $(( KK)4 0 8 < ; < 8 %%++C0 1Q 6 X W X 6  1 $ 9 8 9 $  v % $!<<>qAii/2D1EQ GN  +"3t;;"H-. **;7;;= ACN O ~nrr)rrrrr\rrrrrrs  +6rrc Tt||dk(r t}t||}|jdd}|d}t j |}g} g} g} t |D]} |j} |j}|jt||| ||| |df}||_ |j| j| | j|| j|t| | | }|s|S|js |js|S)a Start multiple processes with ``spawn`` method for parallel training. .. note:: ``spawn`` now only supports GPU or XPU collective mode. The collective mode of GPU and XPU cannot be started at the same time, so the option `gpus` and `xpus` cannot be configured at the same time. Args: func (function): The target function is called by spawned process. This function need to be able to pickled, so it must be defined at the top level of a module. args (list|tuple, optional): Arguments passed to ``func``. nprocs (int, optional): Number of processed to start. Default: -1. when nprocs is -1, the available device will be obtained from the environment variable when the model is executed: If use GPU, the currently available device ID is obtained from the environment variable CUDA_VISIBLE_DEVICES; If use XPU, the currently available device ID is obtained from the environment variable XPU_VISIBLE_DEVICES. join (bool, optional): Perform a blocking join on all spawned processes. Default: True. daemon (bool, optional): The spawned processes' daemon flag. Default: False. **options(dict, optional): Other initial parallel execution environment configuration options. The following options are currently supported: (1) start_method (string): the way to start a process. The start method can be ``spawn`` , ``fork`` , ``forkserver`` . Because the CUDA runtime does not support the ``fork`` start method, when use CUDA in subprocesses, we should start process by ``spawn`` or ``forkserver`` method. Default: "spawn" ; (2) gpus (string): The training process will run on the selected gpus, such as "0,1,2,3". Default: None; (3) xpus (string): The training process will run on the selected xpus, such as "0,1,2,3". Default: None; (5) ips (string): Paddle cluster nodes ips, such as "192.168.0.16,192.168.0.17". Default: "127.0.0.1" . Returns: ``MultiprocessContext`` object, it hold the spawned processes. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env:DISTRIBUTED) >>> import paddle >>> import paddle.nn as nn >>> import paddle.optimizer as opt >>> import paddle.distributed as dist >>> class LinearNet(nn.Layer): ... def __init__(self): ... super().__init__() ... self._linear1 = nn.Linear(10, 10) ... self._linear2 = nn.Linear(10, 1) ... def forward(self, x): ... return self._linear2(self._linear1(x)) >>> def train(print_result=False): ... # 1. initialize parallel environment ... group = dist.init_parallel_env() ... process_group = group.process_group if group else None ... # 2. create data parallel layer & optimizer ... layer = LinearNet() ... dp_layer = paddle.DataParallel(layer, group = process_group) ... loss_fn = nn.MSELoss() ... adam = opt.Adam( ... learning_rate=0.001, parameters=dp_layer.parameters()) ... # 3. run layer ... inputs = paddle.randn([10, 10], 'float32') ... outputs = dp_layer(inputs) ... labels = paddle.randn([10, 1], 'float32') ... loss = loss_fn(outputs, labels) ... if print_result is True: ... print("loss:", loss.numpy()) ... loss.backward() ... adam.step() ... adam.clear_grad() >>> # Usage 1: only pass function. >>> # If your training method no need any argument, and >>> # use all visible devices for parallel training. >>> if __name__ == '__main__': ... dist.spawn(train) >>> # Usage 2: pass function and arguments. >>> # If your training method need some arguments, and >>> # use all visible devices for parallel training. >>> if __name__ == '__main__': ... dist.spawn(train, args=(True,)) >>> # Usage 3: pass function, arguments and nprocs. >>> # If your training method need some arguments, and >>> # only use part of visible devices for parallel training. >>> # If your machine hold 8 cards {0,1,2,3,4,5,6,7}, >>> # this case will use cards {0,1}; If you set >>> # CUDA_VISIBLE_DEVICES=4,5,6,7, this case will use >>> # cards {4,5} >>> if __name__ == '__main__': ... dist.spawn(train, args=(True,), nprocs=2) >>> # Usage 4: pass function, arguments, nprocs and gpus. >>> # If your training method need some arguments, and >>> # only use part of visible devices for parallel training, >>> # but you can't set your machine's environment variable >>> # CUDA_VISIBLE_DEVICES, such as it is None or all cards >>> # {0,1,2,3,4,5,6,7}, you can pass `gpus` to >>> # select the GPU cards you want to use. For example, >>> # this case will use cards {4,5} if your machine hold 8 cards. >>> if __name__ == '__main__': ... dist.spawn(train, args=(True,), nprocs=2, gpus='4,5') r!Nspawnr%)targetrf)r.r@rprWr8 get_contextrZ SimpleQueueProcessrdaemonstartrcrr\)rrfrdr\rr*procs_env_listr!mprrrirrrcontexts rrrs9d!|$& .fg>N;;~t4L  $ $\ 2BLMI 6]nn& ~~' ** q! "     K(\*!%(")\=IG lln lln Nr)rrTF)&r8rXrrr& paddle.baser paddle.devicerpaddle.distributed.cloud_utilsrr$paddle.distributed.fleet.cloud_utilsrpaddle.distributed.fleet.launchr%paddle.distributed.fleet.launch_utilsr r r %paddle.distributed.utils.launch_utilsr r rpaddle.frameworkr__all__rr.r@rFrOrprvr|rrrrrrrs $AA  ' %%2B   zz(20$??Dhr