Ë
    ˆ\;jî¹  ã                   ó4   — d dl mZ dad„ Zedk(  r e«        yy)é   )ÚContextNc                  óÀ;  — t        «       at        j                  «       rddlm}  | j	                  «        yt        j                  «       �rÐddl}ddl}ddl}ddl	}ddl
}ddl}ddlm} ddlm} ddlm}	m}
m}m}m} dd	lm} |j                  «       }t        j0                  j2                  j5                  d
«      st7        d«      ‚	 t9        t        j0                  j2                  d«      5 }|j;                  |«      }ddd«       |j=                  d«      }|j?                  |j@                  «       |jB                  jE                  |jB                  jG                  t        j0                  j2                  «      |jB                  jI                  t        j0                  j2                  «      jK                  d«      d   › d�«      }|jM                  |d¬«      }|j?                  |j@                  «       |jO                  d«      }|jQ                  |«       |jS                  |«       t        j0                  jT                  j5                  d«      rs|jV                  jY                  d«      dk(  r-|jZ                  ddddddt        j0                  jT                  g}nC|jZ                  dt        j0                  jT                  g}nt        j0                  jT                  g}|j]                  t        j0                  j^                  «       |ja                  t        j0                  j^                  «      }t        j0                  jb                  sd}n2te        t        j0                  jb                  jK                  d«      «      }t        j0                  jf                  }ti        |tj        «      rtm        |jK                  d«      d   «      }ntm        |«      }|d <   ||d!<   ||d    z  |d"<   |jY                  d#d«      sd$d%i|d#<   |jY                  d&d«      }|jB                  jE                  |jB                  jG                  t        j0                  j2                  «      |jB                  jI                  t        j0                  j2                  «      jK                  d«      d   › d'�«      }g }d}|dkD  �rdd(l7m8} d)t        j0                  jr                  v sJ ‚t        j0                  jr                  ju                  d)«      jK                  d«      \  }}  ||| ¬*«      }!|!jw                  d+«       |!jy                  d«       ddl=}"	 |"j}                  «       }#|"j                  |"j�                  |#«      «      }|d,k7  sJ ‚|d#   jY                  d-d«      �r?|jƒ                  d.«       d/|› �}$|!j…                  |$|› j‡                  d0«      «      s4|jƒ                  d«       |!j…                  |$|› j‡                  d0«      «      sŒ4t‰        |!j‹                  d/«      «      }%te        |%«      }&|&|k7  r^|jƒ                  d«       |!j…                  |$|› j‡                  d0«      «       t‰        |!j‹                  d/«      «      }%te        |%«      }&|&|k7  rŒ^t�        |%D �'cg c]  }'|'d   j�                  «       ‘Œ c}'«      }|j‘                  d1te        |«      › d2|› d�«       |jY                  d3d4«      }(|(|d3<   |(t        _I        d5|vr|(n|jY                  d5«      })|jY                  d6d«      }*d7}+ |«       },d}-d}.d8t        j0                  _J        |ja                  t        «      }/|jY                  d9i «      jY                  d:d;«      d;k(  �r´|ja                  |«      }0d<|0d#<    ||0«      }1|1j—                  «       }2d}3|2�rî|ja                  |/«      ad=j™                  |2d:   |2d>   |2d?   |2d@   |2dA   |2dB   |2dC   |2dD   |2dE   «	      }4|4t        j0                  _M        |-dz  }-dFtk        |-«      z   }5|5t        j0                  _N         |
||2|0«      }6|6t        j0                  _/        t        jž                  j‘                  dGj™                  |5|4|2«      «       |j‘                  dGj™                  |5|4|2«      «       |j¡                  t        «      }7|7j£                  «         |t        j0                  jš                  dH|dI   d$   t        j0                  jœ                  › dJ�¬K«      \  }8}9}:|:dz  rKt        jž                  j¥                  dL|4› �«       |j¥                  dL|4› �«       d8|2dM<   d|2|dI   d$   <   |9|2dN<   |:dz  rKt        jž                  j¥                  dO|4› �«       |j¥                  dO|4› �«       d8|2dM<   d|2|dI   d$   <   dP|2dN<   |:dQz  r@|:dz  s;t        jž                  j¥                  dR|4› �«       |j¥                  dR|4› �«       d|2dN<   |:s|8|2dM<   |8|2|dI   d$   <   |9|2dN<   |:dz  s|:dz  rn†|-|2dS<   |2d:   }3 |,j¦                  dŸi |2¤Ž |7j©                  dT¬U«       |,j«                  dV«       |1j—                  «       };|ja                  |;«      }2|1j§                  |2«       |jƒ                  dW«       |2r�Œî|3€t7        dXj™                  |0«      «      ‚|3|d9   d:<   |,j«                  dV«       |,j­                  «        |j                  «       }<t        jž                  j‘                  dY|<|z
  › dZ�«       |j‘                  dY|<|z
  › dZ�«        ||«      }=|j‘                  d[te        |=j®                  j°                  «      › d\�«       |=j—                  «       }>|=j§                  |>«       |>€J d]«       ‚|>�rª|j                  «       }?|ja                  |/«      a|+r|)t        _I        dT}+d:|>v r|>d:   n|d9   d:   }@|@|>d>   z  |>dA   z  |>dC   z  }A|A|>d^<   |@|>d:<   d_|>v r>d`j™                  @|>d>   |>d?   |>d@   |>da   |>dA   |>dB   |>dC   |>dD   |>dE   |>d^   |>d_   «      }4n9dbj™                  @|>d>   |>d?   |>d@   |>da   |>dA   |>dB   |>dC   |>dD   |>dE   |>d^   «      }4|jB                  jE                  |jB                  jG                  t        j0                  j2                  «      |4«      t        j0                  _M        |-dz  }-dctk        |-«      z   }5|5t        j0                  _N         |
||>|«      }B|Bt        j0                  _/        t        jž                  j‘                  dd|5› de|4› �«       |j‘                  dd|5› de|4› �«        |t        |>|«      atm        t        j0                  jf                  jK                  d«      d   «      }C|�r×|dC }D||Dv�rÍ!jY                  df|4› �«      d   }>|j                  «       }E|>sX|j                  «       }F|FEz
  |d3   dgz   kD  rt7        dh|4› di�«      ‚|jƒ                  dW«       |!jY                  df|4› �«      d   }>|>sŒX|j‘                  dj|4› dk�«       t        jž                  j‘                  dj|4› dk�«       |j³                  |>j�                  «       «      }>|=j´                  j·                  d8«       |=j§                  |>«        |,j¦                  dŸi |>¤Ž |,j¹                  |dI   d$   |dI   dl   ¬m«      \  }G}:|:s7t        jž                  j‘                  dnG› �«       |j‘                  dn|G› �«       n0t        jž                  j‘                  do«       |j‘                  do«       d_|>v r|>d_   r |	|>||,jº                  «       |>dp   }H|Hr|.dz  }.|>dq   }Ite        |=j®                  j°                  «      }J|=j®                  j¼                  }Kt        jž                  j‘                  drj™                  |K|J|K|-z
  |.|It¿        |J|Kz
  |(z  dsz  d«      «      «       |j‘                  drj™                  |K|J|K|-z
  |.|It¿        |J|Kz
  |(z  dsz  d«      «      «       |,j«                  |«       |=j—                  «       }L|ja                  |L«      }>|=j§                  |>«       �ŒÝ|j¡                  t        «      }7|d#   d$   d%k7  r.d-|d#   v r'|>dA   dk(  rdt|jV                  du<   ndv|jV                  du<   |7j£                  «        |j                  «       }Mt¿        |M|?z
  d«      |>dw<   t        jž                  j‘                  dxj™                  |5|4|>dw   «      «       |j‘                  dxj™                  |5|4|>dw   «      «        |t        j0                  jš                  dH|dI   d$   t        j0                  jœ                  › dJ�¬K«      \  }8}9}:d7}N|:dz  }OCdkD  �rPdf|-› dy|› �}$OrŒ!j…                  |$dPj‡                  d0«      «      s3|jƒ                  d«       |!j…                  |$dPj‡                  d0«      «      sŒ3t        jž                  j‘                  dz|$› �«       |j‘                  dz|$› �«       �nátÁ        |7d{«      r›|7jÂ                  d|k(  rŒ!j…                  |$d}j‡                  d0«      «      s3|jƒ                  d«       |!j…                  |$d}j‡                  d0«      «      sŒ3t        jž                  j‘                  d~|$› �«       |j‘                  d~|$› �«       �n:tÁ        |7d{«      s¤|7jÄ                  jÆ                  dk(  r‹!j…                  |$d}j‡                  d0«      «      s3|jƒ                  d«       |!j…                  |$d}j‡                  d0«      «      sŒ3t        jž                  j‘                  d~|$› �«       |j‘                  d~|$› �«       nŠ!j…                  |$dj‡                  d0«      «      s3|jƒ                  d«       |!j…                  |$dj‡                  d0«      «      sŒ3t        jž                  j‘                  d€|$› �«       |j‘                  d€|$› �«       t‰        |!j‹                  df|-› dy�«      «      }Pte        |P«      }&|&Ck7  r@|jƒ                  d«       t‰        |!j‹                  df|-› dy�«      «      }Pte        |P«      }&|&Ck7  rŒ@PD �'cg c]  }'|'d   j�                  «       ‘Œ }Q}'t        jž                  j‘                  d�|-› d‚|Q› �«       |j‘                  d�|-› d‚|Q› �«       dP|Qv rdT}Nd7}Ond}QvrdT}NdT}H|:dz  rSt        jž                  j¥                  dƒ|4› d„�«       |j¥                  dƒ|4› d„�«       d8|>dM<   d|>|dI   d$   <   Os|9ndP|>dN<   d7}H|:dz  rMt        jž                  j¥                  |4› d…�«       |j¥                  |4› d…�«       d8|>dM<   d|>|dI   d$   <   dP|>dN<   d7}H|:dQz  rF|:dz  sAt        jž                  j¥                  d†|4› d„�«       |j¥                  d†|4› d„�«       OsdndP|>dN<   HsNr|8|>dM<   |8|>|dI   d$   <   Os|9ndP|>dN<   HsNsd8|>dM<   d|>|dI   d$   <   OsdndP|>dN<   |dI   d$   |>vrd|>|dI   d$   <   |-|>dS<   d‡|d#   v rßdˆ|d#   d‡   v rÕd_|>vrÑ|>|dI   d$      }R|d#   d‡   dˆ   }S |t        j0                  jš                  dH|S¬‰«      }T|d#   d‡   jY                  dŠd‹g«      }Utm        |d#   d‡   jY                  dŒd�«      «      }V|d#   d‡   jY                  dŽdT«      }W|UD ]A  }XWr	Vd�z  Xz  }YnVdQz  Xz  }YRrTrt¿        T|TYz   z  Rz  d.«      nd}Z|Z|>d�X› d‘|dI   d$   › �<   ŒC H|>dp<   |Hr|.dz  }.d}I|I|>dq<   te        |=j®                  j°                  «      }J|=j®                  j¼                  }Kt        jž                  j‘                  drj™                  |K|J|K|-z
  |.|It¿        |J|Kz
  |(z  dsz  d«      «      «       |j‘                  drj™                  |K|J|K|-z
  |.|It¿        |J|Kz
  |(z  dsz  d«      «      «       |r™|d   }||k(  r�!j…                  df|4› �|jÉ                  |>«      j‡                  d0«      «      sE|jƒ                  d«       |!j…                  df|4› �|jÉ                  |>«      j‡                  d0«      «      sŒE|j‘                  |› d’|4› d“�«        |,j¦                  dŸi |>¤Ž |,j¹                  |dI   d$   |dI   dl   ¬m«      \  }G}:|:s7t        jž                  j‘                  dnG› �«       |j‘                  dn|G› �«       n0t        jž                  j‘                  d”«       |j‘                  d”«       d_|>v r|>d_   r |	|>||,jº                  «       |,j«                  |«       |7j©                  dT¬U«       |=j—                  «       }L|ja                  |L«      }>|=j§                  |>«       tk        |jË                  «       «      }[|jÍ                  d•«      jÏ                  «       }\|\D ]5  }]tk        |]ju                  «       «      }^|^[k7  sŒ"|jÑ                  d–^z   «       Œ7 |jƒ                  dW«       |j                  «       }<|*r|<|z
  tm        |*«      kD  rn|>r�Œª|,j«                  |«       d}_|ja                  |/«      a|dkD  rÖ|jV                  jY                  d—d«      }`|`€J ‚|`k(  re|,j¹                  |dI   d$   |dI   dl   |¬˜«      \  }_}:|:rt7        d™«      ‚|jÉ                  _«      }a!j…                  d+a«      sŽ|jƒ                  d«       Œ$tÓ        dš«      D ]:  }'	 !jY                  d+«      d   j�                  «       }a|j³                  |a«      }__sŒ: n _s2J ‚|,j¹                  |dI   d$   |dI   dl   |¬˜«      \  }_}:|:rt7        d™«      ‚_r_dM   d8k7  sJ ‚|j                  «       }<t        jž                  j‘                  d›|<|z
  › dZ�«       |j‘                  d›|<|z
  › dZ�«       |jY                  dœd7«      r|d#   jY                  d-d«      r|j×                  «         |
|_|d7¬�«      }Bd7t        _l        |Bt        j0                  _/        d+t        j0                  _N        t        jž                  j‘                  dž|_› �«       |j‘                  dž|_› �«       |jB                  jE                  |jB                  jG                  t        j0                  j2                  «      d+«      xt        j0                  _M        t        j0                  _M        |j¡                  t        «      }7|7j£                  «        |7j©                  d7¬U«       ydd	lm} |j¡                  t        «      }7|7j£                  «        |7j©                  «        y# 1 sw Y   �Œ[xY w#  t7        d«      ‚xY w#  d,}Y �ŒxY wc c}'w c c}'w # tÔ        $ rL}bt        jž                  j¥                  b«       |j¥                  |b«       |jƒ                  d«       Y d}b~b�Œ¬d}b~bww xY w) av4  
    Paddle distribution training entry ``python -m paddle.distributed.launch``.

    Usage:
        .. code-block:: bash
            :name: code-block-bash1

            python -m paddle.distributed.launch [-h] [--master MASTER] [--rank RANK]
                   [--log_level LOG_LEVEL] [--nnodes NNODES]
                   [--nproc_per_node NPROC_PER_NODE] [--log_dir LOG_DIR]
                   [--run_mode RUN_MODE] [--job_id JOB_ID] [--devices DEVICES]
                   [--host HOST] [--servers SERVERS] [--trainers TRAINERS]
                   [--trainer_num TRAINER_NUM] [--server_num SERVER_NUM]
                   [--gloo_port GLOO_PORT] [--with_gloo WITH_GLOO]
                   [--max_restart MAX_RESTART] [--elastic_level ELASTIC_LEVEL]
                   [--elastic_timeout ELASTIC_TIMEOUT]
                   training_script ...


    Base Parameters:
        - ``--master``: The master/rendezvous server, support ``http://`` and ``etcd://``, default with ``http://``. e.g., ``--master=127.0.0.1:8080``. Default ``--master=None``.

        - ``--rank``: The rank of the node, can be auto assigned by master. Default ``--rank=-1``.

        - ``--log_level``: The log level to set for logging.setLevel which can be CRITICAL/ERROR/WARNING/INFO/DEBUG/NOTSET, case insensitive. Default ``--log_level=INFO``.

        - ``--nnodes``: The number of nodes for a distributed job, it can be a range in elastic mode, e.g., ``--nnodes=2:3``. Default ``--nnodes=1``.

        - ``--nproc_per_node``: The number of processes to launch on a node. In gpu training, it should be less or equal to the gpus number of you system.  e.g., ``--nproc_per_node=8``

        - ``--log_dir``: The path for each process's log. e.g., ``--log_dir=output_dir``. Default ``--log_dir=log``.

        - ``--run_mode``: The run mode of job, can be:collective/ps/ps-heter/rpc. e.g., ``--run_mode=ps``. Default ``--run_mode=collective``.

        - ``--job_id``: The job unique id, it affects the log files' name. e.g., ``--job_id=job1``. Default ``--job_id=default``.

        - ``--devices``: The selected accelerate devices on nodes, can be gpu/xpu etc.. e.g., ``--devices=0,1,2,3`` will launch four training processes each bound to one device.

        - ``training_script``: The full path to the single GPU training program/script to be launched in parallel, followed by all the arguments for the training script. e.g., ``training.py``

        - ``training_script_args``: The args of training_script. e.g., ``--lr=0.1``

    Collective Parameters:
        - ``--ips``: [DEPRECATED] Paddle cluster nodes ips, e.g., ``--ips=192.168.0.16,192.168.0.17``. Default ``--ips=127.0.0.1``.

    Parameter-Server Parameters:
        - ``--servers``: User defined servers ip:port, e.g., ``--servers="192.168.0.16:6170,192.168.0.17:6170"``

        - ``--trainers``: User defined trainers ip:port, e.g., ``--trainers="192.168.0.16:6171,192.168.0.16:6172,192.168.0.17:6171,192.168.0.17:6172"``

        - ``--workers``: [DEPRECATED] The same as trainers.

        - ``--trainer_num``: Number of trainers on each node, can be 0.

        - ``--worker_num``: [DEPRECATED] The same as trainer_num.

        - ``--server_num``: Number of servers on each node, can be 0.

        - ``--heter_workers``: User defined heter workers ip1:port1;ip2:port2, e.g., ``--heter_workers="192.168.0.16:6172;192.168.0.17:6172"``

        - ``--heter_worker_num``: Number of heter_workers in each stage (It recommend to set when in the emulated distributed environment using single node)

        - ``--heter_devices``: Type of heter_device in each stage

        - ``--gloo_port``: Gloo http Port. Default ``--gloo_port=6767``.

        - ``--with_gloo``: Using gloo or not. Default ``--with_gloo=0``.

    Elastic Parameters:
        - ``--max_restart``: The maximum restart times for an elastic job. Default ``--max_restart=3``.

        - ``--elastic_level``: The elastic level: -1: disable, 0: failed exit, peers hold, 1: internal restart. Default ``--elastic_level=-1``.

        - ``--elastic_timeout``: Seconds to wait before elastic job begin to train. Default ``--elastic_timeout=30``.

    IPU Parameters:
        IPU distributed launch only requires and allowes three arguments ``--devices``, ``training_script`` and ``training_script_args``.
        The ``--devices`` is the number of IPU devices. e.g., ``--devices=4`` will launch the training program with four IPU devices.
        The ``training_script`` is only allowed to set as ``ipu``.
        The ``training_script_args`` includes arguments required by IPU distributed launch and illustrated as below.
        ``Examples 10`` has provided a example of paddle.distributed.launch with IPUs.

        - ``--hosts``: The hosts for IPU distributd training. Each host is able to include multiple processes.

        - ``--nproc_per_host``: The number of processes launched per host. Each process is able to include multiple replicas.

        - ``--ipus_per_replica``: The number of IPUs requested per replica. Each replica is able to include multiple IPUs.

        - ``--ipu_partition``: The partition name of IPU devices.

        - ``--vipu_server``: The ip of the IPU device manager.

        - ``training_script``: The full path to the IPU distributed training program/script to be launched in parallel. e.g., ``training.py``.

        - ``training_script_args``: The args of the IPU distributed training program/script. e.g., ``--lr=0.1``.

    Returns:
        - ``None``

    Examples 0 (master, ip/port auto detection):
        .. code-block:: bash
            :name: code-block-example-bash0

            # For training on multi node, run the following command in one of the nodes

            python -m paddle.distributed.launch --nnodes 2 train.py

            # Then the following info will be print

            # Copy the following command to other nodes to run.
            # --------------------------------------------------------------------------------
            # python -m paddle.distributed.launch --master 10.0.0.1:38714 --nnodes 2 train.py
            # --------------------------------------------------------------------------------

            # Follow the instruction above and paste the command in other nodes can launch a multi nodes training job.

            # There are two ways to launch a job with the same command for multi nodes training
            # 1) using the following command in every nodes, make sure the ip is one of the training node and the port is available on that node
            # python -m paddle.distributed.launch --master 10.0.0.1:38714 --nnodes 2 train.py
            # 2) using the following command in every nodes with a independent etcd service
            # python -m paddle.distributed.launch --master etcd://10.0.0.1:2379 --nnodes 2 train.py

            # This functionality works will for both collective and ps mode and even with other arguments.


    Examples 1 (collective, single node):
        .. code-block:: bash
            :name: code-block-example-bash1

            # For training on single node using 4 gpus.

            python -m paddle.distributed.launch --devices=0,1,2,3 train.py --lr=0.01

    Examples 2 (collective, multi node):
        .. code-block:: bash
            :name: code-block-example-bash2

            # For training on multiple nodes, e.g., 192.168.0.16, 192.168.0.17

            # On 192.168.0.16:

            python -m paddle.distributed.launch --devices=0,1,2,3 --master=192.168.0.16:8090 train.py --lr=0.01

            # On 192.168.0.17:
            python -m paddle.distributed.launch --devices=0,1,2,3 --master=192.168.0.16:8090 train.py --lr=0.01

    Examples 3 (ps, cpu, single node):
        .. code-block:: bash
            :name: code-block-example-bash3

            # To simulate distributed environment using single node, e.g., 2 servers and 4 workers.

            python -m paddle.distributed.launch --server_num=2 --worker_num=4 train.py --lr=0.01

    Examples 4 (ps, cpu, multi node):
        .. code-block:: bash
            :name: code-block-example-bash4

            # For training on multiple nodes, e.g., 192.168.0.16, 192.168.0.17 where each node with 1 server and 2 workers.

            # On 192.168.0.16:

            python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.16:6172,192.168.0.17:6171,192.168.0.17:6172" train.py --lr=0.01

            # On 192.168.0.17:

            python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.16:6172,192.168.0.17:6171,192.168.0.17:6172" train.py --lr=0.01

            # Or with master, the following command run 2 server and 2 trainer on each node.

            python -m paddle.distributed.launch --master 192.168.0.16:9090 --server_num=2 --trainer_num=2 --nnodes 2 train.py


    Examples 5 (ps, gpu, single node):
        .. code-block:: bash
            :name: code-block-example-bash5

            # To simulate distributed environment using single node, e.g., 2 servers and 4 workers, each worker use single gpu.

            export CUDA_VISIBLE_DEVICES=0,1,2,3
            python -m paddle.distributed.launch --server_num=2 --worker_num=4 train.py --lr=0.01

    Examples 6 (ps, gpu, multi node):
        .. code-block:: bash
            :name: code-block-example-bash6

            # For training on multiple nodes, e.g., 192.168.0.16, 192.168.0.17 where each node with 1 server and 2 workers.

            # On 192.168.0.16:

            export CUDA_VISIBLE_DEVICES=0,1
            python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.16:6172,192.168.0.17:6171,192.168.0.17:6172" train.py --lr=0.01

            # On 192.168.0.17:

            export CUDA_VISIBLE_DEVICES=0,1
            python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.16:6172,192.168.0.17:6171,192.168.0.17:6172" train.py --lr=0.01

    Examples 7 (ps-heter, cpu + gpu, single node):
        .. code-block:: bash
            :name: code-block-example-bash7

            # To simulate distributed environment using single node, e.g., 2 servers and 4 workers, two workers use gpu, two workers use cpu.

            export CUDA_VISIBLE_DEVICES=0,1
            python -m paddle.distributed.launch --server_num=2 --worker_num=2 --heter_worker_num=2 train.py --lr=0.01

    Examples 8 (ps-heter, cpu + gpu, multi node):
        .. code-block:: bash
            :name: code-block-example-bash8

            # For training on multiple nodes, e.g., 192.168.0.16, 192.168.0.17 where each node with 1 server, 1 gpu worker, 1 cpu worker.

            # On 192.168.0.16:

            export CUDA_VISIBLE_DEVICES=0
            python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.17:6171" --heter_workers="192.168.0.16:6172,192.168.0.17:6172" train.py --lr=0.01

            # On 192.168.0.17:

            export CUDA_VISIBLE_DEVICES=0
            python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.17:6171" --heter_workers="192.168.0.16:6172,192.168.0.17:6172" train.py --lr=0.01

    Examples 9 (elastic):
        .. code-block:: bash
            :name: code-block-example-bash9

            # With the following command, the job will begin to run immediately if 4 nodes are ready,
            # or it will run after elastic_timeout if only 2 or 3 nodes ready
            python -m paddle.distributed.launch --master etcd://10.0.0.1:2379 --nnodes 2:4 train.py

            # once the number of nodes changes between 2:4 during training, the strategy holds

    Examples 10 (ipu):
        .. code-block:: bash
            :name: code-block-example-bash10

            # With the following command, the job will begin to run the distributhed program with IPUs
            # Require `devices` as the number of IPUs
            # Require `training_script` to be set as `ipu`
            # Require `training_script_args` as the arguments of IPU distributed training instead of the arguments of the training program/script
            # Please Check the `IPU Parameters` for details
            python -m paddle.distributed.launch --devices 4 ipu --hosts=localhost --nproc_per_host=2 --ipus_per_replica=1 --ipu_partition=pod16 --vipu_server=127.0.0.1 train.py

    Examples 11 (rpc, cpu, single node):
        .. code-block:: bash
            :name: code-block-example-bash11

            # Training on single node with two local servers
            python -m paddle.distributed.launch --master 127.0.0.1:8765 --nnodes 1 --nproc_per_node 2 --rank 0 --run_mode rpc train.py

    Examples 12 (rpc, cpu, multi node):
        .. code-block:: bash
            :name: code-block-example-bash12

            # For training on multiple nodes, e.g., 192.168.0.16, 192.168.0.17 where each node with 2 servers.

            # On 192.168.0.16

            python -m paddle.distributed.launch --master 192.168.0.16:8765 --nnodes 2 --nproc_per_node 2 --rank 0 --run_mode rpc train.py

            # On 192.168.0.17

            python -m paddle.distributed.launch --master 192.168.0.16:8765 --nnodes 2 --nproc_per_node 2 --rank 1 --run_mode rpc train.py

    é    )ÚlaunchNé   )ÚHistoryRecorder)Ú	AutoTuner)Úadd_overlap_performanceÚgen_new_argsÚgen_new_ctxÚread_logÚread_step_time_logr   )Úcontrollersz.jsonz+Please use '.json' as the file name suffix.Úrz0Please check your auto tuner json whether valid.Ú
auto_tunerÚ.z_auto_tuner.logÚw)Úmodez4%(asctime)s - %(name)s - %(levelname)s - %(message)sz.pyÚWITH_COVERAGEÚONz-uz-mÚcoverageÚrunz--branchz-pé   Ú,Ú:ÚnodesÚgpus_per_nodeÚnum_gpusÚsearch_algoÚnameÚgridr   z_history.csv)Ú
ETCDClientzetcd://)ÚhostÚportÚbest_cfgz	127.0.0.1Úestimated_num_gpusé   zauto_tuner/ip/zlatin-1zThe total count of nodes is z and sorted ips are Úmax_time_per_taski  Úwarmup_timeÚmax_search_timeTéÿÿÿÿÚ	model_cfgÚglobal_batch_sizeÚautoÚgbsz\GBSSearch/GBS{}_DP{}_MP{}_PP{}_Sharding_degree_{}_stage_{}_MBS{}_Recompute_{}_granularity_{}Ú	dp_degreeÚ	mp_degreeÚ	pp_degreeÚsharding_degreeÚsharding_stageÚmicro_batch_sizeÚuse_recomputeÚrecompute_granularityÚ
gbs_tuner_z=Launch task from auto tuner: job_id {}, log_dir {}, config {}zworkerlog.0Ú
metric_cfgz.gpu.log)ÚpathÚmetric_fileÚtarget_metricÚmemory_filez#Read metric failed for parameters: ÚtimeÚmax_mem_usagezOut of memory for parameters: ÚOOMé   z)Read memory usage failed for parameters: Újob_idF)Úexitz./tuner_gbs_history.csvé   zTNo valid global batch size found, check memory or valid search time. cur_tuner_cfg{}z!AtuoTuner for GBS search ends in zs.zLaunch z tasks by auto tuner: zNo config can run.Ú	acc_stepsÚsharding_overlapzdGBS{}_DP{}_MP{}_PP{}_VPP{}_Sharding{}_Stage{}_MBS{}_Recompute_{}_Granularity_{}_AccStep{}_Overlap_{}Ú
vpp_degreezYGBS{}_DP{}_MP{}_PP{}_VPP{}_Sharding{}_Stage{}_MBS{}_Recompute_{}_Granularity_{}_AccStep{}Úauto_tuner_zLaunch task: job_id z
, log_dir zauto_tuner/é   zWait z failedzReceive that task z has ended by etcd.ÚOptimizationDirection)ÚmetricÚ	directionzCurrent best config: z7Get best config failed. Currently no config can be run.Ú	has_errorÚ
error_infozaAuto Tuner Schedule: [{}/{}], Pruned nums {}, Error nums {}, Error info {}, Remaining time {} miné<   Ú1Ú$FLAGS_shard_bypass_dygraph_optimizerÚ0Ú	exec_timez(Task: job_id {}, log_dir {} ended in {}sÚ/zPut OOM to Úsiginté   ÚOKz
Put OK to ÚErrorzPut Error to zStatus of auto_tuner/z/: zRead metric of z failed.z OOM.zRead memory usage of Ú
conversionÚ	step_time)r:   Úfiler<   Úcomm_bwéd   Úmodel_size_bé   Úampé   Úbw_Ú_z put auto_tuner/z successfully.z-Get best config failed, no config can be run.z;fuser -v /dev/nvidia* |awk '{for(i=1;i<=NF;i++) print $i;}'zkill -9 ÚCOLLECTIVE_MASTER_IP)rK   rL   r   zCGet best config failed. Currently there are no appropriate configs.é
   zAutoTuner ended in Úrun_best)rf   zLaunch best cfg: © )mr   ÚctxÚis_legacy_modeÚpaddle.distributed.fleetr   Úis_auto_tuner_modeÚcopyÚjsonÚloggingÚosÚsysr>   Úauto_tuner.recorderr   Úauto_tuner.tunerr	   Úauto_tuner.utilsr
   r   r   r   r   Ú r   ÚargsÚauto_tuner_jsonÚendswithÚ
ValueErrorÚopenÚloadÚ	getLoggerÚsetLevelÚINFOr:   ÚjoinÚdirnameÚbasenameÚsplitÚFileHandlerÚ	FormatterÚsetFormatterÚ
addHandlerÚtraining_scriptÚenvironÚgetÚ
executableÚextendÚtraining_script_argsÚdeepcopyÚdevicesÚlenÚnnodesÚ
isinstanceÚstrÚintÚutils.etcd_clientr"   ÚmasterÚstripÚdeleteÚdelete_prefixÚsocketÚgethostnameÚgethostbynameÚgetfqdnÚsleepÚputÚencodeÚlistÚ
get_prefixÚsortedÚdecodeÚinfor(   Úmax_restartÚsearch_onceÚformatÚlog_dirrB   ÚloggerÚinitr   ÚwarningÚadd_cfgÚfinalizeÚstore_historyÚclean_historyÚalgoÚ	all_tasksÚloadsÚhistory_cfgsÚpopÚget_bestÚhistoryÚidxÚroundÚhasattrrU   ÚpodÚ	exit_codeÚdumpsÚgetpidÚpopenÚ	readlinesÚsystemÚrangeÚ	ExceptionrC   rf   )cr   rl   rm   rn   ro   rp   r>   r   r	   r
   r   r   r   r   r   Ú
start_timeÚfÚ	tuner_cfgr¨   Úauto_tuner_log_pathÚhandlerÚ	formatterÚ
entrypointÚraw_argsr   r�   r   Úhistory_file_pathÚ
sorted_ipsÚipr"   Ú	master_ipr$   Úclientr˜   Úhostnamer:   ÚipsÚsizeÚir(   r)   r*   Úis_first_taskÚrecorderrB   Úerror_task_numsÚraw_ctxÚgbs_tuner_cfgÚ	gbs_tunerÚgbs_cur_cfgÚbest_gbsr§   Útask_job_idÚgbs_new_argsÚcrK   ÚmemÚerrÚgbs_new_cfgÚend_timer   Úcur_cfgÚtask_start_timer-   rE   Únew_argsÚactual_nnodesÚactual_exec_ipsÚwait_start_timeÚwait_end_timeÚcur_best_cfgsrM   rN   Ú	task_numsÚcur_task_idÚnew_cfgÚtask_end_timeÚtimeout_flagÚOOM_flagÚresultÚstatusÚsingle_dp_performanceÚstep_time_metricrZ   r\   r^   r`   ÚbwÚ	comm_timeÚmulti_dp_performaceÚself_pidÚ	processesÚprocessÚpidr%   Úcollective_master_ipÚdataÚesc                                                                                                      úgG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/distributed/launch/main.pyr   r      s”  € ô\ ‹)€Cä
×ÑÔå3à�‰�ä	×	Ñ	Õ	!ÛÛÛÛÛÛå9Ý0÷	
õ 	
õ 	"à—Y‘Y“[ˆ
ä�x‰x×'Ñ'×0Ñ0°Ô9ÜÐJÓKÐKð	QÜ”c—h‘h×.Ñ.°Ô4¸Ø ŸI™I a›L�	÷ 5ð
 ×"Ñ" <Ó0ˆØ�‰˜Ÿ™Ô%Ø Ÿg™gŸl™lØ�G‰G�O‰OœCŸH™H×4Ñ4Ó5Ø�w‰w×Ñ¤§¡× 8Ñ 8Ó9×?Ñ?ÀÓDÀQÑGÐHÈÐXó
Ðð ×%Ñ%Ð&9ÀÐ%ÓDˆØ×Ñ˜Ÿ™Ô&Ø×%Ñ%ØBó
ˆ	ð 	×Ñ˜YÔ'Ø×Ñ˜'Ô"ô �8‰8×#Ñ#×,Ñ,¨UÔ3Ø�z‰z�~‰~˜oÓ.°$Ò6à—N‘NØØØØØØÜ—H‘H×,Ñ,ð	‘
ð "Ÿn™n¨d´C·H±H×4LÑ4LÐM‘
äŸ(™(×2Ñ2Ð3ˆJØ×Ñœ#Ÿ(™(×7Ñ7Ô8Ø—=‘=¤§¡×!>Ñ!>Ó?ˆô �x‰x×ÒØ‰Mä¤§¡× 0Ñ 0× 6Ñ 6°sÓ ;Ó<ˆMÜ—‘—‘ˆÜ�fœcÔ"Ü˜Ÿ™ cÓ*¨1Ñ-Ó.‰Fä˜“[ˆFØ#ˆ	�'ÑØ%2ˆ	�/Ñ"Ø -°	¸'Ñ0BÑ Bˆ	�*ÑØ�}‰}˜]¨DÔ1Ø(.°Ð'7ˆI�mÑ$Ø�}‰}˜V TÓ*ˆàŸG™GŸL™LØ�G‰G�O‰OœCŸH™H×4Ñ4Ó5Ø�w‰w×Ñ¤§¡× 8Ñ 8Ó9×?Ñ?ÀÓDÀQÑGÐHÈÐUó
Ðð ˆ
ØˆØ�A‹:Ý5à¤§¡§¡Ñ/Ð/Ð/Ü!Ÿh™hŸo™o×3Ñ3°IÓ>×DÑDÀSÓI‰OˆI�tÙ Y°TÔ:ˆFØ�M‰M˜*Ô%Ø× Ñ  Ô.ãð!Ø!×-Ñ-Ó/�Ø×)Ñ)¨&¯.©.¸Ó*BÓC�ð ˜Ò$Ð$Ð$Ø˜Ñ'×+Ñ+Ð,@À$ÕGð —
‘
˜1”Ø'¨ tÐ,�Ø Ÿ*™* T¨b¨T¯>©>¸)Ó+DÔEØ—J‘J˜q”Mð !Ÿ*™* T¨b¨T¯>©>¸)Ó+DÕEô ˜6×,Ñ,Ð-=Ó>Ó?�Ü˜3“x�Ø˜f’nØ—J‘J˜q”MØ—J‘J˜t¨ t§^¡^°IÓ%>Ô?Ü˜v×0Ñ0Ð1AÓBÓC�CÜ˜s›8�Dð	 ˜f“nô
 $¹CÓ$@¹C°q Q q¡T§[¡[¥]¸CÑ$@ÓA�
Ø—‘Ø2´3°z³?Ð2CÐCWÐXbÐWcÐcdÐeôð
 &ŸM™MÐ*=¸tÓDÐØ):ˆ	Ð%Ñ&Ø 1ŒÔð
  IÑ-ñ à—‘˜}Ó-ð 	ð $Ÿ-™-Ð(9¸4Ó@ˆàˆá"Ó$ˆàˆØˆØ!Œ�‰ÔØ—-‘-¤Ó$ˆð �M‰M˜+ rÓ*×.Ñ.Ð/BÀFÓKØóð !ŸM™M¨)Ó4ˆMØ+0ˆM˜-Ñ(Ù! -Ó0ˆIà#×/Ñ/Ó1ˆKØˆHÚØ—m‘m GÓ,�Øx×ÑØÐ 3Ñ4Ø Ñ,Ø Ñ,Ø Ñ,ØÐ 1Ñ2ØÐ 0Ñ1ØÐ 2Ñ3Ø Ñ0ØÐ 7Ñ8ó
�ð $+”—‘Ô ð ˜!‘�Ø*¬S°«[Ñ8�Ø"-”—‘”ñ  ,Ø˜k¨=ó �ð 1=”—‘Ô-ô —
‘
—‘ØS×ZÑZØ# W¨kóôð
 —‘ØS×ZÑZØ# W¨kóôð
  ×$Ñ$¤SÓ)�Ø—‘”ñ
 $,ÜŸ™×)Ñ)Ø -Ø"+¨LÑ"9¸&Ñ"AÜ#&§8¡8§?¡?Ð"3°8Ð <ô	$Ñ �˜˜Sð ˜&’>Ü—J‘J×&Ñ&Ø=¸g¸YÐGôð —N‘NØ=¸g¸YÐGôð +-�K Ñ'ØCG�K 	¨,Ñ 7¸Ñ ?Ñ@Ø36�K Ñ0à˜&’>Ü—J‘J×&Ñ&Ø8¸¸	ÐBôð —N‘NÐ%CÀGÀ9Ð#MÔNà*,�K Ñ'ØCG�K 	¨,Ñ 7¸Ñ ?Ñ@Ø38�K Ñ0ð ˜&’>¨#°ª.Ü—J‘J×&Ñ&ØCÀGÀ9ÐMôð —N‘NØCÀGÀ9ÐMôð 48�K Ñ0áà*0�K Ñ'ØCI�K 	¨,Ñ 7¸Ñ ?Ñ@Ø36�K Ñ0à˜&’> S¨F¢^àð )/�˜HÑ%Ø&Ð':Ñ;�Ø �× Ñ Ñ/ ;Ò/Ø—
‘
 �
Ô&Ø×&Ñ&Ð'@ÔAð (×3Ñ3Ó5�Ø"Ÿm™m¨KÓ8�Ø×!Ñ! +Ô.ð —
‘
˜1”óS ðV ÐÜ Øj×qÑqØ%óóð ð ;CˆI�kÑ"Ð#6Ñ7à×"Ñ"Ð#<Ô=Ø×"Ñ"Ô$à—y‘y“{ˆHÜ�J‰J�O‰OØ3°H¸ZÑ4GÐ3HÈÐKôð �K‰KØ3°H¸ZÑ4GÐ3HÈÐKôñ
 ˜yÓ)ˆ
Ø�‰Ø”c˜*Ÿ/™/×3Ñ3Ó4Ð5Ð5KÐLô	
ð ×(Ñ(Ó*ˆØ×Ñ˜7Ô#ØÐ"Ð8Ð$8Ó8Ð"ÚØ"Ÿi™i›kˆOØ—-‘- Ó(ˆCÙØ(3”Ô%Ø!ˆMð '¨'Ñ1ð Ð+Ò,à˜{Ñ+Ð,?Ñ@ð ð "Ø˜;Ñ'ñ(àÐ,Ñ-ñ.ð Ð-Ñ.ñ/ð ð $-ˆG�KÑ Ø+<ˆGÐ'Ñ(Ø! WÑ,ð A÷  Hñ  HØ%Ø˜KÑ(Ø˜KÑ(Ø˜KÑ(Ø˜LÑ)ØÐ-Ñ.ØÐ,Ñ-ØÐ.Ñ/Ø˜OÑ,ØÐ3Ñ4Ø˜KÑ(ØÐ.Ñ/ó‘ð v×|Ñ|Ø%Ø˜KÑ(Ø˜KÑ(Ø˜KÑ(Ø˜LÑ)ØÐ-Ñ.ØÐ,Ñ-ØÐ.Ñ/Ø˜OÑ,ØÐ3Ñ4Ø˜KÑ(ó�ð  "Ÿw™wŸ|™|Ø—‘—‘¤§¡× 8Ñ 8Ó9¸7ó ŒC�H‰HÔð
 �a‰KˆFØ'¬#¨f«+Ñ5ˆKØ)ŒC�H‰HŒOñ $ H¨g°yÓAˆHØ,4ŒC�H‰HÔ)ô �J‰J�O‰OØ& { m°:¸g¸YÐGôð �K‰KÐ.¨{¨m¸:ÀgÀYÐOÔPñ œc 7¨IÓ6ˆCÜ¤§¡§¡× 5Ñ 5°cÓ :¸1Ñ =Ó>ˆMÚØ",¨^¨mÐ"<�Ø˜_Ò,Ø$Ÿj™j¨;°w°iÐ)@ÓAÀ!ÑD�GØ&*§i¡i£k�OÙ%Ø(,¯	©	«˜à)¨OÑ;Ø'Ð(;Ñ<¸rÑAòBô #-¨u°W°I¸WÐ-EÓ"FÐFØŸ
™
 1œØ"(§*¡*¨{¸7¸)Ð-DÓ"EÀaÑ"H˜ò &ð —K‘KØ,¨W¨IÐ5HÐIôô —J‘J—O‘OØ,¨W¨IÐ5HÐIôð #Ÿj™j¨¯©Ó)9Ó:�GØ×+Ñ+×/Ñ/°Ô3Ø×&Ñ& wÔ/Ø$�H×$Ñ$Ñ/ wÒ/Ø)1×):Ñ):Ø(¨Ñ6°vÑ>Ø"+¨LÑ"9Ø3ñ#ð *;ó *Ñ&�M 3ñ ÜŸ
™
Ÿ™Ð*?À¸Ð(OÔPØŸ™Ð&;¸M¸?Ð$KÕLäŸ
™
Ÿ™ØUôð Ÿ™ØUôð +¨gÑ5Ø#Ð$6Ò7á/Ø# Y°×0@Ñ0@ôð !(¨Ñ 4�IÙ Ø'¨1Ñ,˜Ø!(¨Ñ!6�JÜ # J§O¡O×$=Ñ$=Ó >�IØ",§/¡/×"5Ñ"5�KÜ—J‘J—O‘OØ{÷  Cñ  CØ'Ø%Ø'¨&Ñ0Ø+Ø&Ü!Ø!*¨[Ñ!8Ø"3ñ!4à"$ñ!%ð !"ó	óôð —K‘KØ{÷  Cñ  CØ'Ø%Ø'¨&Ñ0Ø+Ø&Ü!Ø!*¨[Ñ!8Ø"3ñ!4à"$ñ!%ð !"ó	óôð ×*Ñ*Ð+<Ô=à(×4Ñ4Ó6�GØ"Ÿm™m¨GÓ4�GØ×&Ñ& wÔ/Ùà× Ñ ¤Ó%ˆAà˜Ñ'¨Ñ/°6Ò9à'¨9°]Ñ+CÑCØÐ0Ñ1°QÒ6ØMP˜Ÿ
™
Ð#IÒJàMP˜Ÿ
™
Ð#IÑJØ�E‰EŒGà ŸI™I›KˆMÜ#(¨¸Ñ)HÈ!Ó#LˆG�KÑ Ü�J‰J�O‰OØ:×AÑAØ ¨'°+Ñ*>óôð
 �K‰KØ:×AÑAØ ¨'°+Ñ*>óôñ  (Ü—X‘X×%Ñ%Ø)Ø'¨Ñ5°fÑ=Ü"Ÿx™xŸ™Ð/¨xÐ8ô	 ÑˆF�C˜ð  ˆLØ˜f‘~ˆHØ˜qÓ Ø$ V H¨A¨b¨TÐ2�ÙØ$Ÿj™j¨¨u¯|©|¸IÓ/FÔGØŸ
™
 1œð %Ÿj™j¨¨u¯|©|¸IÓ/FÕGä—J‘J—O‘O k°$°Ð$8Ô9Ø—K‘K +¨d¨VÐ 4Ö5Ü˜Q Ô)¨a¯h©h¸"ªnØ$Ÿj™j¨¨t¯{©{¸9Ó/EÔFØŸ
™
 1œð %Ÿj™j¨¨t¯{©{¸9Ó/EÕFä—J‘J—O‘O j°°Ð$7Ô8Ø—K‘K *¨T¨FÐ 3Ö4Ü   HÔ-°!·%±%·/±/ÀQÒ2FØ$Ÿj™j¨¨t¯{©{¸9Ó/EÔFØŸ
™
 1œð %Ÿj™j¨¨t¯{©{¸9Ó/EÕFä—J‘J—O‘O j°°Ð$7Ô8Ø—K‘K *¨T¨FÐ 3Õ4à$Ÿj™j¨¨w¯~©~¸iÓ/HÔIØŸ
™
 1œð %Ÿj™j¨¨w¯~©~¸iÓ/HÕIä—J‘J—O‘O m°D°6Ð$:Ô;Ø—K‘K -°¨vÐ 6Ô7ä˜f×/Ñ/°+¸f¸XÀQÐ0GÓHÓI�Ü˜6“{�Ø˜mÒ+Ø—J‘J˜q”MÜ! &×"3Ñ"3°kÀ&ÀÈÐ4KÓ"LÓM�FÜ˜v›;�Dð ˜mÓ+ñ
 28Ó8±¨A˜!˜A™$Ÿ+™+�-°�Ð8Ü—
‘
—‘Ð"7¸°x¸sÀ6À(Ð KÔLØ—‘Ð3°F°8¸3¸v¸hÐGÔHà˜F‘?Ø#(�LØ#‘HØ Ñ'Ø#(�LàˆIØ�fŠ~Ü—
‘
×"Ñ" _°W°I¸XÐ#FÔGØ—‘ °°	¸ÐBÔCà"$�˜‘Ø;?�˜	 ,Ñ/°Ñ7Ñ8Ù6>©3ÀE�˜Ñ(Ø �	à�fŠ~Ü—
‘
×"Ñ" g Y¨eÐ#4Ô5Ø—‘ ' ¨%Ð0Ô1à"$�˜‘Ø;?�˜	 ,Ñ/°Ñ7Ñ8Ø+0�˜Ñ(Ø �	ð �fŠ~ c¨V¢nÜ—
‘
×"Ñ"Ð%:¸7¸)À8Ð#LÔMØ—‘Ð!6°w°i¸xÐHÔIÙ7?©4ÀU�˜Ñ(á¡à"(�˜‘Ø;A�˜	 ,Ñ/°Ñ7Ñ8Ù6>©3ÀE�˜Ñ(á¡\Ø"$�˜‘Ø;?�˜	 ,Ñ/°Ñ7Ñ8Ù7?©4ÀU�˜Ñ(à˜Ñ& vÑ.°gÑ=Ø;?�˜	 ,Ñ/°Ñ7Ñ8à &ˆG�HÑð  	¨-Ñ 8Ñ8Ø 9¨]Ñ#;¸LÑ#IÑIØ&¨gÑ5à(/°	¸,Ñ0GÈÑ0OÑ(PÐ%Ø#,¨]Ñ#;¸LÑ#IØñ$Ð ñ /ÜŸ™×)Ñ)Ø&Ø"2ô�	ð $ MÑ2°<Ñ@×DÑDØ ˜uó�ô  #Ø˜mÑ,¨\Ñ:×>Ñ>Ø&¨óó �ð
   Ñ.¨|Ñ<×@Ñ@ÀÈÓN�Û!�BÙØ$0°EÑ$:¸RÑ$?™	à$0°1Ñ$4°rÑ$9˜	ñ 1±Yô Ø%Ø(¨9Ñ4ñ6à3ñ4ð ô	ð "ð (ð ,ð Ø˜b˜T  9¨\Ñ#:¸6Ñ#BÐ"CÐDòð "ð& $-ˆG�KÑ ÙØ 1Ñ$�ØˆJØ$.ˆG�LÑ!Ü˜JŸO™O×5Ñ5Ó6ˆIØ$Ÿ/™/×-Ñ-ˆKÜ�J‰J�O‰OØs×zÑzØØØ &Ñ(Ø#ØÜØ" [Ñ0Ð4EÑEÈÑJØóó
ôð �K‰KØs×zÑzØØØ &Ñ(Ø#ØÜØ" [Ñ0Ð4EÑEÈÑJØóó
ôñ Ø& q™M�	Ø˜’?Ø$Ÿj™jØ% g YÐ/ØŸ
™
 7Ó+×2Ñ2°9Ó=ôð Ÿ
™
 1œð	 %Ÿj™jØ% g YÐ/ØŸ
™
 7Ó+×2Ñ2°9Ó=õð
 —K‘K 2 $Ð&6°w°i¸~Ð NÔOØˆH×ÑÑ'˜wÒ'Ø!)×!2Ñ!2Ø  Ñ.¨vÑ6Ø# LÑ1Ð2IÑJð "3ó "ÑˆM˜3ñ Ü—
‘
—‘Ð"7¸°Ð GÔHØ—‘Ð3°M°?ÐCÕDä—
‘
—‘Ð OÔPØ—‘ÐKÔLð " WÑ,°Ð9KÒ1LÙ'¨°¸H×<LÑ<LÔMà×"Ñ"Ð#4Ô5Ø�J‰J˜EˆJÔ"ð !×,Ñ,Ó.ˆGØ—m‘m GÓ,ˆGØ×Ñ˜wÔ'ô ˜2Ÿ9™9›;Ó'ˆHØŸ™ØMóç‰i‹kð ó %�Ü˜'Ÿ-™-›/Ó*�Ø˜(“?Ø—I‘I˜j¨3Ñ.Õ/ð %ð �J‰J�qŒMØ—y‘y“{ˆHÙ H¨zÑ$9¼SØó>ò $ð óg ðh 	×ÑÐ0Ô1ð ˆØ�m‰m˜GÓ$ˆØ�AŠ:Ø#%§:¡:§>¡>Ð2HÈ$Ó#OÐ Ø'Ð3Ð3Ð3ØÐ)Ò)Ø (× 1Ñ 1Ø$ \Ñ2°6Ñ:Ø'¨Ñ5Ð6MÑNØð !2ó !‘�˜#ñ
 Ü$Ø]óð ð —z‘z (Ó+�Ø Ÿ*™* Z°Ô6Ø—J‘J˜q”MØä˜rž�Að&Ø%Ÿz™z¨*Ó5°aÑ8×?Ñ?ÓA˜Ø#'§:¡:¨dÓ#3˜ò
  Ùð #ñ  ��xà$×-Ñ-Ø  Ñ.¨vÑ6Ø# LÑ1Ð2IÑJØð .ó ‰MˆH�cñ
 Ü ØYóð ñ ˜H VÑ,°Ò2Ð2Ð2à—9‘9“;ˆÜ�
‰
�‰Ð-¨h°zÑ.AÐ-BÀ"ÐEÔFØ�‰Ð)¨(°:Ñ*=Ð)>¸bÐAÔBð �}‰}˜Z¨Ô.°)¸MÑ2J×2NÑ2NØ  $ô3
ð �H‰HŒJÙ ¨(°IÈÔMˆØŒŒØ(0Œ�‰Ô%Ø$Œ�‰ŒÜ�
‰
�‰Ð+¨H¨:Ð6Ô7Ø�‰Ð'¨ zÐ2Ô3Ø.0¯g©g¯l©lØ�G‰G�O‰OœCŸH™H×4Ñ4Ó5°zó/
ð 	
Œ�‰Ôœ3Ÿ8™8Ô+ð ×ÑœSÓ!ˆØ	�‰ŒØ	�
‰
˜ˆ
Õõ 	"ð ×ÑœSÓ!ˆð 	
�‰Œð 	
�
‰
�÷c 5Ñ4ûð	QÜÐOÓPÐPûðR!Ø “üò" %Aùòb 9øô\ %ò &ÜŸ
™
×*Ñ*¨1Ô-ØŸ™ qÔ)ØŸ
™
 1Ÿšûð&úsl   Ã$Au$ Ã,AuÃ>Au$ Ô>0Au4 ÚAu>ÁQ>AvÁk93AvÁuAu!ÁuAu$ Áu$Au1Áu4Au;Áv	AwÁvAAwÁwAwÚ__main__)Úcontextr   rh   r   Ú__name__rg   ó    rþ   Ú<module>r     s*   ðõ à
€ò_ðD! ˆzÒÙ
…Hð r  