ืื ืืืืขืจืืขืฆืื ื ืคืื ืืขื ืึทืจืืืงื ืืื ืืขืืืขื ืฆืืืขืืจืืื ืืืืฃ ืื ืขืจืขืื ืคืื ืื ืึธื ืืืื ืคืื ืื ืงืืจืก
ืคืื ืื ืืขืจืืขืืืืื ืืจืืื ืื ื ืืืืฃ ืงืืืคื ืืืื-ืคืึธืจืฉืืขืืื ื ืงืึทืืคึผืืืืื ื ืื ืกืืึทื ืกืื ืงืขื ืขื ืจืขืืืฆืืจื ืื ืืจืืื ืื ื ืฆืืื ืคืื ืืึธืืขืจื ืืืฃ ื ืขืืจืึทื ื ืขืืืืึธืจืงืก ืืืืฃ ืืจืืืก ืึทืืึทืื ืฅ ืคืื ืืึทืื ืคืื ืืืึธืื ืฆื ืฉืขื ืึธืืขืจ ืืคืืื ืืื ืื, ืืืึธืก ืืืื ืืขื ืืจืืื ืื ื ืืขืื ืืง ืคืึทืจืฉืคึผืจืืื ืืื ืคึผืจืึทืงืืืฉ ืึทืคึผืืึทืงืืืฉืึทื ื ืคืื ืืืฃ ืืขืจื ืขื. ืืืืขืจื ืืืื ืคึฟืึทืจืฉืืืื ืืื ืฆื ืืืืื ืืื ืกืื ืืงืจืึทื ืืื ืืึทืื ืืื ืงืืืคื ืื ืกืืึทื ืกืื, ืืืึธืก ืืื ืงืขืจ ืืื ืึท ืืืืคึผื ืคึผืจืึทื ืืืืฃ ืกืงืืืืื ื ืขืคืขืงืืืืืงืืึทื. ืืื ืึทืืืฉืึทื, ืืืืขืจื ืืึธื ืืืื ืืืืกื ืืื ืฆื ืฆืขืืืืงืืขื ืึท ืืจืืื ืื ื ืฉืจืืคื ืืืึธืก ืืืืคื ืืืืฃ ืึท ืืืื ืืืึทืฉืคึผืื ืฆื ืงืืืคื ืื ืกืืึทื ืกืื.
ืืื ืืขื ืึทืจืืืงื ืืืจ ืืืขืื ืจืขืื ืืืขืื ืึท ืฉื ืขื ืืื ืืจืื ื ืืืขื ืฆื ืคืึทืจืฉืคึผืจืืืื ืืขืจื ืขื ืืื ืื ืึธืคึฟื ืืืฃ ืืขืจื ืขื ืืืืืืึธืืขืง Apache MXNet ืืื ืื Horovod ืคืื ืื ืืขืจืืขืืืืื ืืขืจื ืขื ืคืจืืืืืืขืจืง. ืืืจ ืืืขืื ืงืืืจ ืืึทืืืืึทืื ืื ืคืึธืจืฉืืขืืื ื ืืขื ืขืคืืฅ ืคืื ืื Horovod ืคืจืืืืืืขืจืง ืืื ืืึทืืืืึทืื ืืื ืฆื ืฉืจืืึทืื ืึท MXNet ืืจืืื ืื ื ืฉืจืืคื ืึทืืื ืึทื ืขืก ืึทืจืืขื ืืื ืึท ืคืื ืื ืืขืจืืขืืืืื ืฉืืืืืขืจ ืืื Horovod.
ืืืึธืก ืืื Apache MXNet
ืืื ืึทื ืึธืคึฟื-ืืงืืจ ืืืฃ ืืขืจื ืขื ืคืจืืืืืืขืจืง ืืืึธืก ืืื ืืขื ืืฆื ืฆื ืฉืึทืคึฟื, ืืึทื ืืื ืฆืขืืืืงืืขื ืืืฃ ื ืขืืจืึทื ื ืขืืืืึธืจืงืก. MXNet ืึทืืกืืจืึทืงืฅ ืื ืงืึทืืคึผืืขืงืกืืืื ืคึฟืึทืจืืื ืื ืืื ืืืคึผืืึทืืขื ืื ื ื ืขืืจืึทื ื ืขืืืืึธืจืงืก, ืืื ืืขืืกื ืคึผืขืจืคืึธืจืืึทื ืกืข ืืื ืกืงืึทืืึทืืืข, ืืื ืึธืคืคืขืจืก ืึทืคึผืืก ืคึฟืึทืจ ืคืึธืืงืก ืคึผืจืึธืืจืึทืืืื ื ืฉืคึผืจืึทืื, ืึทืืึท ืืื , , , , , , ืืื ืื ืืขืจืข.
ืคืื ืื ืืขืจืืขืืืืื ืืจืืื ืื ื ืืื MXNet ืืื ืคึผืึทืจืึทืืขืืขืจ ืกืขืจืืืขืจ
ื ืืฆื ืึท ืคึผืึทืจืึทืืขืืขืจ ืกืขืจืืืขืจ ืฆืืืึทื ื. ืขืก ื ืืฆื ืึท ืืึทื ื ืคืื ืคึผืึทืจืึทืืขืืขืจ ืกืขืจืืืขืจืก ืฆื ืืึทืืืขื ืืจืึทืืืขื ืฅ ืคืื ืืขืืขืจ ืึทืจืืขืืขืจ, ืืืจืืคืืจื ืึทืืืจืขืืึทืืืึธื ืืื ืฉืืงื ืืขืจืืืึทื ืืืงื ืืจืึทืืืขื ืฅ ืฆืืจืืง ืฆื ืืืขืจืก ืคึฟืึทืจ ืืขืจ ืืืืึทืืขืจ ืืืขืจืึทืืืึธื ืคืื ืึทืคึผืืึทืืึทืืืืฉืึทื. ืืืืขืจืืึทื ืื ื ืื ืจืืืืืง ืคืึทืจืืขืืืขื ืืฉ ืคืื ืกืขืจืืืขืจืก ืฆื ืืืขืจืก ืืื ืืขืจ ืฉืืืกื ืฆื ืขืคืขืงืืืื ืกืงืืืืื ื. ืืืื ืขืก ืืื ืืืืื ืืืื ืคึผืึทืจืึทืืขืืขืจ ืกืขืจืืืขืจ, ืขืก ืงืขื ืืืื ืึท ืืึทืืึทืื ืขืง ืืื ืื ืืฉืืื ืืช. ืงืึธื ืืืขืจืกืขืื, ืืืื ืฆื ืคืืืข ืกืขืจืืืขืจืก ืืขื ืขื ืืขื ืืฆื, ืคืืืข-ืฆื-ืคืืืข ืงืึธืืื ืืงืึทืฆืืข ืงืขื ืขื ืคืึทืจืืืืื ืึทืืข ื ืขืฅ ืงืึทื ืขืงืฉืึทื ื.
ืืืึธืก ืืื ืืึธืจืึธืืืึธื
ืืื ืึทื ืึธืคึฟื ืคืื ืื ืืขืจืืขืืืืื ืืืฃ ืืขืจื ืขื ืคืจืืืืืืขืจืง ืืขืืืขืืึธืคึผืขื ืืื ืืืขืจ. ืขืก ืืขืืืขืจืึทืืืฉืึทื ืขืคืขืงืืืื ืงืจืืึทื-ืืคึผื ืืื ืงืจืืึทื-ื ืึธืืข ืืขืงื ืึทืืึทืืืฉืื ืึทืืึท ืืื NVIDIA Collective Communications Library (NCCL) ืืื Message Passing Interface (MPI) ืฆื ืคืึทืจืฉืคึผืจืืืื ืืื ืฆืื ืืืคืืืกื ืืึธืืขื ืคึผืึทืจืึทืืขืืขืจืก ืึทืจืืืขืจ ืืืึธืจืขืงืก. ืขืก ืึธืคึผืืืืืืขืก ืื ื ืืฆื ืคืื ื ืขืฅ ืืึทื ืืืืืื ืืื ืืืึธื ืืขืืื ื ืืืขื ืืจืืขืื ืืื ืืืฃ ื ืขืืจืึทื ื ืขืฅ ืืึธืืขืืก. ืขืก ืืขืจืืืืึทื ืฉืืืฆื ืขืืืขืืข ืคืึธืืงืก ืืึทืฉืื ืืขืจื ืขื ืคืจืึทืืขืืืึธืจืงืก, ื ืืืืื , Tensorflow, Keras ืืื PyTorch.
MXNet ืืื Horovod ืื ืึทืืจืืืฉืึทื
MXNet ืื ืืึทืืจืืืฅ ืืื Horovod ืืืจื ืื ืืืกืืจืืืืืืื ืืขืจื ืขื ืึทืคึผืืก ืืืคืืื ื ืืื Horovod. ืืึธืจืึธืืืึธื ืงืึธืืื ืืงืึทืฆืืข ืึทืคึผืืก horovod.broadcast(), horovod.allgather() ะธ horovod.allreduce() ืืืคึผืืึทืืขื ืึทื ืืื ืืืกืื ืืงืจืึทื ืึทืก ืงืึทืืืืึทืงืงืก ืคืื ืื MXNet ืืึธืืึธืจ, ืืื ืึท ืืืื ืคืื ืืืื ืึทืจืืขื ืืจืึทืคืืง. ืืื ืืขื ืืืขื, ืืึทืื ืืืคึผืขื ืืึทื ืกืื ืฆืืืืฉื ืงืึธืืื ืืงืึทืฆืืข ืืื ืงืึทืืคึผืืึทืืืืฉืึทื ืืขื ืขื ืืืืื ืืึทื ืืึทืื ืืืจื ืื MXNet ืืึธืืึธืจ ืฆื ืืืกืืืืื ืคืึธืจืฉืืขืืื ื ืืึธืกืกืขืก ืจืขืื ืฆื ืกืื ืืงืจืึทื ืึทืืืืฉืึทื. ืคืื ืื ืืขืจืืขืืืืื ืึธืคึผืืืืืืขืจ ืืืืคืขืฅ ืืืคืืื ื ืืื ืืึธืจืึธืืืึธื ืืึธืจืึธืืืึธื.ืืืกืืจืืืืืืึทื ืึธืคึผืืืืืืขืจ ืืงืกืคึผืึทื ืื ืึธืคึผืืืืืืขืจ ืืื MXNet ืึทืืื ืึทื ืขืก ืจืืคื ืื ืงืึธืจืึทืกืคึผืึทื ืืื ื ืืึธืจืึธืืืึธื ืึทืคึผืืก ืคึฟืึทืจ ืคืื ืื ืืขืจืืขืืืืื ืคึผืึทืจืึทืืขืืขืจ ืืขืจืืืึทื ืืืงืื ืืขื. ืึทืืข ืื ืืืคึผืืึทืืขื ืืืืฉืึทื ืืขืืึทืืืก ืืขื ืขื ืืจืึทื ืกืคึผืขืจืึทื ื ืฆื ืื ืกืืฃ ื ืืฆืขืจืก.
ืฉื ืขื ืึธื ืืืื
ืืืจ ืงืขื ืขื ืืขืฉืืืื ื ืึธื ืืืืื ืืจืืื ืื ื ืึท ืงืืืื ืงืึทื ืืืึทืืืฉืึทื ืึทื ื ืขืืจืึทื ื ืขืฅ ืืืืฃ ืื MNIST ืืึทืืึทืืืืก ืืื MXNet ืืื Horovod ืืืืฃ ืืืื ืืึทืงืืึธืึธืง.
ืขืจืฉืืขืจ, ืื ืกืืึทืืืจื mxnet ืืื horovod ืคึฟืื PyPI:
pip install mxnet
pip install horovodืืึทืืขืจืงืื ื: ืืืื ืืืจ ืืจืขืคื ืึท ืืขืืช ืืขืฉืึทืก ืคึผืืคึผ ืื ืกืืึทืืืจื ืืึธืจืึธืืืึธืืืคึฟืฉืจ ืืืจ ืืึทืจืคึฟื ืฆื ืืืืื ืึท ืืืึทืืขืืืืืง MACOSX_DEPLOYMENT_TARGET=10.ืืืืื vv - ืืึธืก ืืื ืื ืืืขืจืกืืข ืคืื โโโโืืืื MacOS ืืืขืจืกืืข, ืคึฟืึทืจ ืืืึทืฉืคึผืื, ืคึฟืึทืจ MacOSX Sierra ืืืจ ืืืขื ืืึทืจืคึฟื ืฆื ืฉืจืืึทืื MACOSX_DEPLOYMENT_TARGET=10.12 ืคึผืืคึผ ืื ืกืืึทืืืจื ืืึธืจืึธืืืึธื
ืืขืจื ืึธื ืื ืกืืึทืืืจื OpenMPI .
ืืื ืื ืกืืฃ, ืืจืืคืงืืคืืข ืื ืคึผืจืืืืจื ืฉืจืืคื mxnet_mnist.py ืืื ืืืืคื ืื ืคืืืืขื ืืข ืงืึทืืึทื ืื ืืื ืื ืืึทืงืืึธืึธืง ืืืึธืงืืึทื ืืื ืื ืึทืจืืขื ืืืขืืืืืึทืืขืจ:
mpirun -np 2 -H localhost:2 -bind-to none -map-by slot python mxnet_mnist.pyืืึธืก ืืืขื ืืืืคื ืืจืืื ืื ื ืืืืฃ ืฆืืืื ืงืึธืจืขืก ืคืื ืืืื ืคึผืจืึทืกืขืกืขืจ. ืืขืจ ืจืขืืืืืึทื ืืืขื ืืืื ืื ืคืืืืขื ืืข:
INFO:root:Epoch[0] Batch [0-50] Speed: 2248.71 samples/sec accuracy=0.583640
INFO:root:Epoch[0] Batch [50-100] Speed: 2273.89 samples/sec accuracy=0.882812
INFO:root:Epoch[0] Batch [50-100] Speed: 2273.39 samples/sec accuracy=0.870000ืคืึธืจืฉืืขืืื ื ืืขืืึธ
ืืืขื ืืจืืื ืื ื ืึท ResNet50-v1 ืืึธืืขื ืืืืฃ ืึท ImageNet ืืึทืืึทืกืขื ืืืืฃ 64 ืืคึผืืก ืืื ืึทืื ืื ืกืืึทื ืกืื p3.16xlarge EC2, ืืขืืขืจ ืืื 8 NVIDIA Tesla V100 GPUs ืืืืฃ AWS ืืืึธืืงื, ืืืจ ืึทืืฉืืืื ืึท ืืจืืื ืื ื ืืจืืคึผืื ืคืื 45000 ืืืืืขืจ / ืกืขืง (ื"ื ืื ื ืืืขืจ ืคืื ืืจืืื ื ืกืึทืืคึผืึทืื ืคึผืขืจ ืกืขืงืื ืืข). ืืจืึทืื ืื ื ืืขืขื ืืืงื ืืื 44 ืืื ืื ื ืึธื 90 ืขืคึผืึธืก ืืื ืึท ืืขืกืืขืจ ืึทืงืืขืจืึทืกื ืคืื 75.7%.
ืืืจ ืงืึทืืคึผืขืจื ืืขื ืฆื MXNet ืก ืคืื ืื ืืขืจืืขืืืืื ืืจืืื ืื ื ืฆืืืึทื ื ืคืื ื ืืฆื ืคึผืึทืจืึทืืขืืขืจ ืกืขืจืืืขืจืก ืืืืฃ 8, 16, 32 ืืื 64 ืืคึผืืก ืืื ืึท ืืืื ืคึผืึทืจืึทืืขืืขืจ ืกืขืจืืืขืจ ืืื ืึท ืกืขืจืืืขืจ ืฆื ืึทืจืืขืืขืจ ืคืึทืจืืขืืืขื ืืฉ ืคืื 1 ืฆื 1 ืืื 2 ืฆื 1 ืจืืกืคึผืขืงืืืืืื. ืืืจ ืงืขื ืขื ืืขื ืื ืจืขืืืืืึทื ืืื ืคืืืืจืข 1 ืืื ืื. ืืืืฃ ืื ื-ืึทืงืก ืืืืฃ ืื ืืื ืงืก, ืื ืืึทืจืก ืจืขืคึผืจืึทืืขื ืฅ ืื ื ืืืขืจ ืคืื ืืืืืขืจ ืฆื ืืึทื ืคึผืขืจ ืกืขืงืื ืืข, ืื ืฉืืจืืช ืคืึทืจืืจืึทืืื ืืื ืื ืกืงืืืืื ื ืขืคืขืงืืืืืงืืึทื (ืืึธืก ืืื, ืื ืคืึทืจืืขืืืขื ืืฉ ืคืื ืคืึทืงืืืฉ ืฆื ืืืขืึทื ืืจืืคึผืื) ืืืืฃ ืื ื-ืึทืงืก ืืืืฃ ืื ืจืขืื. ืืื ืืืจ ืงืขื ืขื ืืขื, ืื ืืจืืจื ืคืื ืื ื ืืืขืจ ืคืื ืกืขืจืืืขืจืก ืึทืคืขืงืฅ ืื ืกืงืืืืื ื ืขืคืขืงืืืืืงืืึทื. ืืืื ืขืก ืืื ืืืืื ืืืื ืคึผืึทืจืึทืืขืืขืจ ืกืขืจืืืขืจ, ืื ืกืงืืืืื ื ืขืคืขืงืืืืืงืืึทื ืคืึทืื ืฆื 38% ืืืืฃ 64 ืืคึผืืก. ืฆื ืืขืจืืจืืืื ืื ืืขืืืข ืกืงืืืืื ื ืขืคืขืงืืืืืงืืึทื ืืื ืืื ืืึธืจืึธืืืึธื, ืืืจ ืืึทืจืคึฟื ืฆื ืืึธืคึผื ืื ื ืืืขืจ ืคืื ืกืขืจืืืขืจืก ืงืึธืจืขืื ืฆื ืื ื ืืืขืจ ืคืื ืืืขืจืก.
ืคืืืืจืข 1. ืคืึทืจืืืืึทื ืคืื ืคืื ืื ืืขืจืืขืืืืื ืืขืจื ืขื ื ืืฆื ืืงืกื ืขื ืืื ืืึธืจืึธืืืึธื ืืื ืืื ืคึผืึทืจืึทืืขืืขืจ ืกืขืจืืืขืจ
ืืื ืืึทืืืข 1 ืืื ืื, ืืืจ ืคืึทืจืืืืึทืื ืื ืืขืฆื ืคึผืจืืึทื ืคึผืขืจ ืืืึทืฉืคึผืื ืืืขื ืคืืืกื ืืืง ืืงืกืคึผืขืจืึทืืึทื ืฅ ืืืืฃ 64 ืืคึผืืก. ื ืืฆื MXNet ืืื Horovod ืืื ืื ืืขืกืืขืจ ืืจืืคึผืื ืฆื ืื ืืึธืืึทืกื ืคึผืจืืึทื.
ืืืฉ 1. ืคึผืจืืึทื ืคืึทืจืืืืึทื ืฆืืืืฉื ืืึธืจืึธืืืึธื ืืื ืคึผืึทืจืึทืืขืืขืจ ืกืขืจืืืืจืขืจ ืืื ืึท ืกืขืจืืืขืจ ืฆื ืึทืจืืขืืขืจ ืคืึทืจืืขืืืขื ืืฉ ืคืื 2 ืฆื 1.
ืกืืขืคึผืก ืฆื ืจืขืคึผืจืึธืืืฆืืจื
ืืื ืืขืจ ืืืืึทืืขืจ ืกืืขืคึผืก, ืืืจ ืืืขืื ืืืืึทืื ืืืจ ืืื ืฆื ืจืขืคึผืจืึธืืืฆืืจื ืื ืจืขืืืืืึทื ืคืื ืคืื ืื ืืขืจืืขืืืืื ืืจืืื ืื ื ืืื MXNet ืืื Horovod. ืืืืขื ืขื ืืขืจ ืืืขืื ืคืื ืื ืืขืจืืขืืืืื ืืขืจื ืขื ืืื MXNet .
ืฉืจืื ืงืกื ืืืงืก
ืฉืึทืคึฟื ืึท ืงื ืืื ืคืื ืืึธืืืึทืืืฉืื ืืึทืก ืื ืกืืึทื ืกืื ืืื MXNet ืืืขืจืกืืข 1.4.0 ืึธืืขืจ ืืขืืขืจ ืืื Horovod ืืืขืจืกืืข 0.16.0 ืึธืืขืจ ืืขืืขืจ ืฆื ื ืืฆื ืคืื ืื ืืขืจืืขืืืืื ืืขืจื ืขื. ืืืจ ืืืขื ืืืื ืืึทืจืคึฟื ืฆื ืื ืกืืึทืืืจื ืืืืืจืขืจืื ืคึฟืึทืจ ืืคึผื ืืจืืื ืื ื. ืคึฟืึทืจ ืืื ืืืขืจ ืื ืกืืึทื ืกืื, ืืืจ ืืืืกืืขืจืืืืืื Ubuntu 16.04 ืืื ืืงืก, ืืื ืืคึผื ืืจืืืืืขืจ 396.44, CUDA 9.2, CUDNN 7.2.1 ืืืืืืึธืืขืง, NCCL 2.2.13 ืงืึทืืืื ืึทืงืืืืขืจ ืืื OpenMPI 3.1.1. ืืืื ืืืจ ืงืขื ืขื ื ืืฆื , ืืื ืื ืืืืืจืขืจืื ืืขื ืขื ืฉืืื ืคืึทืจ-ืืื ืกืืึทืืืจื.
ืฉืจืื ืงืกื ืืืงืก
ืืืื ืื ืคืืืืงืืื ืฆื ืึทืจืืขืื ืืื ืื Horovod API ืฆื ืืืื MXNet ืืจืืื ืื ื ืฉืจืืคื. ืื ืืื ืื ืฉืจืืคื ืืืืืจื ืืืืฃ ืื MXNet Gluon API ืงืขื ืขื ืืืื ืืขืืืืื ื ืืื ืึท ืคึผืฉืื ืืืกืืขืจ. ืื ืฉืืจืืช ืืื ืืจืืืกื ืืขื ืขื ืืืจืฃ ืืืื ืืืจ ืฉืืื ืืึธืื ืึท ืงืึธืจืึทืกืคึผืึทื ืืื ื ืืจืืื ืื ื ืฉืจืืคื. ืืึธ ืืขื ืขื ืึท ืืืกื ืงืจืืืืฉ ืขื ืืขืจืื ืืขื ืืืจ ืืึทืจืคึฟื ืฆื ืืึทืื ืฆื ืืขืจื ืขื ืืื Horovod:
- ืฉืืขืื ืืขื ืงืึธื ืืขืงืกื ืืืื ืื ืืืืข ืืึธืจืึธืืืึธื ืจืื (ืฉืืจื 8) ืฆื ืคึฟืึทืจืฉืืืื ืึทื ืืจืืื ืื ื ืืื ืืืจืืืขืงืึธืื ืืืืฃ ืื ืจืืืืืง ืืจืึทืคืืงืก ืืึทืจืฅ.
- ืคืึธืจื ืขืจืฉื ืคึผืึทืจืึทืืขืืขืจืก ืคืื ืืืื ืึทืจืืขืืขืจ ืฆื ืึทืืข (ืฉืืจื 18) ืฆื ืขื ืฉืืจ ืึทื ืึทืืข ืืืขืจืก ืึธื ืืืืื ืืื ืื ืืขืืืข ืขืจืฉื ืคึผืึทืจืึทืืขืืขืจืก.
- ืฉืึทืคึฟื ืึท ืืึธืจืึธืืืึธื ืืืกืืจืืืืืืื ืึธืคึผืืืืืืขืจ (ืฉืืจื 25) ืฆื ืืขืจืืืึทื ืืืงื ืื ืคึผืึทืจืึทืืขืืขืจืก ืืื ืึท ืคืื ืื ืืขืจืืขืืืืื ืฉืืืืืขืจ.
ืฆื ืืึทืงืืืขื ืื ืคืื ืฉืจืืคื, ืืืืข ืึธืคึผืฉืืงื ืฆื ืื Horovod-MXNet ืืืืฉืคืืื ะธ .
1 import mxnet as mx
2 import horovod.mxnet as hvd
3
4 # Horovod: initialize Horovod
5 hvd.init()
6
7 # Horovod: pin a GPU to be used to local rank
8 context = mx.gpu(hvd.local_rank())
9
10 # Build model
11 model = ...
12
13 # Initialize parameters
14 model.initialize(initializer, ctx=context)
15 params = model.collect_params()
16
17 # Horovod: broadcast parameters
18 hvd.broadcast_parameters(params, root_rank=0)
19
20 # Create optimizer
21 optimizer_params = ...
22 opt = mx.optimizer.create('sgd', **optimizer_params)
23
24 # Horovod: wrap optimizer with DistributedOptimizer
25 opt = hvd.DistributedOptimizer(opt)
26
27 # Create trainer and loss function
28 trainer = mx.gluon.Trainer(params, opt, kvstore=None)
29 loss_fn = ...
30
31 # Train model
32 for epoch in range(num_epoch):
33 ...ืฉืจืื ืงืกื ืืืงืก
ืงืืึธืฅ ืืื ืฆื ืืืื ืขืจ ืคืื ืื ืืืขืจืก ืฆื ืึธื ืืืืื ืคืื ืื ืืขืจืืขืืืืื ืืจืืื ืื ื ื ืืฆื ืื MPI ืืืจืขืงืืืื. ืืื ืืขื ืืืึทืฉืคึผืื, ืคืื ืื ืืขืจืืขืืืืื ืืจืืื ืื ื ืืืืคื ืืืืฃ ืคืืจ ืื ืกืืึทื ืกืื ืืื 4 ืืคึผืืก ืืขืืขืจ, ืืื ืึท ืืึทื ืฅ ืคืื 16 ืืคึผืืก ืืื ืืขื ืงื ืืื. ืื ืกืืึธืืฉืึทืกืืืง ืืจืึทืืืขื ื ืึทืจืึธืคึผืืึทื ื (SGD) ืึธืคึผืืืืืืขืจ ืืืขื ืืืื ืืขืืืืื ื ืืื ืื ืคืืืืขื ืืข ืืืืคึผืขืจืคึผืึทืจืึทืืึทืืขืจ:
- ืืื ื ืคึผืขืงื ืืจืืืก: 256
- ืืขืจื ืขื ืงืืจืก: 0.1
- ืืึธืืขื ืืื: 0.9
- ืืืึธื ืคืึทืจืคืืืื: 0.0001
ืืืขื ืืืจ ืกืงืืืื ืคืื ืืืื ืืคึผื ืฆื 64 ืืคึผืืก, ืืืจ ืืื ืขืึทืจืื ืกืงืืืื ืื ืืจืืื ืื ื ืงืืจืก ืืืื ืื ื ืืืขืจ ืคืื ืืคึผืืก (ืคืื 0,1 ืคึฟืึทืจ 1 ืืคึผื ืฆื 6,4 ืคึฟืึทืจ 64 ืืคึผื), ืืฉืขืช ืืืจ ืืึทืืื ืื ื ืืืขืจ ืคืื ืืืืืขืจ ืคึผืขืจ ืืคึผื ืืื 256 (ืคืื ืึท ืคึผืขืงื ืคืื 256 ืืืืืขืจ ืคึฟืึทืจ 1 ืืคึผื ืฆื 16 ืคึฟืึทืจ 384 ืืคึผื). ืื ืืืึธื ืคืึทืจืคืืืื ืืื ืืึธืืขื ืืื ืคึผืึทืจืึทืืขืืขืจืก ืืขืืืื ืืื ืื ื ืืืขืจ ืคืื ืืคึผืืก ืืขืืืืงืกื. ืืืจ ืืขืืืืื ื ืืขืืืฉื ืคึผืื ืืืขืืงืืึทื ืืจืืื ืื ื ืืื ืื ืคืืึธืื 64 ืืึทืื ืืืคึผ ืคึฟืึทืจ ืื ืคืึธืจืืืก ืคืึธืจื ืืื ืคืืึธืื 16 ืคึฟืึทืจ ืืจืึทืืืึทื ืฅ ืฆื ืคืึทืจืืืืขืจื ืื ืคืืึธืื 32 ืืฉืืื ืืช ืืขืฉืืืฆื ืืืจื NVIDIA ืืขืกืืึท ืืคึผืืก.
$ mpirun -np 16
-H server1:4,server2:4,server3:4,server4:4
-bind-to none -map-by slot
-mca pml ob1 -mca btl ^openib
python mxnet_imagenet_resnet50.pyืกืึธืฃ
ืืื ืืขื ืึทืจืืืงื, ืืืจ ืืขืงืืงื ืืืืฃ ืึท ืกืงืึทืืึทืืืข ืฆืืืึทื ื ืฆื ืคืื ืื ืืขืจืืขืืืืื ืืึธืืขื ืืจืืื ืื ื ืืื Apache MXNet ืืื Horovod. ืืืจ ืืขืืึทื ืกืืจืืืืื ืื ืกืงืืืืื ื ืขืคืขืงืืืืืงืืึทื ืืื ืงืึธืก-ืืคืขืงืืืืื ืึทืก ืงืึทืืคึผืขืจื ืืื ืื ืคึผืึทืจืึทืืขืืขืจ ืกืขืจืืืขืจ ืฆืืืึทื ื ืืืืฃ ืื ImageNet ืืึทืืึทืกืขื ืืืืฃ ืืืึธืก ืื ResNet50-v1 ืืึธืืขื ืืื ืืจืืื ื. ืืืจ ืืึธืื ืืืื ืึทืจืืึทื ืืขืจืขืื ื ืกืืขืคึผืก ืืืึธืก ืืืจ ืงืขื ืขื ื ืืฆื ืฆื ืืึธืืืคืืฆืืจื ืึท ืืืืืกืืื ื ืฉืจืืคื ืฆื ืืืืคื ืืืืื-ืื ืกืืึทื ืก ืืจืืื ืื ื ืืื Horovod.
ืืืื ืืืจ ื ืึธืจ ืึธื ืืืืื ืืื MXNet ืืื ืืืฃ ืืขืจื ืขื, ืืืื ืฆื ืื ืืึทื ืืึธื ืืืจืื ื ืืืึทื ืฆื ืขืจืฉืืขืจ ืืืืขื MXNet. ืืืจ ืืืื ืจืขืงืึธืืขื ืืืจื ืฆื ืืืืขื ืขื ืืขื ืึทืจืืืงื ืฆื ืึธื ืืืืื.
ืืืื ืืืจ ืืึธื ืฉืืื ืืขืืจืืขื ืืื MXNet ืืื ืืืจ ืืืืื ืฆื ืคึผืจืืืืจื ืคืื ืื ืืขืจืืขืืืืื ืืขืจื ืขื ืืื Horovod, ืงืืง ืืื , ืืืืขื ืขืก ืคึฟืื MXNet ืืื ื ืึธืืืืื ืืขื ืืืึทืฉืคึผืื ืึธืืขืจ .
* ืคึผืจืืึทื ืืื ืงืึทืืงืืึทืืืืืื ืืืืืจื ืืืืฃ AWS ืคึฟืึทืจ EC2 ืื ืกืืึทื ืกืื
ืืขืจื ืขื ืืขืจ ืืืขืื ืืขื ืงืืจืก
ืืงืืจ: www.habr.com