IB+TF+DOCKER

[fychao]

IB+TF+DOCKER

[fychao]

infiniband test

role	command
Server	ib_write_bw -a -d mlx5_1 &
Client	ib_write_bw -a -F $server_IP -d mlx5_1 --report_gbits

[fychao]

root@83d01dcb874c:/#
************************************
* Waiting for client to connect... *
************************************
---------------------------------------------------------------------------------------
                    RDMA_Write BW Test
 Dual-port       : OFF          Device         : mlx5_1
 Number of qps   : 1            Transport type : IB
 Connection type : RC           Using SRQ      : OFF
 CQ Moderation   : 100
 Mtu             : 4096[B]
 Link type       : IB
 Max inline data : 0[B]
 rdma_cm QPs     : OFF
 Data ex. method : Ethernet
---------------------------------------------------------------------------------------
 local address: LID 0x11 QPN 0x0108 PSN 0xa7f05e RKey 0x0056ec VAddr 0x007fe3411c0000
 remote address: LID 0x12 QPN 0x0109 PSN 0xa7f05e RKey 0x008b5e VAddr 0x007f37d094e000
---------------------------------------------------------------------------------------
 #bytes     #iterations    BW peak[MB/sec]    BW average[MB/sec]   MsgRate[Mpps]
 8388608    5000             95.31              95.31              0.001420
---------------------------------------------------------------------------------------

[1]+  Done                    ib_write_bw -a -d mlx5_1

root@e280959c3867:/# ib_write_bw -a -F 172.16.129.2 -d mlx5_1 --report_gbits
---------------------------------------------------------------------------------------
                    RDMA_Write BW Test
 Dual-port       : OFF          Device         : mlx5_1
 Number of qps   : 1            Transport type : IB
 Connection type : RC           Using SRQ      : OFF
 TX depth        : 128
 CQ Moderation   : 100
 Mtu             : 4096[B]
 Link type       : IB
 Max inline data : 0[B]
 rdma_cm QPs     : OFF
 Data ex. method : Ethernet
---------------------------------------------------------------------------------------
 local address: LID 0x12 QPN 0x0109 PSN 0xa7f05e RKey 0x008b5e VAddr 0x007f37d094e000
 remote address: LID 0x11 QPN 0x0108 PSN 0xa7f05e RKey 0x0056ec VAddr 0x007fe3411c0000
---------------------------------------------------------------------------------------
 #bytes     #iterations    BW peak[Gb/sec]    BW average[Gb/sec]   MsgRate[Mpps]
 2          5000           0.056738            0.055943            3.496431
 4          5000             0.17               0.16               5.140075
 8          5000             0.34               0.33               5.176208
 16         5000             0.69               0.66               5.150361
 32         5000             1.47               1.45               5.673912
 64         5000             3.54               3.44               6.717482
 128        5000             6.92               6.25               6.107578
 256        5000             14.19              14.16              6.915320
 512        5000             14.46              14.20              3.466014
 1024       5000             41.03              39.79              4.857483
 2048       5000             71.03              70.80              4.321457
 4096       5000             82.61              81.94              2.500763
 8192       5000             92.40              92.09              1.405127
 16384      5000             93.99              93.82              0.715753
 32768      5000             94.59              94.58              0.360785
 65536      5000             94.55              94.55              0.180338
 131072     5000             94.72              94.70              0.090315
 262144     5000             95.11              95.11              0.045353
 524288     5000             95.06              95.06              0.022663
 1048576    5000             95.15              95.14              0.011341
 2097152    5000             95.24              95.19              0.005674
 4194304    5000             95.00              95.00              0.002831
 8388608    5000             95.31              95.31              0.001420
---------------------------------------------------------------------------------------

[fychao]

root@83d01dcb874c:/# ipython
Python 2.7.12 (default, Dec  4 2017, 14:50:18)
Type "copyright", "credits" or "license" for more information.

IPython 5.5.0 -- An enhanced Interactive Python.
?         -> Introduction and overview of IPython's features.
%quickref -> Quick reference.
help      -> Python's own help system.
object?   -> Details about 'object', use 'object??' for extra details.

In [1]: import tensorflow as tf

In [2]: tf.__version__
Out[2]: '1.4.0'

In [3]:

[fychao]

# Run the following commands on host_0 (172.16.129.2):
python tf_cnn_benchmarks.py --local_parameter_device=gpu --num_gpus=8 \
--batch_size=64 --model=inception3 --variable_update=distributed_replicated \
--tf_random_seed=6813 --mkl=true --num_epochs=1 --all_reduce_spec=nccl/xring \
--job_name=worker --ps_hosts=172.16.129.2:50000,172.16.130.2:50000 \
--worker_hosts=172.16.129.2:50001,172.16.130.2:50001 --task_index=0

CUDA_VISIBLE_DEVICES=7; python tf_cnn_benchmarks.py --local_parameter_device=gpu --num_gpus=8 \
--batch_size=64 --model=inception3 --variable_update=distributed_replicated \
--tf_random_seed=6813 --mkl=true --num_epochs=1 --all_reduce_spec=nccl/xring \
--job_name=ps --ps_hosts=172.16.129.2:50000,172.16.130.2:50000 \
--worker_hosts=172.16.129.2:50001,172.16.130.2:50001 --task_index=0

# Run the following commands on host_1 (172.16.130.2):
python tf_cnn_benchmarks.py --local_parameter_device=gpu --num_gpus=8 \
--batch_size=64 --model=inception3 --variable_update=distributed_replicated \
--tf_random_seed=6813 --mkl=true --num_epochs=1 --all_reduce_spec=nccl/xring \
--job_name=worker --ps_hosts=172.16.129.2:50000,172.16.130.2:50000 \
--worker_hosts=172.16.129.2:50001,172.16.130.2:50001 --task_index=1

CUDA_VISIBLE_DEVICES=7; python tf_cnn_benchmarks.py --local_parameter_device=gpu --num_gpus=8 \
--batch_size=64 --model=inception3 --variable_update=distributed_replicated \
--tf_random_seed=6813 --mkl=true --num_epochs=1 --all_reduce_spec=nccl/xring \
--job_name=ps --ps_hosts=172.16.129.2:50000,172.16.130.2:50000 \
--worker_hosts=172.16.129.2:50001,172.16.130.2:50001 --task_index=1