Tensorflow multi GPU training using distribute strategies vs Horovod

[leondgarse]

Install horovod

• NVIDIA Collective Communications Library (NCCL) Download Page
• Horovod on GPU

# Not sure if gcc-8 is still needed
sudo apt install gcc-8 g++-8
sudo rm /etc/alternatives/c++ && sudo ln -s /usr/bin/x86_64-linux-gnu-g++-8 /etc/alternatives/c++

sudo apt install openmpi-bin

# Install nvidia-driver and relative NCCL
# If NCCL version not matching CUDA, `horovodrun` may meet ValueError: Neither MPI nor Gloo support has been built
sudo dpkg -i nvidia-machine-learning-repo-ubuntu2004_1.0.0-1_amd64.deb
sudo apt update
sudo apt install libnccl2=2.8.3-1+cuda11.2 libnccl-dev=2.8.3-1+cuda11.2
sudo apt-mark hold libnccl-dev libnccl2

# If throws error: ‘IsOutRange’ is not a member of ‘flatbuffers’, update flatbuffers>=1.12.0
# Download flatbuffers release from: https://github.com/google/flatbuffers/releases
sudo apt remove libflatbuffers-dev flatbuffers-compiler flatbuffers-compiler-dev libflatbuffers1
tar xvf flatbuffers-1.12.0.tar.gz && cd flatbuffers-1.12.0
mkdir build && cd build && cmake .. && make && sudo make install && cd ../..

# Install horovod with NCCL
HOROVOD_WITH_MPI=1 HOROVOD_WITH_TENSORFLOW=1 HOROVOD_GPU_OPERATIONS=NCCL pip install horovod
horovodrun --version
# 0.23.0

# Tensorflow version
pip show tensorflow
# Version: 2.6.0

# Hardware
nvidia-smi
# NVIDIA GeForce RTX 2080 Ti

git clone https://github.com/horovod/horovod.git
cd horovod/examples/tensorflow2/

CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow2_keras_mnist.py

CUDA_VISIBLE_DEVICES='1' horovodrun -np 1 -H localhost:1 python tensorflow2_keras_synthetic_benchmark.py --model MobileNet
# [1,0]<stdout>:Total img/sec on 1 GPU(s): 289.6 +-20.9
CUDA_VISIBLE_DEVICES='1' horovodrun -np 1 -H localhost:1 python tensorflow2_keras_synthetic_benchmark.py --model MobileNet --batch-size 64
# [1,0]<stdout>:Total img/sec on 1 GPU(s): 262.5 +-10.6
CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow2_keras_synthetic_benchmark.py --model MobileNet
# [1,0]<stdout>:Total img/sec on 2 GPU(s): 543.2 +-29.4
CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow2_keras_synthetic_benchmark.py --model MobileNet --fp16-allreduce
# [1,0]<stdout>:Total img/sec on 2 GPU(s): 537.4 +-15.9
CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow2_keras_synthetic_benchmark.py --model MobileNet --batch-size 64
# [1,0]<stdout>:Total img/sec on 2 GPU(s): 497.2 +-6.5

Tensorflow horovod

#!/usr/bin/env python3
import tensorflow as tf
import horovod.tensorflow.keras as hvd
import argparse
import sys
import numpy as np

parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-b", "--batch_size", type=int, help="batch_size", default=8)
parser.add_argument("-e", "--epochs", type=int, help="epochs", default=10)
parser.add_argument("-m", "--model_name", type=str, help="model name", default="MobileNet")
parser.add_argument("-f", "--use_fp16", action="store_true", help="Use fp16")
parser.add_argument('--fp16_allreduce', action='store_true', default=False, help='fp16 compression allreduce')
args = parser.parse_known_args(sys.argv[1:])[0]

# Horovod: initialize Horovod.
hvd.init()
print(">>>> hvd.rank:", hvd.rank(), "hvd.size:", hvd.size())

# Horovod: pin GPU to be used to process local rank (one GPU per process)
gpus = tf.config.experimental.list_physical_devices('GPU')
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

if args.use_fp16:
    tf.keras.mixed_precision.set_global_policy('mixed_float16')

if gpus:
    tf.config.experimental.set_visible_devices(gpus[hvd.local_rank()], 'GPU')

data = np.random.uniform(size=[1024, 224, 224, 3])
target = np.random.uniform(size=[1024, 1], low=0, high=999).astype("int64")
dataset = tf.data.Dataset.from_tensor_slices((data, target)).repeat().batch(args.batch_size)
steps_per_epoch = int(np.ceil(data.shape[0] / args.batch_size))

model = getattr(tf.keras.applications, args.model_name)(weights=None)

opt = tf.optimizers.SGD(0.01)
compression = hvd.Compression.fp16 if args.fp16_allreduce else hvd.Compression.none
opt = hvd.DistributedOptimizer(opt, compression=compression)

model.compile(loss=tf.losses.SparseCategoricalCrossentropy(), optimizer=opt, experimental_run_tf_function=False)
callbacks = [
    hvd.callbacks.BroadcastGlobalVariablesCallback(0),
    hvd.callbacks.MetricAverageCallback(),
]
verbose = 1 if hvd.rank() == 0 else 0
model.fit(dataset, steps_per_epoch=steps_per_epoch // hvd.size(), callbacks=callbacks, epochs=args.epochs, verbose=verbose)

Tensorflow distribute

#!/usr/bin/env python3
import tensorflow as tf
import argparse
import sys
import numpy as np

parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-s", "--strategy", type=int, help="{1: OneDeviceStrategy, 2: MirroredStrategy, 3: MultiWorkerMirroredStrategy}", default=1)
parser.add_argument("-b", "--batch_size", type=int, help="batch size", default=8)
parser.add_argument("-e", "--epochs", type=int, help="epochs", default=10)
parser.add_argument("-m", "--model_name", type=str, help="model name", default="MobileNet")
parser.add_argument("-f", "--use_fp16", action="store_true", help="Use fp16")
args = parser.parse_known_args(sys.argv[1:])[0]

gpus = tf.config.experimental.list_physical_devices('GPU')
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

if args.use_fp16:
    tf.keras.mixed_precision.set_global_policy('mixed_float16')

if args.strategy == 2:
    strategy = tf.distribute.MirroredStrategy()
elif args.strategy == 3:
    strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy(tf.distribute.experimental.CollectiveCommunication.NCCL)
else:
    strategy = tf.distribute.OneDeviceStrategy(device="/gpu:0")

batch_size = args.batch_size * strategy.num_replicas_in_sync
data = np.random.uniform(size=[1024, 224, 224, 3])
target = np.random.uniform(size=[1024, 1], low=0, high=999).astype("int64")
dataset = tf.data.Dataset.from_tensor_slices((data, target)).batch(batch_size)

with strategy.scope():
    model = getattr(tf.keras.applications, args.model_name)(weights=None)

# opt = tf.optimizers.Adam(0.001)
opt = tf.optimizers.SGD(0.01)
model.compile(optimizer=opt, loss=tf.keras.losses.SparseCategoricalCrossentropy())
callbacks = []
model.fit(dataset, callbacks=callbacks, epochs=args.epochs, verbose=1)

Test results

echo "========== MobileNet distribute =========="
CUDA_VISIBLE_DEVICES='1' python tensorflow_distribute.py -b 32 -s 2 -e 6 -m MobileNet
CUDA_VISIBLE_DEVICES='1' python tensorflow_distribute.py -b 64 -s 2 -e 6 -m MobileNet

CUDA_VISIBLE_DEVICES='0,1' python tensorflow_distribute.py -b 32 -s 2 -e 6 -m MobileNet
CUDA_VISIBLE_DEVICES='0,1' python tensorflow_distribute.py -b 64 -s 2 -e 6 -m MobileNet
CUDA_VISIBLE_DEVICES='0,1' python tensorflow_distribute.py -b 64 -s 2 -e 6 -m MobileNet -f
CUDA_VISIBLE_DEVICES='0,1' python tensorflow_distribute.py -b 128 -s 2 -e 6 -m MobileNet -f

echo "========== MobileNet horovod =========="
CUDA_VISIBLE_DEVICES='1' horovodrun -np 1 -H localhost:1 python tensorflow_horovod.py -b 32 -e 6 -m MobileNet
CUDA_VISIBLE_DEVICES='1' horovodrun -np 1 -H localhost:1 python tensorflow_horovod.py -b 64 -e 6 -m MobileNet

CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow_horovod.py -b 32 -e 6 -m MobileNet
CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow_horovod.py -b 64 -e 6 -m MobileNet
CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow_horovod.py -b 64 -e 6 -m MobileNet -f
CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow_horovod.py -b 128 -e 6 -m MobileNet -f

echo "========== ResNet50 distribute =========="
CUDA_VISIBLE_DEVICES='1' python tensorflow_distribute.py -b 64 -s 2 -e 6 -m ResNet50 -f
CUDA_VISIBLE_DEVICES='0,1' python tensorflow_distribute.py -b 64 -s 2 -e 6 -m ResNet50 -f

echo "========== ResNet50 horovod =========="
CUDA_VISIBLE_DEVICES='1' horovodrun -np 1 -H localhost:1 python tensorflow_horovod.py -b 64 -e 6 -m ResNet50 -f
CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow_horovod.py -b 64 -e 6 -m ResNet50 -f

echo "========== ResNet101V2 distribute =========="
CUDA_VISIBLE_DEVICES='1' python tensorflow_distribute.py -b 32 -s 2 -e 6 -m ResNet101V2
CUDA_VISIBLE_DEVICES='1' python tensorflow_distribute.py -b 64 -s 2 -e 6 -m ResNet101V2 -f

CUDA_VISIBLE_DEVICES='0,1' python tensorflow_distribute.py -b 32 -s 2 -e 6 -m ResNet101V2
CUDA_VISIBLE_DEVICES='0,1' python tensorflow_distribute.py -b 32 -s 2 -e 6 -m ResNet101V2 -f
CUDA_VISIBLE_DEVICES='0,1' python tensorflow_distribute.py -b 64 -s 2 -e 6 -m ResNet101V2 -f

echo "========== ResNet101V2 horovod =========="
CUDA_VISIBLE_DEVICES='1' horovodrun -np 1 -H localhost:1 python tensorflow_horovod.py -b 32 -e 6 -m ResNet101V2
CUDA_VISIBLE_DEVICES='1' horovodrun -np 1 -H localhost:1 python tensorflow_horovod.py -b 64 -e 6 -m ResNet101V2 -f

CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow_horovod.py -b 32 -e 6 -m ResNet101V2
CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow_horovod.py -b 32 -e 6 -m ResNet101V2 -f
CUDA_VISIBLE_DEVICES='0,1' horovodrun -np 2 -H localhost:2 python tensorflow_horovod.py -b 64 -e 6 -m ResNet101V2 -f

strategy	batch size	mean time	GPU memory
MobileNet
OneDeviceStrategy	32	126 ms/step	9085 MiB
OneDeviceStrategy	64	228 ms/step	9085 MiB
MirroredStrategy	32 * 2	143 ms/step	9085 MiB * 2
MirroredStrategy	64 * 2	250 ms/step	9085 MiB * 2
MirroredStrategy, float16	64 * 2	111 ms/step	4989 MiB * 2
MirroredStrategy, float16	128 * 2	186 ms/step	9085 MiB * 2
horovod, cuda 1	32	144 ms/step	9109 MiB
horovod, cuda 1	64	294 ms/step	9109 MiB
horovod, cuda 0,1	32 * 2	161 ms/step	9109 MiB * 2
horovod, cuda 0,1	64 * 2	307 ms/step	9109 MiB * 2
horovod, cuda 0,1, float16	64 * 2	155 ms/step	5013 MiB * 2
horovod, cuda 0,1, float16	128 * 2	279 ms/step	9109 MiB * 2
ResNet50
OneDeviceStrategy, float16	64	212 ms/step	9085 MiB
MirroredStrategy, float16	64 * 2	250 ms/step	9085 MiB * 2
horovod, cuda 1, float16	64	257 ms/step	9109 MiB
horovod, cuda 0,1, float16	64 * 2	260 ms/step	9109 MiB * 2
ResNet101V2
OneDeviceStrategy	32	283.4 ms/step	9083 MiB
OneDeviceStrategy, float16	64	244.6 ms/step	9083 MiB
MirroredStrategy	32 * 2	365.8 ms/step	9085 MiB * 2
MirroredStrategy, float16	32 * 2	249.0 ms/step	4989 MiB * 2
MirroredStrategy, float16	64 * 2	352.8 ms/step	9083 MiB * 2
horovod, cuda 1	32	312.8 ms/step	9109 MiB
horovod, cuda 1, float16	64	309.8 ms/step	9109 MiB
horovod, cuda 0,1	32 * 2	334.8 ms/step	9109 MiB * 2
horovod, cuda 0,1, float16	32 * 2	237.2 ms/step	5013 MiB * 2
horovod, cuda 0,1, float16	64 * 2	328.6 ms/step	9109 MiB * 2

[yunjiangster]

Have you tried more than 2 gpus? horovod with ring allreduce presumably works better when the ring is not a complete graph :)

[leondgarse]

Nop, I only have 2 GPUs. Besides, this is a test last year, may things have changed. Never looked back here since then. :)

[leondgarse]

OK, I've updated those results using currently latest horovod=0.23.0, but still with 2 GPUs only. Seems still no big changes.