Hi All,
I’m having serieus problems setting up a new DRBD cluster. At the moment i have
2 clusters in production without any problems for around 2 years now.
Debian Squeeze 6 + drbd 8.3.11 running at a nice speed of 800MB/s with 16x SATA
1TB disks HW raid 6 :-) and 10G NIC’s
No partitions on the DRBD device it’s raw block device.
Bought some new hardware:
2x Xeon 6core CPU’s with HT - 16GB ram - Areca 1882i - 24x 600GB SAS 10k HDD’s
- 10G NIC's
Started of with Debian Wheezy and DRBD 8.3.13 but performance was really bad +-
400MB/s with DRBD primary/secondary while local raw speed of /dev/sdb = 1.5GB/s
Then went on with Ubuntu 13.10 and DRBD 8.4.3 same deal. Then installed with
the latest stable DRBD 8.4.4 but no difference.
I can cranck up the initial sync to 750MB/s when playing with the c-rate’s
(still not a good performance) while after the initial sync it drops back to
400MB/s
I tested all the different performance settings on everything! I mean local
block tuning, NIC tuning, DRBD tuning still i can’t get it to perform well.
It’s worse then my production DRBD with SATA drives on 7200rpm!
Current config:
root@vstore7:~# drbdsetup /dev/drbd0 show
disk {
size 0s _is_default; # bytes
on-io-error detach;
fencing dont-care _is_default;
no-disk-flushes ;
max-bio-bvecs 0 _is_default;
}
net {
timeout 60 _is_default; # 1/10 seconds
max-epoch-size 8000;
max-buffers 8000;
unplug-watermark 128 _is_default;
connect-int 10 _is_default; # seconds
ping-int 10 _is_default; # seconds
sndbuf-size 2097152; # bytes
rcvbuf-size 0 _is_default; # bytes
ko-count 0 _is_default;
after-sb-0pri disconnect _is_default;
after-sb-1pri disconnect _is_default;
after-sb-2pri disconnect _is_default;
rr-conflict disconnect _is_default;
ping-timeout 5 _is_default; # 1/10 seconds
on-congestion block _is_default;
congestion-fill 0s _is_default; # byte
congestion-extents 127 _is_default;
}
syncer {
rate 1228800k; # bytes/second
after -1 _is_default;
al-extents 3389;
on-no-data-accessible io-error _is_default;
c-plan-ahead 0 _is_default; # 1/10 seconds
c-delay-target 10 _is_default; # 1/10 seconds
c-fill-target 0s _is_default; # bytes
c-max-rate 102400k _is_default; # bytes/second
c-min-rate 4096k _is_default; # bytes/second
}
protocol C;
_this_host {
device minor 0;
disk "/dev/sdb";
meta-disk internal;
address ipv4 172.16.252.7:7789;
}
_remote_host {
address ipv4 172.16.252.8:7789;
}
Here are some test results of iperf and dd so you see my local speeds and
network speeds aren’t the problem. The final setup will also be a bond of 10G
cards but i took that out for troubleshooting.
root@vstore7:~# iperf -c 172.16.252.8 -t 200 -i 2
------------------------------------------------------------
Client connecting to 172.16.252.8, TCP port 5001
TCP window size: 96.7 KByte (default)
------------------------------------------------------------
[ 3] local 172.16.252.7 port 35149 connected with 172.16.252.8 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0- 2.0 sec 2.32 GBytes 9.96 Gbits/sec
[ 3] 2.0- 4.0 sec 2.31 GBytes 9.93 Gbits/sec
[ 3] 4.0- 6.0 sec 2.31 GBytes 9.93 Gbits/sec
----total-cpu-usage---- -dsk/total- -net/total- ---paging-- ---system--
usr sys idl wai hiq siq| read writ| recv send| in out | int csw
0 0 99 0 0 0| 96k 246k| 0 0 | 0 0 |9227 186
0 3 96 0 0 1| 0 0 |3692k 1190M| 0 0 | 93k 252
0 3 96 0 0 1| 0 0 |3695k 1189M| 0 0 | 94k 252
0 3 96 0 0 1| 0 0 |3697k 1189M| 0 0 | 94k 268
root@vstore7:~# dd if=/dev/zero of=/dev/sdb bs=1M count=100000
100000+0 records in
100000+0 records out
104857600000 bytes (105 GB) copied, 98.921 s, 1.1 GB/s
root@vstore8:~# dd if=/dev/zero of=/dev/sdb bs=1M count=100000
100000+0 records in
100000+0 records out
104857600000 bytes (105 GB) copied, 98.3511 s, 1.1 GB/s
root@vstore7:~# dd if=/dev/zero of=/dev/sdb bs=1M count=100000 oflag=direct
100000+0 records in
100000+0 records out
104857600000 bytes (105 GB) copied, 59.5348 s, 1.8 GB/s
root@vstore8:~# dd if=/dev/zero of=/dev/sdb bs=1M count=100000 oflag=direct
100000+0 records in
100000+0 records out
104857600000 bytes (105 GB) copied, 59.3226 s, 1.8 GB/s{\rtf1\ansi\ansicpg1252\cocoartf1265
{\fonttbl\f0\fswiss\fcharset0 Helvetica;}
{\colortbl;\red255\green255\blue255;}
\paperw11900\paperh16840\margl1440\margr1440\vieww28600\viewh17040\viewkind0
\pard\tx566\tx1133\tx1700\tx2267\tx2834\tx3401\tx3968\tx4535\tx5102\tx5669\tx6236\tx6803\pardirnatural
\f0\fs24 \cf0 root@vstore7:~# fio --bs=4m --direct=1 --rw=read --ioengine=libaio --iodepth=64 --runtime=60 --name=/dev/sdb\
/dev/sdb: (g=0): rw=read, bs=4M-4M/4M-4M, ioengine=libaio, iodepth=64\
2.0.8\
Starting 1 process\
Jobs: 1 (f=1): [R] [100.0% done] [2112M/0K /s] [528 /0 iops] [eta 00m:00s]\
/dev/sdb: (groupid=0, jobs=1): err= 0: pid=2072\
read : io=126312MB, bw=2103.1MB/s, iops=525 , runt= 60035msec\
slat (usec): min=217 , max=25748 , avg=1897.18, stdev=2908.85\
clat (msec): min=30 , max=151 , avg=119.65, stdev= 3.93\
lat (msec): min=37 , max=151 , avg=121.55, stdev= 2.67\
clat percentiles (msec):\
| 1.00th=[ 114], 5.00th=[ 115], 10.00th=[ 115], 20.00th=[ 116],\
| 30.00th=[ 121], 40.00th=[ 122], 50.00th=[ 122], 60.00th=[ 122],\
| 70.00th=[ 122], 80.00th=[ 122], 90.00th=[ 123], 95.00th=[ 123],\
| 99.00th=[ 126], 99.50th=[ 130], 99.90th=[ 137], 99.95th=[ 139],\
| 99.99th=[ 145]\
bw (MB/s) : min= 1754, max= 2162, per=99.95%, avg=2153.50, stdev=37.74\
lat (msec) : 50=0.03%, 100=0.26%, 250=99.71%\
cpu : usr=0.12%, sys=12.68%, ctx=8029, majf=0, minf=65560\
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8%\
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%\
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%\
issued : total=r=31578/w=0/d=0, short=r=0/w=0/d=0\
\
Run status group 0 (all jobs):\
READ: io=126312MB, aggrb=2103.1MB/s, minb=2103.1MB/s, maxb=2103.1MB/s, mint=60035msec, maxt=60035msec\
\
Disk stats (read/write):\
sdb: ios=252228/0, merge=0/0, ticks=8466596/0, in_queue=8468932, util=99.91%\
root@vstore7:~# fio --bs=4m --direct=1 --rw=write --ioengine=libaio --iodepth=64 --runtime=60 --name=/dev/sdb\
/dev/sdb: (g=0): rw=write, bs=4M-4M/4M-4M, ioengine=libaio, iodepth=64\
2.0.8\
Starting 1 process\
Jobs: 1 (f=1): [W] [100.0% done] [0K/1866M /s] [0 /466 iops] [eta 00m:00s]\
/dev/sdb: (groupid=0, jobs=1): err= 0: pid=2077\
write: io=112032MB, bw=1866.5MB/s, iops=466 , runt= 60037msec\
slat (usec): min=323 , max=21417 , avg=2138.85, stdev=2817.99\
clat (msec): min=34 , max=170 , avg=134.99, stdev= 4.36\
lat (msec): min=40 , max=170 , avg=137.13, stdev= 3.41\
clat percentiles (msec):\
| 1.00th=[ 130], 5.00th=[ 130], 10.00th=[ 131], 20.00th=[ 131],\
| 30.00th=[ 137], 40.00th=[ 137], 50.00th=[ 137], 60.00th=[ 137],\
| 70.00th=[ 137], 80.00th=[ 139], 90.00th=[ 139], 95.00th=[ 139],\
| 99.00th=[ 139], 99.50th=[ 139], 99.90th=[ 151], 99.95th=[ 153],\
| 99.99th=[ 163]\
bw (MB/s) : min= 1705, max= 1925, per=99.92%, avg=1909.31, stdev=22.20\
lat (msec) : 50=0.02%, 100=0.19%, 250=99.79%\
cpu : usr=16.77%, sys=8.56%, ctx=7192, majf=0, minf=23\
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8%\
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%\
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%\
issued : total=r=0/w=28008/d=0, short=r=0/w=0/d=0\
\
Run status group 0 (all jobs):\
WRITE: io=112032MB, aggrb=1866.5MB/s, minb=1866.5MB/s, maxb=1866.5MB/s, mint=60037msec, maxt=60037msec\
\
Disk stats (read/write):\
sdb: ios=91/224032, merge=0/0, ticks=132/8376124, in_queue=8376948, util=99.85%\
\
WITH DRBD BLOCK:\
\
\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural
\cf0 root@vstore7:~# fio --bs=4m --direct=1 --rw=read --ioengine=libaio --iodepth=64 --runtime=60 --name=/dev/drbd0\
/dev/drbd0: (g=0): rw=read, bs=4M-4M/4M-4M, ioengine=libaio, iodepth=64\
2.0.8\
Starting 1 process\
Jobs: 1 (f=1): [R] [100.0% done] [2096M/0K /s] [524 /0 iops] [eta 00m:00s]\
/dev/drbd0: (groupid=0, jobs=1): err= 0: pid=2083\
read : io=126364MB, bw=2104.8MB/s, iops=526 , runt= 60037msec\
slat (usec): min=264 , max=16907 , avg=1896.20, stdev=2822.14\
clat (msec): min=31 , max=152 , avg=119.61, stdev= 3.72\
lat (msec): min=38 , max=152 , avg=121.50, stdev= 2.46\
clat percentiles (msec):\
| 1.00th=[ 114], 5.00th=[ 115], 10.00th=[ 115], 20.00th=[ 116],\
| 30.00th=[ 121], 40.00th=[ 121], 50.00th=[ 122], 60.00th=[ 122],\
| 70.00th=[ 122], 80.00th=[ 122], 90.00th=[ 123], 95.00th=[ 123],\
| 99.00th=[ 125], 99.50th=[ 127], 99.90th=[ 130], 99.95th=[ 133],\
| 99.99th=[ 141]\
bw (MB/s) : min= 1745, max= 2170, per=99.96%, avg=2154.33, stdev=38.07\
lat (msec) : 50=0.03%, 100=0.26%, 250=99.71%\
cpu : usr=0.18%, sys=15.22%, ctx=8031, majf=0, minf=65560\
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.8%\
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%\
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%\
issued : total=r=31591/w=0/d=0, short=r=0/w=0/d=0\
\
Run status group 0 (all jobs):\
READ: io=126364MB, aggrb=2104.8MB/s, minb=2104.8MB/s, maxb=2104.8MB/s, mint=60037msec, maxt=60037msec\
\
Disk stats (read/write):\
drbd0: ios=252436/0, merge=0/0, ticks=8493664/0, in_queue=8496044, util=99.90%\
root@vstore7:~# fio --bs=4m --direct=1 --rw=write --ioengine=libaio --iodepth=64 --runtime=60 --name=/dev/drbd0\
/dev/drbd0: (g=0): rw=write, bs=4M-4M/4M-4M, ioengine=libaio, iodepth=64\
2.0.8\
Starting 1 process\
Jobs: 1 (f=1): [W] [100.0% done] [0K/1176M /s] [0 /294 iops] [eta 00m:00s]\
/dev/drbd0: (groupid=0, jobs=1): err= 0: pid=2087\
write: io=46368MB, bw=788509KB/s, iops=192 , runt= 60216msec\
slat (usec): min=293 , max=8397 , avg=468.14, stdev=133.57\
clat (msec): min=58 , max=624 , avg=331.84, stdev=160.70\
lat (msec): min=58 , max=625 , avg=332.31, stdev=160.69\
clat percentiles (msec):\
| 1.00th=[ 217], 5.00th=[ 217], 10.00th=[ 217], 20.00th=[ 217],\
| 30.00th=[ 217], 40.00th=[ 219], 50.00th=[ 219], 60.00th=[ 237],\
| 70.00th=[ 420], 80.00th=[ 502], 90.00th=[ 627], 95.00th=[ 627],\
| 99.00th=[ 627], 99.50th=[ 627], 99.90th=[ 627], 99.95th=[ 627],\
| 99.99th=[ 627]\
bw (KB/s) : min=420102, max=1209996, per=100.00%, avg=789233.34, stdev=349225.53\
lat (msec) : 100=0.12%, 250=60.56%, 500=19.32%, 750=20.00%\
cpu : usr=5.43%, sys=3.97%, ctx=11655, majf=0, minf=23\
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.3%, >=64=99.5%\
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%\
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%\
issued : total=r=0/w=11592/d=0, short=r=0/w=0/d=0\
\
Run status group 0 (all jobs):\
WRITE: io=46368MB, aggrb=788508KB/s, minb=788508KB/s, maxb=788508KB/s, mint=60216msec, maxt=60216msec\
\
Disk stats (read/write):\
drbd0: ios=90/92736, merge=0/0, ticks=108/30413668, in_queue=30467752, util=99.78%\
}
PLEASE SOMEONE TELL ME WHAT I MISS!
Tested DRBD settings:
On DISKS:
block/sdb/queue/scheduler = deadline
block/sdb/queue/iosched/front_merges = 0
block/sdb/queue/iosched/read_expire = 150
block/sdb/queue/iosched/write_expire = 1500
On NIC:
net.core.rmem_max = 33554432
net.core.wmem_max = 33554432
net.ipv4.tcp_rmem = 4096 87380 33554432
net.ipv4.tcp_wmem = 4096 65536 33554432
net.core.netdev_max_backlog = 30000
net.ipv4.tcp_window_scaling = 1
on DRBD:
global {
usage-count no;
# minor-count dialog-refresh disable-ip-verification
}
common {
handlers {
pri-on-incon-degr "echo o > /proc/sysrq-trigger ; halt -f";
pri-lost-after-sb "echo o > /proc/sysrq-trigger ; halt -f";
local-io-error "echo o > /proc/sysrq-trigger ; halt -f";
}
startup {
wfc-timeout 240;
degr-wfc-timeout 120;
outdated-wfc-timeout 120;
}
disk {
on-io-error detach;
disk-barrier no;
disk-flushes no;
md-flushes no;
c-plan-ahead 20; tested different ranges 100 and 200 as well
c-fill-target 2M; tested different settings
c-min-rate 1M; tested different settings
c-max-rate 1500M; tested different settings
resync-rate 1500M; doesn’t do anything when c-rate disabled
al-extents 3389;
}
net {
protocol C;
sndbuf-size 2M; tested different settings
# rcvbuf-size 2M; tested different settings
# ping-int 4;
# timeout 30;
max-buffers 8000; tested between and 131072
max-epoch-size 8000;
# unplug-watermark 8000; tested from 16 and higher
use-rle;
# verify-alg md5;
}
}
Mvg,
Joeri
--
Joeri Casteels
Department of Information Technology
Internet Based Communication Networks and Services (IBCN)
Ghent University - iMinds
Gaston Crommenlaan 8 (Bus 201), B-9050 Gent, Belgium
T: +32 9 33 14964
T Secretariaat: +32 9 33 14900
F: +32 9 33 14899
E: [email protected]
W : www.ibcn.intec.UGent.be
_______________________________________________
drbd-user mailing list
[email protected]
http://lists.linbit.com/mailman/listinfo/drbd-user
