Hello All, I installed ceph luminous with openstack, an using fio in a virtual machine I got slow random writes:
fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=test --filename=random_read_write.fio --bs=4k --iodepth=64 --size=4G --readwrite=randrw --rwmixread=75 Run status group 0 (all jobs): READ: bw=52.7MiB/s (55.3MB/s), 52.7MiB/s-52.7MiB/s (55.3MB/s-55.3MB/s), io=3070MiB (3219MB), run=58211-58211msec WRITE: bw=17.6MiB/s (18.5MB/s), 17.6MiB/s-17.6MiB/s (18.5MB/s-18.5MB/s), io=1026MiB (1076MB), run=58211-58211msec [root@tst2-osctrl01 ansible]# ceph osd tree ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF -1 108.26358 root default -3 36.08786 host p2-ceph-01 0 hdd 3.62279 osd.0 up 1.00000 1.00000 1 hdd 3.60529 osd.1 up 1.00000 1.00000 2 hdd 3.60529 osd.2 up 1.00000 1.00000 3 hdd 3.60529 osd.3 up 1.00000 1.00000 4 hdd 3.60529 osd.4 up 1.00000 1.00000 5 hdd 3.62279 osd.5 up 1.00000 1.00000 6 hdd 3.60529 osd.6 up 1.00000 1.00000 7 hdd 3.60529 osd.7 up 1.00000 1.00000 8 hdd 3.60529 osd.8 up 1.00000 1.00000 9 hdd 3.60529 osd.9 up 1.00000 1.00000 -5 36.08786 host p2-ceph-02 10 hdd 3.62279 osd.10 up 1.00000 1.00000 11 hdd 3.60529 osd.11 up 1.00000 1.00000 12 hdd 3.60529 osd.12 up 1.00000 1.00000 13 hdd 3.60529 osd.13 up 1.00000 1.00000 14 hdd 3.60529 osd.14 up 1.00000 1.00000 15 hdd 3.62279 osd.15 up 1.00000 1.00000 16 hdd 3.60529 osd.16 up 1.00000 1.00000 17 hdd 3.60529 osd.17 up 1.00000 1.00000 18 hdd 3.60529 osd.18 up 1.00000 1.00000 19 hdd 3.60529 osd.19 up 1.00000 1.00000 -7 36.08786 host p2-ceph-03 20 hdd 3.62279 osd.20 up 1.00000 1.00000 21 hdd 3.60529 osd.21 up 1.00000 1.00000 22 hdd 3.60529 osd.22 up 1.00000 1.00000 23 hdd 3.60529 osd.23 up 1.00000 1.00000 24 hdd 3.60529 osd.24 up 1.00000 1.00000 25 hdd 3.62279 osd.25 up 1.00000 1.00000 26 hdd 3.60529 osd.26 up 1.00000 1.00000 27 hdd 3.60529 osd.27 up 1.00000 1.00000 28 hdd 3.60529 osd.28 up 1.00000 1.00000 29 hdd 3.60529 osd.29 up 1.00000 1.00000 Each osd server has 10 4TB osd and two ssd (2 x 2TB). Each ssd is partitioned with 5 partitions (ehach partiotion is 384 GB) for bluestore db and wal. Each osd and mon host have two 10GB nics for public and cluster ceph network. Osd servers are Power Edge R7425 with 256 GB RAM and MegaRAID SAS-3 3108. No nvme disks are present. Ceph.conf is the following: [global] fsid = 9a33214b-86df-4ef0-9199-5f7637cff1cd public_network = 10.102.189.128/25 cluster_network = 10.102.143.16/28 mon_initial_members = tst2-osctrl01, tst2-osctrl02, tst2-osctrl03 mon_host = 10.102.189.200,10.102.189.201,10.102.189.202 auth_cluster_required = cephx auth_service_required = cephx auth_client_required = cephx osd pool default size = 3 osd pool default min size = 2 mon_max_pg_per_osd = 1024 osd max pg per osd hard ratio = 20 [mon] mon compact on start = true [osd] bluestore cache autotune = 0 #bluestore cache kv ratio = 0.2 #bluestore cache meta ratio = 0.8 bluestore cache size ssd = 8G bluestore csum type = none bluestore extent map shard max size = 200 bluestore extent map shard min size = 50 bluestore extent map shard target size = 100 bluestore rocksdb options = compression=kNoCompression,max_write_buffer_number=32,min_write_buffer_number_to_merge=2,recycle_log_file_num=32,compaction_style=kCompactionStyleLevel,write_buffer_size=67108864,target_file_size_base=67108864,max_background_compactions=31,level0_file_num_compaction_trigger=8,level0_slowdown_writes_trigger=32,level0_stop_writes_trigger=64,max_bytes_for_level_base=536870912,compaction_threads=32,max_bytes_for_level_multiplier=8,flusher_threads=8,compaction_readahead_size=2MB osd map share max epochs = 100 osd max backfills = 5 osd memory target = 4294967296 osd op num shards = 8 osd op num threads per shard = 2 Any help, please ? Ignazio
_______________________________________________ ceph-users mailing list ceph-users@lists.ceph.com http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
