Hi, Gregory!
It turns out that this error is internal CEPH feature. I wrote standalone
program to create 132M object in striper mode. It works only for 4M
stripe. If you set stripe_unit = 2M it still creates 4M stripe_unit.
Anything bigger than 4M causes crash here
<https://github.com/ceph/ceph/blob/master/src/osdc/Striper.cc#L64\>:
__u32 object_size = layout->object_size;
__u32 su = layout->stripe_unit;
__u32 stripe_count = layout->stripe_count;
assert(object_size >= su); <------------
I'm curious where it gets layout->object_size for object that is just been
created.
As I understod striper mode was created by CERN guys. In there document
<https://indico.cern.ch/event/542464/contributions/2202295/attachments/1289543/1919853/HEPUsageOfCeph.pdf>
they recommend 8M stripe_unit. But it does not work in luminous.
Created I/O context.
Connected to pool backup with rados_striper_create
Stripe unit OK 8388608
Stripe count OK 1
/build/ceph-12.2.0/src/osdc/Striper.cc: In function 'static void
Striper::file_to_extents(CephContext*, const char*, const file_layout_t*,
uint64_t, uint64_t, uint64_t, std::map<object_t, std::vector<ObjectExtent>
>&, uint64_t)' thread 7f13bd5c1e00 time 2017-10-07 21:44:58.654778
/build/ceph-12.2.0/src/osdc/Striper.cc: 64: FAILED assert(object_size >= su)
ceph version 12.2.0 (32ce2a3ae5239ee33d6150705cdb24d43bab910c) luminous
(rc)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char
const*)+0x102) [0x7f13b3f3b332]
2: (Striper::file_to_extents(CephContext*, char const*, file_layout_t
const*, unsigned long, unsigned long, unsigned long, std::map<object_t,
std::vector<ObjectExtent, std::allocator<ObjectExtent> >,
std::less<object_t>, std::allocator<std::pair<object_t const,
std::vector<ObjectExtent, std::allocator<ObjectExtent> > > > >&, unsigned
long)+0x1e1e) [0x7f13bce235ee]
3: (Striper::file_to_extents(CephContext*, char const*, file_layout_t
const*, unsigned long, unsigned long, unsigned long,
std::vector<ObjectExtent, std::allocator<ObjectExtent> >&, unsigned
long)+0x51) [0x7f13bce23691]
4:
(libradosstriper::RadosStriperImpl::internal_aio_write(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&,
boost::intrusive_ptr<libradosstriper::MultiAioCompletionImpl>,
ceph::buffer::list const&, unsigned long, unsigned long, ceph_file_layout
const&)+0x224) [0x7f13bcda4184]
5:
(libradosstriper::RadosStriperImpl::write_in_open_object(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, ceph_file_layout
const&, std::__cxx11::basic_string<char, std::char_traits<char>,
std::allocator<char> > const&, ceph::buffer::list const&, unsigned long,
unsigned long)+0x13c) [0x7f13bcda476c]
6:
(libradosstriper::RadosStriperImpl::write(std::__cxx11::basic_string<char,
std::char_traits<char>, std::allocator<char> > const&, ceph::buffer::list
const&, unsigned long, unsigned long)+0xd5) [0x7f13bcda4bd5]
7: (rados_striper_write()+0xdb) [0x7f13bcd9ba0b]
8: (()+0x10fb) [0x55dd87b410fb]
9: (__libc_start_main()+0xf1) [0x7f13bc9d72b1]
10: (()+0xbca) [0x55dd87b40bca]
On Fri, Sep 29, 2017 at 11:46 PM, Gregory Farnum <gfar...@redhat.com> wrote:
> I haven't used the striper, but it appears to make you specify sizes,
> stripe units, and stripe counts. I would expect you need to make sure that
> the size is an integer multiple of the stripe unit. And it probably
> defaults to a 4MB object if you don't specify one?
>
> On Fri, Sep 29, 2017 at 2:09 AM Alexander Kushnirenko <
> kushnire...@gmail.com> wrote:
>
>> Hi,
>>
>> I'm trying to use CEPH-12.2.0 as storage for with Bareos-16.2.4 backup
>> with libradosstriper1 support.
>>
>> Libradosstriber was suggested on this list to solve the problem, that
>> current CEPH-12 discourages users from using object with very big size
>> (>128MB). Bareos treat Rados Object as Volume and in CEPH-10 it created
>> objects with very big size (10G and more). CEPH-10 allowed such behaviour,
>> put recovery indeed take very long time. So stripping objects seems to be
>> the right thing to do.
>>
>> Bareos supports libradosstriper and the code seems to work. But for some
>> reason it run only with stripe_unit=4194304, which seems to be typical
>> value for RadosGW for example. I tried several other values for
>> stripe_unit, but the code exit with error.
>>
>> Is there a particular reason why only 4M size works? Can one use some
>> CLI to test different stripe sizes?
>>
>> Basic flow of creating object in Bareos is the following:
>> rados_ioctx_create(m_cluster, m_rados_poolname, &m_ctx);
>> rados_striper_create(m_ctx, &m_striper);
>> rados_striper_set_object_layout_stripe_unit(m_striper, m_stripe_unit);
>> rados_striper_set_object_layout_stripe_count(m_striper, m_stripe_count);
>> .....
>> status = rados_striper_write(m_striper, m_virtual_filename, buffer,
>> count, offset);
>>
>> Alexander
>>
>> _______________________________________________
>> ceph-users mailing list
>> ceph-users@lists.ceph.com
>> http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
>>
>
#include <stdio.h>
#include <string.h>
#include <rados/librados.h>
#include <radosstriper/libradosstriper.h>
#include <stdlib.h>
#define EXIT_FAILURE 1
int main (int argc, const char * argv[])
{
/* Declare the cluster handle and required arguments. */
rados_t cluster;
char cluster_name[] = "ceph";
char user_name[] = "client.admin";
uint64_t flags;
rados_striper_t m_striper;
/* Initialize the cluster handle with the "ceph" cluster name and the "client.admin" user */
int err;
err = rados_create2(&cluster, cluster_name, user_name, flags);
if (err < 0) {
fprintf(stderr, "%s: Couldn't create the cluster handle! %s\n", argv[0], strerror(-err));
exit(EXIT_FAILURE);
} else {
printf("\nCreated a cluster handle.\n");
}
/* Read a Ceph configuration file to configure the cluster handle. */
err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
if (err < 0) {
fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
exit(EXIT_FAILURE);
} else {
printf("\nRead the config file.\n");
}
/* Read command line arguments */
err = rados_conf_parse_argv(cluster, argc, argv);
if (err < 0) {
fprintf(stderr, "%s: cannot parse command line arguments: %s\n", argv[0], strerror(-err));
exit(EXIT_FAILURE);
} else {
printf("\nRead the command line arguments.\n");
}
/* Connect to the cluster */
err = rados_connect(cluster);
if (err < 0) {
fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
exit(EXIT_FAILURE);
} else {
printf("\nConnected to the cluster.\n");
}
/*
* Continued from previous C example, where cluster handle and
* connection are established. First declare an I/O Context.
*/
rados_ioctx_t io;
char *poolname = "backup";
err = rados_ioctx_create(cluster, poolname, &io);
if (err < 0) {
fprintf(stderr, "%s: cannot open rados pool %s: %s\n", argv[0], poolname, strerror(-err));
rados_shutdown(cluster);
exit(EXIT_FAILURE);
} else {
printf("\nCreated I/O context.\n");
}
err = rados_striper_create(io, &m_striper);
if (err < 0) {
fprintf (stderr, "Unable to create RADOS striper object for pool %s: ERR=%s\n", poolname, strerror(-err));
rados_shutdown(cluster);
exit(EXIT_FAILURE);
} else {
fprintf (stderr, "Connected to pool %s with rados_striper_create \n", poolname);
}
//int stripe_unit = 2097152;
int stripe_unit = 4194304;
//int stripe_unit = 16777216;
//int stripe_unit = 8388608;
err = rados_striper_set_object_layout_stripe_unit(m_striper, stripe_unit);
if (err < 0) {
fprintf (stderr, "Unable to set RADOS striper unit size to %d for pool %s: ERR=%s\n", stripe_unit, poolname, err);
rados_shutdown(cluster);
exit(EXIT_FAILURE);
} else {
fprintf (stderr, "Stripe unit OK %d \n", stripe_unit);
}
int stripe_count = 1;
err = rados_striper_set_object_layout_stripe_count(m_striper, stripe_count);
if (err < 0) {
fprintf (stderr, "Unable to set RADOS striper stripe count to %d for pool %s: ERR=%s\n", stripe_count, poolname, err);
rados_shutdown(cluster);
exit(EXIT_FAILURE);
} else {
fprintf (stderr, "Stripe count OK %d \n", stripe_count);
}
/* Write data to the cluster synchronously. */
char read_res[1048576];
memset(read_res, 0xcc, 1048576);
for (int i=0; i<2048; i++) {
err = rados_striper_write(m_striper, "striper", read_res, 65536, i*65536);
if (err < 0) {
fprintf(stderr, "Cannot write object striper to pool %s:", strerror(-err));
rados_ioctx_destroy(io);
rados_shutdown(cluster);
exit(1);
} else {
//printf("\nWrote \"Hello World\" to object \"hw\".\n");
}
}
rados_ioctx_destroy(io);
rados_shutdown(cluster);
}
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
