I build two ceph clusters.
for the first cluster, I do the follow steps
1??create pools
sudo ceph osd pool create .us-east.rgw.root 64 64
sudo ceph osd pool create .us-east.rgw.control 64 64
sudo ceph osd pool create .us-east.rgw.gc 64 64
sudo ceph osd pool create .us-east.rgw.buckets 64 64
sudo ceph osd pool create .us-east.rgw.buckets.index 64 64
sudo ceph osd pool create .us-east.rgw.buckets.extra 64 64
sudo ceph osd pool create .us-east.log 64 64
sudo ceph osd pool create .us-east.intent-log 64 64
sudo ceph osd pool create .us-east.usage 64 64
sudo ceph osd pool create .us-east.users 64 64
sudo ceph osd pool create .us-east.users.email 64 64
sudo ceph osd pool create .us-east.users.swift 64 64
sudo ceph osd pool create .us-east.users.uid 64 64
2??create a keyring
sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.radosgw.keyring
sudo chmod +r /etc/ceph/ceph.client.radosgw.keyring
sudo ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n
client.radosgw.us-east-1 --gen-key
sudo ceph-authtool -n client.radosgw.us-east-1 --cap osd 'allow rwx' --cap mon
'allow rwx' /etc/ceph
sudo ceph -k /etc/ceph/ceph.client.admin.keyring auth add
client.radosgw.us-east-1 -i /etc/ceph/ceph.client.radosgw.keyring
3??create a region
sudo radosgw-admin region set --infile us.json --name client.radosgw.us-east-1
sudo radosgw-admin region default --rgw-region=us --name
client.radosgw.us-east-1
sudo radosgw-admin regionmap update --name client.radosgw.us-east-1
the content of us.json:
cat us.json
{ "name": "us",
"api_name": "us",
"is_master": "true",
"endpoints": [
"http:\/\/WH-CEPH-TEST01.MATRIX.CTRIPCORP.COM:80\/",
"http:\/\/WH-CEPH-TEST02.MATRIX.CTRIPCORP.COM:80\/"],
"master_zone": "us-east",
"zones": [
{ "name": "us-east",
"endpoints": [
"http:\/\/WH-CEPH-TEST01.MATRIX.CTRIPCORP.COM:80\/"],
"log_meta": "true",
"log_data": "true"},
{ "name": "us-west",
"endpoints": [
"http:\/\/WH-CEPH-TEST02.MATRIX.CTRIPCORP.COM:80\/"],
"log_meta": "true",
"log_data": "true"}],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement"}
4??create zones
sudo radosgw-admin zone set --rgw-zone=us-east --infile us-east-secert.json
--name client.radosgw.us-east-1
sudo radosgw-admin regionmap update --name client.radosgw.us-east-1
cat us-east-secert.json
{ "domain_root": ".us-east.domain.rgw",
"control_pool": ".us-east.rgw.control",
"gc_pool": ".us-east.rgw.gc",
"log_pool": ".us-east.log",
"intent_log_pool": ".us-east.intent-log",
"usage_log_pool": ".us-east.usage",
"user_keys_pool": ".us-east.users",
"user_email_pool": ".us-east.users.email",
"user_swift_pool": ".us-east.users.swift",
"user_uid_pool": ".us-east.users.uid",
"system_key": { "access_key": "XNK0ST8WXTMWZGN29NF9", "secret_key":
"7VJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"},
"placement_pools": [
{ "key": "default-placement",
"val": { "index_pool": ".us-east.rgw.buckets.index",
"data_pool": ".us-east.rgw.buckets"}
}
]
}
#5 Create Zone Users system user
sudo radosgw-admin user create --uid="us-east" --display-name="Region-US
Zone-East" --name client.radosgw.us-east-1 --access_key="XNK0ST8WXTMWZGN29NF9"
--secret="7VJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5" --system
sudo radosgw-admin user create --uid="us-west" --display-name="Region-US
Zone-West" --name client.radosgw.us-east-1 --access_key="AAK0ST8WXTMWZGN29NF9"
--secret="AAJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5" --system
#6 creat zone users not system user
sudo radosgw-admin user create --uid="us-test-east" --display-name="Region-US
Zone-East-test" --name client.radosgw.us-east-1
--access_key="DDK0ST8WXTMWZGN29NF9"
--secret="DDJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
#7 subuser create
sudo radosgw-admin subuser create --uid="us-test-east"
--subuser="us-test-east:swift" --access=full --name client.radosgw.us-east-1
--key-type swift --secret="ffJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
sudo /etc/init.d/ceph -a restart
sudo /etc/init.d/httpd re
sudo /etc/init.d/ceph-radosgw restart
for the second cluster, I do the follow steps
1??create pools
sudo ceph osd pool create .us-west.rgw.root 64 64
sudo ceph osd pool create .us-west.rgw.control 64 64
sudo ceph osd pool create .us-west.rgw.gc 64 64
sudo ceph osd pool create .us-west.rgw.buckets 64 64
sudo ceph osd pool create .us-west.rgw.buckets.index 64 64
sudo ceph osd pool create .us-west.rgw.buckets.extra 64 64
sudo ceph osd pool create .us-west.log 64 64
sudo ceph osd pool create .us-west.intent-log 64 64
sudo ceph osd pool create .us-west.usage 64 64
sudo ceph osd pool create .us-west.users 64 64
sudo ceph osd pool create .us-west.users.email 64 64
sudo ceph osd pool create .us-west.users.swift 64 64
sudo ceph osd pool create .us-west.users.uid 64 64
2??create a keyring
sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.radosgw.keyring
sudo chmod +r /etc/ceph/ceph.client.radosgw.keyring
sudo ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n
client.radosgw.us-west-1 --gen-key
sudo ceph -k /etc/ceph/ceph.client.admin.keyring auth del
client.radosgw.us-west-1
sudo ceph -k /etc/ceph/ceph.client.admin.keyring auth add
client.radosgw.us-west-1 -i /etc/ceph/ceph.client.radosgw.keyring
3?? create a region
sudo radosgw-admin region set --infile us.json --name client.radosgw.us-west-1
sudo radosgw-admin region default --rgw-region=us --name
client.radosgw.us-west-1
sudo radosgw-admin regionmap update --name client.radosgw.us-west-1
cat us.json
the content of us.json:
{ "name": "us",
"api_name": "us",
"is_master": "true",
"endpoints": [
"http:\/\/WH-CEPH-TEST01.MATRIX.CTRIPCORP.COM:80\/",
"http:\/\/WH-CEPH-TEST02.MATRIX.CTRIPCORP.COM:80\/"],
"master_zone": "us-east",
"zones": [
{ "name": "us-east",
"endpoints": [
"http:\/\/WH-CEPH-TEST01.MATRIX.CTRIPCORP.COM:80\/"],
"log_meta": "true",
"log_data": "true"},
{ "name": "us-west",
"endpoints": [
"http:\/\/WH-CEPH-TEST02.MATRIX.CTRIPCORP.COM:80\/"],
"log_meta": "true",
"log_data": "true"}],
"placement_targets": [
{
"name": "default-placement",
"tags": []
}
],
"default_placement": "default-placement"}
4??create zones
sudo radosgw-admin zone set --rgw-zone=us-west --infile us-west-secert.json
--name client.radosgw.us-west-1
sudo radosgw-admin regionmap update --name client.radosgw.us-west-1
the content of us-west-secert.json is:
cat us-west-secert.json
{ "domain_root": ".us-east.domain.rgw",
"control_pool": ".us-east.rgw.control",
"gc_pool": ".us-east.rgw.gc",
"log_pool": ".us-east.log",
"intent_log_pool": ".us-east.intent-log",
"usage_log_pool": ".us-east.usage",
"user_keys_pool": ".us-east.users",
"user_email_pool": ".us-east.users.email",
"user_swift_pool": ".us-east.users.swift",
"user_uid_pool": ".us-east.users.uid",
"system_key": { "access_key": "XNK0ST8WXTMWZGN29NF9", "secret_key":
"7VJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"},
"placement_pools": [
{ "key": "default-placement",
"val": { "index_pool": ".us-east.rgw.buckets.index",
"data_pool": ".us-east.rgw.buckets"}
}
]
}
5??Create Zone Users system user
sudo radosgw-admin user create --uid="us-east" --display-name="Region-US
Zone-East" --name client.radosgw.us-west-1 --access_key="XNK0ST8WXTMWZGN29NF9"
--secret="7VJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5" --system
sudo radosgw-admin user create --uid="us-west" --display-name="Region-US
Zone-West" --name client.radosgw.us-west-1 --access_key="AAK0ST8WXTMWZGN29NF9"
--secret="AAJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5" --system
6??reboot
sudo /etc/init.d/ceph -a restart
sudo /etc/init.d/httpd restart
sudo /etc/init.d/ceph-radosgw restart
after all of above, on the first cluster, i do the follow steps
1??source self.env
the content of self.env is :
cat self.env
export ST_AUTH="http://10.18.5.49/auth/1.0";
export ST_KEY=ffJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5
export ST_USER=us-test-east:swift
2??swift list
3??swift upload test self.env
4??swift list test
self.env
3??sudo radosgw-agent -c ./ceph-data-sync.conf
the content of ceph-data-sync.conf is??
cat ceph-data-sync.conf
src_access_key: XNK0ST8WXTMWZGN29NF9
src_secret_key: 7VJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5
destination: http://WH-CEPH-TEST02.MATRIX.CTRIPCORP.COM
dest_access_key: XNK0ST8WXTMWZGN29NF9
dest_secret_key: 7VJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5
log_file: /var/log/radosgw/radosgw-sync-us-east-west.log
there is some error as bellow??
sudo radosgw-agent -c ./ceph-data-sync.conf
region map is: {u'us': [u'us-west', u'us-east']}
INFO:radosgw_agent.sync:Starting sync
INFO:radosgw_agent.worker:24062 is processing shard number 0
INFO:radosgw_agent.worker:finished processing shard 0
INFO:radosgw_agent.worker:24062 is processing shard number 1
INFO:radosgw_agent.sync:1/64 items processed
INFO:radosgw_agent.worker:finished processing shard 1
INFO:radosgw_agent.sync:2/64 items processed
INFO:radosgw_agent.worker:24062 is processing shard number 2
INFO:radosgw_agent.worker:finished processing shard 2
INFO:radosgw_agent.sync:3/64 items processed
INFO:radosgw_agent.worker:24062 is processing shard number 3
INFO:radosgw_agent.worker:finished processing shard 3
INFO:radosgw_agent.sync:4/64 items processed
INFO:radosgw_agent.worker:24062 is processing shard number 4
...
...
...
INFO:radosgw_agent.worker:syncing bucket "test"
ERROR:radosgw_agent.worker:failed to sync object test/self.env: state is error
INFO:radosgw_agent.worker:syncing bucket "test"
ERROR:radosgw_agent.worker:failed to sync object test/self.env: state is error
INFO:radosgw_agent.worker:finished processing shard 69
on the second cluster ??i do the follow steps??
1??source self.env
the content of self.env is :
cat self.env
export ST_AUTH="http://10.18.5.51/auth/1.0";
export ST_KEY=ffJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5
export ST_USER=us-test-east:swift
2??swift list
Auth GET failed: http://10.18.5.51/auth/1.0 403 Forbidden
3??radosgw-admin --name client.radosgw.us-west-1 user info
--uid="us-test-east"
{ "user_id": "us-test-east",
"display_name": "Region-US Zone-East-test",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"auid": 0,
"subusers": [
{ "id": "us-test-east:swift",
"permissions": "full-control"}],
"keys": [
{ "user": "us-test-east",
"access_key": "DDK0ST8WXTMWZGN29NF9",
"secret_key": "DDJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"}],
"swift_keys": [
{ "user": "us-test-east:swift",
"secret_key": "ffJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"}],
"caps": [],
"op_mask": "read, write, delete",
"default_placement": "",
"placement_tags": [],
"bucket_quota": { "enabled": false,
"max_size_kb": -1,
"max_objects": -1},
"user_quota": { "enabled": false,
"max_size_kb": -1,
"max_objects": -1},
"temp_url_keys": []}
4??radosgw-admin --name client.radosgw.us-west-1 bucket list
[
"test"]
5??radosgw-admin --name client.radosgw.us-west-1 --bucket=test bucket list
[]
it seems like that metadata is replicated from the first cluster?? data is
not??
I don't known why?
------------------ ???????? ------------------
??????: "Craig Lewis";<cle...@centraldesktop.com>;
????????: 2015??5??7??(??????) ????8:46
??????: "TERRY"<316828...@qq.com>;
????: "ceph-users"<ceph-users@lists.ceph.com>;
????: Re: [ceph-users] about rgw region sync
System users are the only ones that need to be created in both zones.
Non-system users (and their sub-users) should be created in the primary zone.
radosgw-agent will replicate them to the secondary zone. I didn't create
sub-users for my system users, but I don't think it matters.
I can read my objects from the primary and secondary zones using the same
non-system user's Access and Secret. Using the S3 API, I only had to change
the host name to use the DNS entries that point at the secondary cluster. eg
http://bucket1.us-east.myceph.com/object and
http://bucket1.us-west.myceph.com/object.
It's possible that adding the non-system users to the secondary zone causes
replication to fail.
I would verify that users, buckets, and objects are being replicated using
radosgw-admin.
`radosgw-admin --name $name bucket list`, `radosgw-admin --name $name user
info --uid=$username`, and `radosgw-admin --name $name --bucket=$bucket bucket
list`. That will let you determine if you have a replication or an access
problem.
On Wed, Apr 29, 2015 at 10:27 PM, TERRY <316828...@qq.com> wrote:
hi:
I am using the following script to setup my cluster.
I upgrade my radosgw-agent from version 1.2.0 to 1.2.2-1. (1.2.0 will
results a error!)
cat repeat.sh
#!/bin/bash
set -e
set -x
#1 create pools
sudo ./create_pools.sh
#2 create a keyring
sudo ceph-authtool --create-keyring /etc/ceph/ceph.client.radosgw.keyring
sudo chmod +r /etc/ceph/ceph.client.radosgw.keyring
sudo ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n
client.radosgw.us-east-1 --gen-key
sudo ceph-authtool /etc/ceph/ceph.client.radosgw.keyring -n
client.radosgw.us-west-1 --gen-key
sudo ceph-authtool -n client.radosgw.us-east-1 --cap osd 'allow rwx' --cap mon
'allow rwx' /etc/ceph/ceph.client.radosgw.keyring
sudo ceph-authtool -n client.radosgw.us-west-1 --cap osd 'allow rwx' --cap mon
'allow rwx' /etc/ceph/ceph.client.radosgw.keyring
sudo ceph -k /etc/ceph/ceph.client.admin.keyring auth del
client.radosgw.us-east-1
sudo ceph -k /etc/ceph/ceph.client.admin.keyring auth del
client.radosgw.us-west-1
sudo ceph -k /etc/ceph/ceph.client.admin.keyring auth add
client.radosgw.us-east-1 -i /etc/ceph/ceph.client.radosgw.keyring
sudo ceph -k /etc/ceph/ceph.client.admin.keyring auth add
client.radosgw.us-west-1 -i /etc/ceph/ceph.client.radosgw.keyring
# 3 create a region
sudo radosgw-admin region set --infile us.json --name client.radosgw.us-east-1
set +e
sudo rados -p .us.rgw.root rm region_info.default
set -e
sudo radosgw-admin region default --rgw-region=us --name
client.radosgw.us-east-1
sudo radosgw-admin regionmap update --name client.radosgw.us-east-1
# try don't do it
sudo radosgw-admin region set --infile us.json --name client.radosgw.us-west-1
set +e
sudo rados -p .us.rgw.root rm region_info.default
set -e
sudo radosgw-admin region default --rgw-region=us --name
client.radosgw.us-west-1
sudo radosgw-admin regionmap update --name client.radosgw.us-west-1
# 4 create zones
# try chanege us-east-no-secert.json file contents
sudo radosgw-admin zone set --rgw-zone=us-east --infile us-east-no-secert.json
--name client.radosgw.us-east-1
sudo radosgw-admin zone set --rgw-zone=us-east --infile us-east-no-secert.json
--name client.radosgw.us-west-1
sudo radosgw-admin zone set --rgw-zone=us-west --infile us-west-no-secert.json
--name client.radosgw.us-east-1
sudo radosgw-admin zone set --rgw-zone=us-west --infile us-west-no-secert.json
--name client.radosgw.us-west-1
set +e
sudo rados -p .rgw.root rm zone_info.default
set -e
sudo radosgw-admin regionmap update --name client.radosgw.us-east-1
# try don't do it
sudo radosgw-admin regionmap update --name client.radosgw.us-west-1
#5 Create Zone Users system user
sudo radosgw-admin user create --uid="us-east" --display-name="Region-US
Zone-East" --name client.radosgw.us-east-1 --access_key="XNK0ST8WXTMWZGN29NF9"
--secret="7VJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5" --system
sudo radosgw-admin user create --uid="us-west" --display-name="Region-US
Zone-West" --name client.radosgw.us-west-1 --access_key="AAK0ST8WXTMWZGN29NF9"
--secret="AAJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5" --system
sudo radosgw-admin user create --uid="us-east" --display-name="Region-US
Zone-East" --name client.radosgw.us-west-1 --access_key="XNK0ST8WXTMWZGN29NF9"
--secret="7VJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5" --system
sudo radosgw-admin user create --uid="us-west" --display-name="Region-US
Zone-West" --name client.radosgw.us-east-1 --access_key="AAK0ST8WXTMWZGN29NF9"
--secret="AAJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5" --system
#6 subuser create
#may create a user without --system?
sudo radosgw-admin subuser create --uid="us-east" --subuser="us-east:swift"
--access=full --name client.radosgw.us-east-1 --key-type swift
--secret="7VJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
sudo radosgw-admin subuser create --uid="us-west" --subuser="us-west:swift"
--access=full --name client.radosgw.us-west-1 --key-type swift
--secret="BBJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
sudo radosgw-admin subuser create --uid="us-east" --subuser="us-east:swift"
--access=full --name client.radosgw.us-west-1 --key-type swift
--secret="7VJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
sudo radosgw-admin subuser create --uid="us-west" --subuser="us-west:swift"
--access=full --name client.radosgw.us-east-1 --key-type swift
--secret="BBJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
#5.5 creat zone users not system user
sudo radosgw-admin user create --uid="us-test-east" --display-name="Region-US
Zone-East-test" --name client.radosgw.us-east-1
--access_key="DDK0ST8WXTMWZGN29NF9"
--secret="DDJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
sudo radosgw-admin user create --uid="us-test-west" --display-name="Region-US
Zone-West-test" --name client.radosgw.us-west-1
--access_key="CCK0ST8WXTMWZGN29NF9"
--secret="CCJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
sudo radosgw-admin user create --uid="us-test-east" --display-name="Region-US
Zone-East-test" --name client.radosgw.us-west-1
--access_key="DDK0ST8WXTMWZGN29NF9"
--secret="DDJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
sudo radosgw-admin user create --uid="us-test-west" --display-name="Region-US
Zone-West-test" --name client.radosgw.us-east-1
--access_key="CCK0ST8WXTMWZGN29NF9"
--secret="CCJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
#6 subuser create
#may create a user without --system?
sudo radosgw-admin subuser create --uid="us-test-east"
--subuser="us-test-east:swift" --access=full --name client.radosgw.us-east-1
--key-type swift --secret="ffJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
sudo radosgw-admin subuser create --uid="us-test-west"
--subuser="us-test-west:swift" --access=full --name client.radosgw.us-west-1
--key-type swift --secret="ggJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
sudo radosgw-admin subuser create --uid="us-test-east"
--subuser="us-test-east:swift" --access=full --name client.radosgw.us-west-1
--key-type swift --secret="ffJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
sudo radosgw-admin subuser create --uid="us-test-west"
--subuser="us-test-west:swift" --access=full --name client.radosgw.us-east-1
--key-type swift --secret="ggJm8uAp71xKQZkjoPZmHu4sACA1SY8jTjay9dP5"
====================
after all of those:
1)??I upload an object to an container on master zone using the gateway
instance us-east-1
command like this:
swift upload mycontaier testobj
2)??then I examin the objext by the gateway instacne us-east-1 using the
command:
swift list
it could list the object
3)??I examin the object by the gateway instacne us-west-1 using the command:
swift list
it could not list the object. there is no error!
4)??I sync the data use the command:
sudo radosgw-agent -c ./ceph-data-sync.conf
5)??I examin the object by the gateway instacne us-west-1 again using the
command:
swift list
there is an error??it said??
Auth GET failed: http://10.18.5.209/auth/1.0 403 Forbidden
my quesiton is??how cloud I access the object from the secondary zone??
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
