Play with ceph-radosgw (by quqi99)

版权声明：本文为博主原创文章，如需转载，请注明出处！ https://blog.csdn.net/quqi99/article/details/80683730

版权声明：可以任意转载，转载时请务必以超链接形式标明文章原始出处和作者信息及本版权声明 (作者：张华 发表于：2018-06-13)

Rapidly install

https://github.com/openstack/charm-ceph-radosgw
juju deploy ceph-radosgw --series xenial
juju add-relation ceph-radosgw ceph
juju add-relation keystone ceph-radosgw
juju expose ceph-radosgw
curl http://juju-332891-mitaka-ceph-13

juju ssh ceph/0 'sudo radosgw-admin metadata list user'
sudo radosgw-admin user create --uid="admin" --display-name="admin"
sudo radosgw-admin usage show --uid=admin
sudo radosgw-admin usage show --show-log-entries=false
sudo radosgw-admin caps add --uid=admin --caps="users=*"
sudo radosgw-admin caps add --uid=admin --caps="usage=read"
sudo radosgw-admin bucket stats
sudo radosgw-admin bucket stats --uid=anonymous

Use s3cmd client to visit RGW

sudo pip install s3cmd
s3cmd --configure
check_ssl_certificate = False
check_ssl_hostname = False
access_key = EJECCFLCB0K53EMVM3DL
secret_key = C281KRmdFFWxzOHyjZ3OsvQorCgf1mxdML7kYJ1t
cloudfront_host = juju-332891-mitaka-ceph-13
host_base = juju-332891-mitaka-ceph-13:80
host_bucket = %(bucket)s.juju-332891-mitaka-ceph-13

s3cmd mb s3://my_test1
s3cmd ls  # qeury bucket
s3cmd put testfile s3://my_test1
s3cmd ls s3://my_test1
s3cmd get s3://my_test1/diff

Enable usage.log

http://www.yangguanjun.com/2016/08/30/rgw-user-statistics/
Add the following configurations into ceph-radosgw/0 node
[client.radosgw.gateway]
rgw enable usage log = true
rgw usage log tick interval = 30
rgw usage log flush threshold = 1024
rgw usage max shards = 32
rgw usage max user shards = 1

sudo ceph osd pool create .usage 64   # create a pool for usage, run in 'ceph/0'
sudo ceph osd pool create .rgw 128 128
sudo ceph osd pool create .rgw.root 128 128
sudo ceph osd pool create .rgw.control 128 128
sudo ceph osd pool create .rgw.gc 128 128
sudo ceph osd pool create .rgw.buckets 128 128
sudo ceph osd pool create .rgw.buckets.index 128 128
sudo ceph osd pool create .log 128 128
sudo ceph osd pool create .intent-log 128 128
sudo ceph osd pool create .usage 128 128
sudo ceph osd pool create .users 128 128
sudo ceph osd pool create .users.email 128 128
sudo ceph osd pool create .users.swift 128 128
sudo ceph osd pool create .users.uid 128 128

sudo /etc/init.d/radosgw restart
ceph daemon /var/run/ceph/ceph-client.radosgw.gateway.asok config show # verify config, run in radosgw node
sudo radosgw-admin usage show --show-log-entries=false  # after 1 hour, use radosgw-admin to see usage data, run in ceph/0

REST API

用REST API调用时遇到一个问题(radosgw-admin usage show无此问题), 就是看到anonymous用户有很多空bucket

$ sudo radosgw-admin usage show --uid=anonymous
user.init failed: (13) Permission denied

$./get_anonymous.py 
DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): juju-332891-mitaka-ceph-13
DEBUG:urllib3.connectionpool:http://juju-332891-mitaka-ceph-13:80 "GET /admin/usage?format=json&show-summary=False&uid=anonymous HTTP/1.1" 200 242
<Response [200]>
{
    "entries":[
        {
            "buckets":[
                {
                    "bucket":"",
                    "categories":[
                        {
                            "bytes_received":0,
                            "bytes_sent":940,
                            "category":"list_buckets",
                            "ops":4,
                            "successful_ops":4
                        }
                    ],
                    "epoch":1528275600,
                    "owner":"anonymous",
                    "time":"2018-06-06 09:00:00.000000Z"
                }
            ],
            "user":"anonymous"
        }
    ]
}

$ cat get_anonymous.py 
#!/usr/bin/python                                                                                                     
import requests
import json
import logging
import argparse
from awsauth import S3Auth

logging.basicConfig(level=logging.DEBUG)
server = 'juju-332891-mitaka-ceph-13'
aws_key = 'EJECCFLCB0K53EMVM3DL'
secret = 'C281KRmdFFWxzOHyjZ3OsvQorCgf1mxdML7kYJ1t'
url = 'http://%s/admin/usage?format=json&show-summary=False&uid=anonymous' % server
response = requests.get(url, auth=S3Auth(aws_key, secret, server))
print response
print json.dumps(response.json(), sort_keys=True, indent=4, separators=(',', ':'))

Set up ceph debug env in boinic

gdb can also debug python by using prefix py- in gdb comand - http://linux-debug.blogspot.com/2015/01/ceph-debugging-python-code-in-gdb.html

git clone https://github.com/ceph/ceph
git submodule update --force --init --recursive
git checkout master
./install-deps.sh
sudo apt install libatomic-ops-dev libboost-dev libboost-iostreams-dev libboost-thread-dev libboost-random-dev libboost-program-options-dev libcurl4-openssl-dev libcunit1 libcunit1-dev liblz4-dev liboath-dev liblttng-ust-dev libcrypto++ libcrypto++-dev libgoogle-perftools4 libtool cython libsnappy-dev libleveldb-dev libblkid-dev libudev-dev libkeyutils-dev libcrypto++-dev libcrypto++-doc libcrypto++-utils libfuse-dev libcurl4-openssl-dev libxml++2.6-dev libssl-dev libgoogle-perftools-dev libgoogle-perftools4 libatomic-ops-dev libaio-dev  xfslibs-dev libboost-iostreams-dev libfcgi-dev
sudo apt install gcr libblockdev-crypto2 libhcrypto4-heimdal libhogweed4 libk5crypto3 libnettle6 libsodium23 openssl python-m2crypto python3-crypto python3-cryptography
#./autogen.sh && ./configure  #old way, need to modifty makefile to '-O0-Wall -g' for debug
./do_cmake.sh -DCMAKE_BUILD_TYPE=Debug
cd build
make -j56
make ceph-osd -j56

MON=1 OSD=3 MDS=1 MGR=1 RGW=1 ../src/vstart.sh -n -d --without-dashboard
../src/stop.sh
./bin/ceph -s
./bin/ceph osd pool stats
ps aux|grep ceph
pstree -p

./bin/radosgw-admin user create --uid=admin --display-name=admin --access-key=admin --secret=password
./bin/radosgw-admin usage show --uid=admin
./bin/radosgw-admin bucket stats --uid=anonymous

sudo gdb attach $(pidof radosgw)
(gdb) set solib-search-path /bak/linux/ceph/src/.libs/
(gdb) set pagination off
(gdb) b rgw/rgw_process.cc:38
(gdb) b process_request
(gdb) c