26、rsync+inotify实现百万级数据实时同步

1、目标:192.168.130.61、192.168.130.62实时同步192.168.130.63的数据


2、配置服务器端192.168.130.61/62

yum -y install xinetd rsync


vim /etc/rsyncd.conf

uid = nobody

gid = nobody

use chroot = no

max connections = 10

strict modes = yes

pid file = /var/run/rsyncd.pid

log file = /var/log/rsyncd.log


[data]

path = /data/

ignore errors = yes

read only = no

write only = no

hosts allow = 192.168.0.0/16

hosts deny = *


list = false

uid = root

gid = root

auth users = username

secrets file = /etc/rsyncd.passwd


mkdir /data

echo "username:password" > /etc/rsyncd.passwd

chmod 600 /etc/rsyncd.passwd

chkconfig rsync on

service xinetd start

netstat -tuanlp | grep 873


mkdir /data


3、配置客户端192.168.130.63

yum -y install https://mirrors.aliyun.com/epel/6Server/x86_64/Packages/e/epel-release-6-8.noarch.rpm

yum -y install rsync inotify-tools


脚本实现当某一目录有文件变化是,自动同步改目录下的所有文件。

vim /etc/rsync.sh

#!/bin/bash

src=/data/

des=data

rsync_passwd_file=/etc/rsyncd.passwd

ip1=192.168.130.61

ip2=192.168.130.62

user=username

cd ${src}                            

/usr/bin/inotifywait -mrq --format  '%Xe %w%f' -e modify,create,delete,attrib,close_write,move ./ | while read file

do

        INO_EVENT=$(echo $file | awk '{print $1}')

        INO_FILE=$(echo $file | awk '{print $2}')

        if [[ $INO_EVENT =~ 'CREATE' ]] || [[ $INO_EVENT =~ 'MODIFY' ]] || [[ $INO_EVENT =~ 'CLOSE_WRITE' ]] || [[ $INO_EVENT =~ 'MOVED_TO' ]] || [[ $INO_EVENT =~ 'DELETE' ]] || [[ $INO_EVENT =~ 'MOVED_FROM' ]]

        then

                rsync -az --delete --password-file=${rsync_passwd_file} $(dirname ${INO_FILE}) ${user}@${ip1}::${des} &&

                rsync -az --delete --password-file=${rsync_passwd_file} $(dirname ${INO_FILE}) ${user}@${ip2}::${des}

        fi

        if [[ $INO_EVENT =~ 'ATTRIB' ]]

        then

                if [ ! -d "$INO_FILE" ]

                then

                        rsync -az --password-file=${rsync_passwd_file} $(dirname ${INO_FILE}) ${user}@${ip1}::${des} &&         

                        rsync -az --password-file=${rsync_passwd_file} $(dirname ${INO_FILE}) ${user}@${ip2}::${des}

                fi

        fi

done


chmod +x /etc/rsync.sh

mkdir /data

echo "/etc/rsync.sh &" >> /etc/rc.local


每两小时做1次全量同步

因为inotify只在启动时会监控目录,他没有启动期间的文件发生更改,他是不知道的,所以这里每2个小时做1次全量同步,防止各种意外遗漏,保证目录一致。

crontab -e

* */2 * * * rsync -az --password-file=/etc/rsyncd.passwd /data/ [email protected]::data && rsync -az --password-file=/etc/rsyncd.passwd /data/ [email protected]::data


echo 50000000 > /proc/sys/fs/inotify/max_user_watches (设置inotifywait或inotifywatch命令可以监视的文件数量(单进程))

echo 50000000 > /proc/sys/fs/inotify/max_queued_events (设置inotify实例事件(event)队列可容纳的事件数量)


echo "echo 50000000 > /proc/sys/fs/inotify/max_user_watches" >> /etc/rc.local

echo "echo 50000000 > /proc/sys/fs/inotify/max_queued_events" >> /etc/rc.local


echo password >> /etc/rsyncd.passwd

chmod 600 /etc/rsyncd.passwd


/etc/rsync.sh










猜你喜欢

转载自blog.51cto.com/kaiyuandiantang/2307795