前景:部署skywalking,elk等服务时会用到es存储数据,运行一段时间会占用磁盘空间,影响服务器性能。在保证服务正常使用的情况下,我们可以对es的索引进行删除操作,舍弃旧数据。所以我编写了一个脚本,仅供参考。如果要做定时任务的话,可以对稍作修改。
[root@dev-sw_es:shell]# vim delete_es_data.sh
#!/bin/bash
#本脚本用于按月份清理ES存储数据
#获取本月份与索引中的日期进行比较,本月份前的索引数据将被删除。
#删除前要确认
NOW_DATE=`date +%Y%m`
ALLLINES=`/usr/bin/curl -s -XGET http://127.0.0.1:9200/_cat/indices?v | egrep *-20*`
echo
echo "THIS IS WHAT SHOULD BE DELETED FOR ES:"
echo
echo "${ALLLINES}" | while read LINE
do
FORMATEDLINE=`echo ${LINE} |awk '{ print $3 }' | awk -F'-' '{ print $2 }' | cut -c 1-6 `
#echo ${FORMATEDLINE}
if [ -n "${FORMATEDLINE}" -a "${FORMATEDLINE}" -lt "${NOW_DATE}" ]
then
TODELETE=`echo ${LINE} | awk '{ print $3 }'`
echo "http://127.0.0.1:9200/${TODELETE}"
fi
done
echo
echo -n "if this make sence, Y to continue N to exit [Y/N]:"
read INPUT
if [ "${INPUT}" == "Y" ] || [ "${INPUT}" == "y" ] || [ "${INPUT}" == "yes" ] || [ "${INPUT}" == "YES" ]
then
echo "${ALLLINES}" | while read LINE
do
FORMATEDLINE=`echo ${LINE} | awk '{ print $3 }' | awk -F'-' '{ print $2 }' | cut -c 1-6 `
if [ -n "${FORMATEDLINE}" -a "${FORMATEDLINE}" -lt "${NOW_DATE}" ]
then
TODELETE=`echo ${LINE} | awk '{ print $3 }'`
/usr/bin/curl -XDELETE http://127.0.0.1:9200/$TODELETE
sleep 1
fi
done
else
echo SCRIPT CLOSED BY USER, BYE ...
echo
exit
fi
~
写进crontab的脚本可以参考:
#!/bin/bash
NOW_DATE=`date +%Y%m`
#NOW_DATE=202010
ALLLINES=`/usr/bin/curl -s -XGET http://127.0.0.1:9200/_cat/indices?v | egrep *-20*`
echo "${ALLLINES}" | while read LINE
do
FORMATEDLINE=`echo ${LINE} |awk '{ print $3 }' | awk -F'-' '{ print $2 }' | cut -c 1-6 `
if [ -n "${FORMATEDLINE}" -a "${FORMATEDLINE}" -lt "${NOW_DATE}" ]
then
TODELETE=`echo ${LINE} | awk '{ print $3 }'`
#echo "http://127.0.0.1:9200/${TODELETE}"
/usr/bin/curl -XDELETE http://127.0.0.1:9200/$TODELETE
sleep 1
fi
ELK的索引删除:
#I use a bash script, just change the 30 with the # of days you want to keep
#!/bin/bash
# Zero padded days using %d instead of %e
DAYSAGO=`date --date="30 days ago" +%Y%m%d`
ALLLINES=`/usr/bin/curl -s -XGET http://127.0.0.1:9200/_cat/indices?v | egrep logstash`
echo
echo "THIS IS WHAT SHOULD BE DELETED FOR ELK:"
echo
echo "$ALLLINES" | while read LINE
do
FORMATEDLINE=`echo $LINE | awk '{ print $3 }' | awk -F'-' '{ print $2 }' | sed 's/\.//g' `
if [ "$FORMATEDLINE" -lt "$DAYSAGO" ]
then
TODELETE=`echo $LINE | awk '{ print $3 }'`
echo "http://127.0.0.1:9200/$TODELETE"
fi
done
echo
echo -n "if this make sence, Y to continue N to exit [Y/N]:"
read INPUT
if [ "$INPUT" == "Y" ] || [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] || [ "$INPUT" == "YES" ]
then
echo "$ALLLINES" | while read LINE
do
FORMATEDLINE=`echo $LINE | awk '{ print $3 }' | awk -F'-' '{ print $2 }' | sed 's/\.//g' `
if [ "$FORMATEDLINE" -lt "$DAYSAGO" ]
then
TODELETE=`echo $LINE | awk '{ print $3 }'`
/usr/bin/curl -XDELETE http://127.0.0.1:9200/$TODELETE
sleep 1
fi
done
else
echo SCRIPT CLOSED BY USER, BYE ...
echo
exit
fi