监控系统搭建

环境介绍

OS: CentOS Linux release 7.2.1511 (Core) 
kernel: 3.10.0-327.el7.x86_64

host: 10.16.31.41 
hostname: localhost

技术选型

流程概要
数据源: 数据来源为运维人员手动编写脚本 或通过某种方式从其他渠道进行数据查询
数据存储: 将数据存储于influxdb数据库中(时序型数据库, X 轴为时间, Y 轴为个人业务数据, 该库数据存在过期策略)
图表展示: grafana读取influxdb等数据源(mysql/es/open-falcon etc.), 通过配置查询语句进行展示

influxdb version: InfluxDB v1.2.0
DOCS: https://docs.influxdata.com/influxdb/v1.2/introduction/getting_started/
grafana version: 5.0.0
DOCS: http://docs.grafana.org/

数据传输

influxdb 概要

database: 数据库
measurement: 数据库中的表
points: 表里面的一行数据

influxDB中独有的一些概念
Point由时间戳（time）、数据（field）和标签（tags）组成

time: 每条数据记录的时间, 为数据插入时间
fields: 各种记录的值, 数据类型类似mysql中 int
tags: 各种有索引的属性, 数据类型类型mysql中 varchar

其中, 至少存在一个tags, 至少存在一个fields

插入数据

插入数据支持多种途径，如 shell(curl 发送post请求)/ python (requests.post方法) / java (httpclinet 包)/ etc.
将数据通过post形式即可存储, 其中, 数据插入成功返回状态码为204

数据查询

通过http://${GRAFANA_SERVER_IP}:3000 即可查看图表展示 或修改查询条件

SHELL DEMO

格式:
# 不需要指定时间, time 为数据插入时间
# curl -i -XPOST 'http://${INFLUXDB_SERVER_IP}:8086/write?db=${DATABASE}' --data-binary '${MEASUREMENTS_NAME},${TAG_NAME_1}=${TAG_VALUE_1},${TAG_NAME_N}=${TAG_VALUE_N} ${FIELDS_NAME_1}=${FIELDS_VALUE_1},${FIELDS_NAME_N}=${FIELDS_VALUE_N}'

# 需要指定时间, time 为指定插入时间
# curl -i -XPOST 'http://${INFLUXDB_SERVER_IP}:8086/write?db=${DATABASE}' --data-binary '${MEASUREMENTS_NAME},${TAG_NAME_1}=${TAG_VALUE_1},${TAG_NAME_N}=${TAG_VALUE_N} ${FIELDS_NAME_1}=${FIELDS_VALUE_1},${FIELDS_NAME_N}=${FIELDS_VALUE_N} '${TIMESTAMP}

${TIMESTAMP} -> `date +%s`000000000

# 注意事项:
fields 与 tags 之间需要空格分割, 多个fields 以"," 分割；多个tags 以"," 分割

示例:
# curl -i -XPOST 'http://10.6.31.41:8086/write?db=ops' --data-binary 'test,type=parse useTime=80,isFlinsh=0'
# curl -i -XPOST 'http://10.6.31.41:8086/write?db=ops' --data-binary 'test,type=parse useTime=60,isFlinsh=1 '1522047676000000000
HTTP/1.1 204 No Content
Content-Type: application/json
Request-Id: 8063c15d-30c3-11e8-a158-000000000000
X-Influxdb-Version: 1.2.0
Date: Mon, 26 Mar 2018 07:01:26 GMT

PYTHON DEMO

${TIMESTAMP} -> str(int(time.time())) + "000000000"

for row in result:
    table=row["inst_code"]+"_"+row["trans_code"]
    sums=str(row["总笔数"])
    success=str(row["成功数"])
    fails=str(row["失败数"])
    Psuccess=str(float(row["成功率"])*100)+"%"

    rows=[paytable,sums,success,fails,Psuccess]
    ### ----- 百分数转换 ------------
    ## 这里将百分比以int 形式存储，grafana设置百分比类型展示即可
    psuccess = rows[4].split("%")[0]

    ## 定义database
    influxdb_url = 'http://10.6.31.41:8086/write?db=ops'
    ## 组装查询插入数据 (注1)
    idata = rows[0]+",table="+(rows[0])+" sums=" + (rows[1])+",success="+(rows[2])+",Psuccess=" + psuccess +",fails="+(rows[3])
    #print(idata)

    # 发送post 请求
    response = requests.post(influxdb_url, data=idata)
    print response
注1:
fields 与 tags 之间需要空格分割, 多个fields 以"," 分割；多个tags 以"," 分割

influxdb启停脚本

#!/bin/bash  
#-------------CopyRight-------------  
#   Name:Influxdb start \ stop script  
#   Version Number:V1.0
#   Type:online  
#   Language:bash shell  
#   Date:2018-03-14  
#   Author: quanzhang
#------------Environment------------  
#   Terminal: column 80 line 24  
#   Linux 2.6.32-431.el6.x86_64  
#   GNU Bash 4.1.2  
#----------------------------------- 

# The working directory
INFLUXDB_HOME=$(cd `dirname $0`; pwd)
sed -i "s#INFLUXDB_HOME#$INFLUXDB_HOME#g" "$INFLUXDB_HOME"/conf/influxdb.conf

# Daemon name, where is the actual executable
DAEMON=${INFLUXDB_HOME}/bin/influxd

# Configuration file
CONFIG=${INFLUXDB_HOME}/conf/influxdb.conf

# Log path
LOG_PATH=${INFLUXDB_HOME}/logs/influxd.log

# PID file for the daemon
PIDFILE=${INFLUXDB_HOME}/var/influxd.pid

# Process name ( For display )
NAME=influxdb

# Daemon options
INFLUXD_OPTS=

FAIL="\e[1;31m"
SUCC="\e[1;32m"
END="\033[0m"
TRUE=/bin/true
FALSE=/bin/false

chmod +x ${INFLUXDB_HOME}/bin/influx*

#
# Source function library.
#
if [ -f /etc/init.d/functions ]; then
    . /etc/init.d/functions
fi

PIDDIR=`dirname $PIDFILE`
if [ ! -d "$PIDDIR" ]; then
    mkdir -p $PIDDIR
fi

# Max open files
OPEN_FILE_LIMIT=65536


# Logging
if [ -z "$STDOUT" ]; then
    STDOUT=/dev/null
fi

if [ ! -f "$STDOUT" ]; then
    mkdir -p $(dirname $STDOUT)
fi

if [ -z "$STDERR" ]; then
    STDERR=${LOG_PATH}
fi

if [ ! -f "$STDERR" ]; then
    mkdir -p $(dirname $STDERR)
fi

function pidofproc() {
    if [ $# -ne 3 ]; then
        action "Expected three arguments, e.g. $0 -p pidfile daemon-name" $FALSE
    fi

    PID=`pgrep -f $3`
    local PIDFILE=`cat $2`

    if [ "x$PIDFILE" == "x" ]; then
        return 1
    fi

    if [ "x$PID" != "x" -a "$PIDFILE" == "$PID" ]; then
        return 0
    fi

    return 1
}

function killproc() {
    if [ $# -ne 3 ]; then
        action "Expected three arguments, e.g. $0 -p pidfile signal" $FALSE
    fi

    PID=`cat $2`

    /bin/kill -s $3 $PID
    while true; do
        pidof `basename $DAEMON` >/dev/null
        if [ $? -ne 0 ]; then
            return 0
        fi

        sleep 1
        n=$(expr $n + 1)
        if [ $n -eq 30 ]; then
            /bin/kill -s SIGKILL $PID
            return 0
        fi
    done
}

function log_failure_msg() {
    action "$@" $FALSE
}

function log_success_msg() {
    action "$@" $TRUE
}

function start() {
    # Check if config file exist
    if [ ! -r $CONFIG ]; then
        log_failure_msg "config file doesn't exist (or you don't have permission to view)"
        exit 4
    fi

    # Checked the PID file exists and check the actual status of process
    if [ -e $PIDFILE ]; then
	PID="$(pgrep -f $PIDFILE)"
	if test -n "$PID" && kill -0 "$PID" &>/dev/null; then
	    # If the status is SUCCESS then don't need to start again.
            log_failure_msg "$NAME process is running"
            exit 0 # Exit
        fi
        # if PID file does not exist, check if writable
    else
        touch $PIDFILE > /dev/null 2>&1
        if [ $? -ne 0 ]; then
            log_failure_msg "$PIDFILE not writable, check permissions"
            exit 5
        fi
    fi

    # Bump the file limits, before launching the daemon. These will carry over to
    # launched processes.
    ulimit -n $OPEN_FILE_LIMIT
    if [ $? -ne 0 ]; then
        log_failure_msg "set open file limit to $OPEN_FILE_LIMIT"
    fi

    log_success_msg "Starting the process $NAME"
    if which start-stop-daemon > /dev/null 2>&1; then
        start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON -- -pidfile $PIDFILE -config $CONFIG $INFLUXD_OPTS >>$STDOUT 2>>$STDERR &
    else
        nohup $DAEMON -pidfile $PIDFILE -config $CONFIG $INFLUXD_OPTS >>$STDOUT 2>>$STDERR &
    fi
    log_success_msg "$NAME process was started"
}

function stop() {
    # Stop the daemon.
    if [ -e $PIDFILE ]; then
	PID="$(pgrep -f $PIDFILE)"
	if test -n "$PID" && kill -0 "$PID" &>/dev/null; then
            if killproc -p $PIDFILE SIGTERM && /bin/rm -rf $PIDFILE; then
                log_success_msg "$NAME process was stopped"
            else
                log_failure_msg "$NAME failed to stop service"
            fi
        fi
    else
        log_failure_msg "$NAME process is not running"
    fi
}

function restart() {
    # Restart the daemon.
    PID="$(pgrep -f $PIDFILE)"
    stop
    while test -n "$PID" && test -d "/proc/$PID" &>/dev/null
    do
        echo "Process $PID is still running..."
        sleep 1
    done
    start
}

function status() {
    # Check the status of the process.
    if [ -e $PIDFILE ]; then
	PID="$(pgrep -f $PIDFILE)"
	if test -n "$PID" && test -d "/proc/$PID" &>/dev/null; then
            log_success_msg "$NAME Process is running"
            exit 0
        else
            log_failure_msg "$NAME Process is not running"
            exit 1
        fi
    else
        log_failure_msg "$NAME Process is not running"
        exit 3
    fi
}

case $1 in
    start)
	start
        ;;

    stop)
	stop
        ;;

    restart)
	restart
        ;;

    status)
	status
        ;;

    version)
        $DAEMON version
        ;;

    *)
        # For invalid arguments, print the usage message.
        echo "Usage: $0 {start|stop|restart|status|version}"
        exit 2
        ;;
esac

grafana启停脚本

#!/bin/bash
#-------------CopyRight-------------  
#   Name:Grafana start \ stop script  
#   Version Number:V1.0
#   Type:online  
#   Language:bash shell  
#   Date:2018-03-14  
#   Author:quanzhang 
#------------Environment------------  
#   Terminal: column 80 line 24  
#   Linux 2.6.32-431.el6.x86_64  
#   GNU Bash 4.1.2  
#----------------------------------- 
	
# The working directory
# GRAFANA_HOME=$(cd `dirname $0`; pwd)
GRAFANA_HOME=/opt/grafana

PATH=/bin:/usr/bin:/sbin:/usr/sbin
NAME=grafana-server
DESC="Grafana Server"

CONF_DIR=${GRAFANA_HOME}/conf
WORK_DIR=${GRAFANA_HOME}
DATA_DIR=${GRAFANA_HOME}/data
PLUGINS_DIR=${GRAFANA_HOME}/plugins
LOG_DIR=${GRAFANA_HOME}/log
CONF_FILE=${CONF_DIR}/grafana.ini
MAX_OPEN_FILES=10000
PID_FILE_DIR=${GRAFANA_HOME}/var
PID_FILE=${PID_FILE_DIR}/$NAME.pid
DAEMON=${GRAFANA_HOME}/bin/$NAME
PID=""
FAIL="\e[1;31m"
SUCC="\e[1;32m"
END="\033[0m"
TRUE=/bin/true
FALSE=/bin/false

if [ ! -x $DAEMON ]; then
  echo "Program not installed or not executable"
  exit 5
fi

#
# Source function library.
#
if [ -f /etc/init.d/functions ]; then
    . /etc/init.d/functions
	
fi

DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR} cfg:default.paths.plugins=${PLUGINS_DIR}"

function isRunning() {
  status -p $PID_FILE $NAME > /dev/null 2>&1
}

case "$1" in
  start)
    #action $"Starting $DESC: .... " $TRUE

    isRunning
    if [ $? -eq 0 ]; then
      action "$DESC Already running."	$FALSE
      exit 0
    fi

    # Prepare environment
    mkdir -p "$LOG_DIR"
    mkdir -p "$DATA_DIR" 
    mkdir -p "$PID_FILE_DIR"
    touch "$PID_FILE" 

    if [ -n "$MAX_OPEN_FILES" ]; then
      ulimit -n $MAX_OPEN_FILES
    fi

    # Start Daemon
    cd $GRAFANA_HOME
    nohup ${DAEMON} ${DAEMON_OPTS} >> /dev/null 2>&1 &
    return=$?
    if [ $return -eq 0 ]
    then
      sleep 1
      # check if pid file has been written two
      if ! [[ -s $PID_FILE ]]; then
        action "FAILED" $FAILED
        exit 1
      fi
      i=0
      timeout=10
      # Wait for the process to be properly started before exiting
      until { cat "$PID_FILE" | xargs kill -0; } >/dev/null 2>&1
      do
        sleep 1
        i=$(($i + 1))
        if [ $i -gt $timeout ]; then
          action "FAILED" $FAILED
          exit 1
        fi
      done
    fi

  action $"Starting $DESC: .... " $TRUE  
  exit $return
    ;;
  stop)
    #echo -n "Stopping $DESC ..."
    if [ -f "$PID_FILE" ]; then
      PID=`cat $PID_FILE`
      kill -9 $PID
      if [ $? -eq 0 ]; then
         action "Stopping $DESC ..." $TRUE
      else
         action "Failed, pls check it." $FAILED
         exit 1
      fi
      rm -f "$PID_FILE"
      exit 0
    else
      action "$DESC not running" $FAILED
    fi

    exit 0
    ;;
  status)
    status -p $PID_FILE $NAME
    exit $?
    ;;
  restart|force-reload)
    if [ -f "$PID_FILE" ]; then
      $0 stop
      sleep 1
    fi
    $0 start
    ;;
  *)
    echo "Usage: $0 {start|stop|restart|force-reload|status}"
    exit 3
    ;;
esac