storm分析系统实时日志
1. 启动zookeeper:
sh /Users/hyy044101331/java_tools/zk-jiqun-start.sh
2.
启动storm:
sh /Users/hyy044101331/java_tools/start-storm.sh
3.
启动web日志:
每3秒钟打印一条日志
4. 收集日志—启动server:
sh start-server.sh storm.starter.topology_02.server.Server
1 #!/bin/bash 2 3 ## 4 # receive client log data,then send to storm spout 5 ## 6 7 jar_path="/Users/hyy044101331/work_storm_topology/topology/target/topology-0.0.1-SNAPSHOT-jar-with-dependencies.jar" 8 9 java -classpath $jar_path $1
public class Constant { /** * SOCKET获取数据的ip地址 */ public static final String DATA_IP = "127.0.0.1"; /** * SOCKET获取数据的端口port */ public static final int DATA_PORT = 5678; public static final int DATA_CLIENT_PORT = 5679; /** * 日志文件 */ public static final String LOG_PATH = "/Users/hyy044101331/work_struts_03/mitty/mitty_error.log"; }
import org.apache.log4j.Logger; import storm.starter.topology_02.Constant; import java.io.BufferedReader; import java.io.InputStreamReader; import java.io.PrintWriter; import java.net.ServerSocket; import java.net.Socket; import java.util.ArrayList; /** * 接受client传输过来的日志数据 * <p/> * Created by mengka */ public class Server { private static final Logger log = Logger.getLogger(Server.class); private static Socket clientSocket; private static ArrayList<PrintWriter> outs = new ArrayList<PrintWriter>(); public static void main(String[] args) throws Exception { /** * 接收log数据 */ ServerSocket serverSocket_client = new ServerSocket(Constant.DATA_CLIENT_PORT); clientSocket = serverSocket_client.accept(); invoke(clientSocket, outs); ServerSocket serverSocket = new ServerSocket(Constant.DATA_PORT); while (true) { Socket socket = serverSocket.accept(); PrintWriter pwriter = new PrintWriter(socket.getOutputStream()); outs.add(pwriter); log.info("-----------, spout socket connected: "+socket.getInetAddress()); } } /** * 转发消息给storm的spout * * @param client * @param writers * @throws Exception */ private static void invoke(final Socket client, final ArrayList<PrintWriter> writers) throws Exception { new Thread(new Runnable() { public void run() { BufferedReader clientReader = null; PrintWriter clientWriter = null; PrintWriter writer = null; try { clientReader = new BufferedReader(new InputStreamReader(client.getInputStream())); clientWriter = new PrintWriter(client.getOutputStream()); while (true) { /** * 接受数据 */ String msg = clientReader.readLine(); System.out.println(msg); clientWriter.println("Server received " + msg); clientWriter.flush(); /** * 数据转发送到多个client */ for (int i = 0; i < writers.size(); i++) { writer = writers.get(i); System.out.println(i); System.out.println("send msg:" + msg); writer.println(msg); writer.flush(); } System.out.println(client.getInetAddress()); if (msg.equals("bye")) { break; } } } catch (Exception e) { e.printStackTrace(); } finally { //释放资源 try { clientReader.close(); } catch (Exception e) { } try { clientWriter.close(); } catch (Exception e) { } try { clientSocket.close(); } catch (Exception e) { } } } }).start(); } }
5.
收集日志—启动client:
sh start-server.sh storm.starter.topology_02.client.Client
import org.apache.log4j.Logger; import storm.starter.topology_02.Constant; import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.io.RandomAccessFile; import java.net.Socket; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; /** * Created by mengka */ public class Client { private static final Logger log = Logger.getLogger(Client.class); private long lastTimeFileSize = 0; //上次文件大小 private RandomAccessFile randomFile = null; public static void main(String[] args) throws Exception { /** * 建立连接 */ Socket socket = new Socket(Constant.DATA_IP, Constant.DATA_CLIENT_PORT); PrintWriter out = new PrintWriter(socket.getOutputStream()); /** * 发送实时日志数据 */ Client client = new Client(); client.realtimeShowLog(out); } /** * @param out * @throws IOException */ public void realtimeShowLog(final PrintWriter out) throws IOException { File logFile = new File(Constant.LOG_PATH); randomFile = new RandomAccessFile(logFile, "r"); //启动一个线程每1秒钟读取新增的日志信息 ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); executorService.scheduleWithFixedDelay(new RealtimeLogTask(out), 0, 1, TimeUnit.SECONDS); } /** * 实时读取日志文件task */ public class RealtimeLogTask implements Runnable { private PrintWriter out; public RealtimeLogTask(PrintWriter out) { this.out = out; } @Override public void run() { try { //获得变化部分的 randomFile.seek(lastTimeFileSize); String tmp = ""; while ((tmp = randomFile.readLine()) != null) { log.info("----------------, realtimeLogTask run..."); System.out.println(new String(tmp.getBytes("ISO8859-1"))); out.println(new String(tmp.getBytes("ISO8859-1"))); out.flush(); } lastTimeFileSize = randomFile.length(); } catch (IOException e) { throw new RuntimeException(e); } } } }
6.
提交任务:
storm jar topology-0.0.1-SNAPSHOT-jar-with-dependencies.jar storm.starter.topology_02.RealtimeLogTopology "mengka-mm"
7. 日志分析结果输出:
mengka-2:storm hyy044101331$ pwd /Users/hyy044101331/logs/storm mengka-2:storm hyy044101331$ vi data Press ENTER or type command to continue 1 apple-0 2 apple-0 3 apple-0 4 tiger-1 5 tiger-1