Storm流计算完整流程Demo 附代码

Storm流计算完整流程Demo 附代码

最近公司有项目要用到流计算，人手不足，我就入伙了，通过两天的学习搭建了一个小Demo，特此记录
原文链接：https://blog.csdn.net/weixin_44259356/article/details/100142397

前言

什么是大数据，流计算，Storm就不多说了，自行百度。
主要流程是通过RocketMQ发送和获取数据，然后通过JStorm计算拿取的数据，最后直接再本地控制台显示计算结果。
RocketMQ的安装可以参考如下：https://www.jianshu.com/p/4a275e779afa
maven搭建Storm可以项目参考如下：http://storm.apache.org/releases/2.0.0/Maven.html
那么重点如下：

Producer.class

发送数据到MQ

package RocketMq;

import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;

public class Producer {
    public static void main(String [] adsf){
        Producer producer=new Producer();
        producer.send();
    }


    public void send() {
        DefaultMQProducer producer = new DefaultMQProducer("Producer");
        producer.setNamesrvAddr("127.0.0.1:9876");
        try {
            producer.start();
            SendResult result;
            Message msg;
            String strDate;
            for(int i=0;i<100;++i) {
                strDate="$:"+i;
                msg = new Message("TestDate",
                        "push",
                        "key："+i,
                        strDate.getBytes());

                result = producer.send(msg);
                System.out.println("id:" + result.getMsgId() +
                        " result:" + result.getSendStatus()+"发送数量："+i);
                if(i>90) {
                    i = 0;
                    Thread.sleep(10000);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally{
            producer.shutdown();
        }
    }
}

Consumer.class

从MQ获取数据

package RocketMq;

import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageExt;

import java.util.ArrayList;
import java.util.List;

public class Consumer {

    private int index=0;

    public static void main(String [] adf){
        Consumer consumer=new Consumer();
        consumer.getMessage(new ArrayList<>());
    }
    public void getMessage(List<String> stringList) {
        DefaultMQPushConsumer consumer =
                new DefaultMQPushConsumer("PushConsumer");
        consumer.setNamesrvAddr("127.0.0.1:9876");
        try {

            consumer.subscribe("TestDate", "push");

            //程序第一次启动从消息队列头取数据
            consumer.setConsumeFromWhere(
                    ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
            consumer.registerMessageListener(new MessageListenerConcurrently() {
                                                 public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> list, ConsumeConcurrentlyContext Context) {
                                                     Message msg = list.get(0);
                                                     String topic = msg.getTopic();
                                                     byte[] body = msg.getBody();
                                                     String date=new String(body);
                                                     stringList.add(date);
                                                     System.out.println("获取的Topic为："+topic+"   获取的数据为："+stringList.get(index));
                                                     ++index;
/*                                                     String keys = msg.getKeys();
                                                     System.out.println("keys = " + keys);
                                                     String tags = msg.getTags();
                                                     System.out.println("tags = " + tags);
                                                     System.out.println("-----------------------------------------------");*/

                                                     return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
                                                 }
                                             }
            );
            consumer.start();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

SpoutTest.class

获取从MQ获取的数据然后给Bolt计算

package Strom;

import RocketMq.Consumer;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import com.sun.deploy.security.SelectableSecurityManager;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class SpoutTest extends BaseRichSpout {

    private SpoutOutputCollector collector;
    private List<String> stringList;

    {
        stringList = new ArrayList<String>();
    }

    private int index=0;

    @Override
    public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        this.collector = spoutOutputCollector;
        Consumer consumer=new Consumer();
        consumer.getMessage(stringList);
        System.out.println("stringList:"+stringList);
        System.out.println("发送端初始化结束");
    }

    @Override
    public void nextTuple() {
        System.out.println("数据发送中");
        if(stringList.size()!=0) {
            System.out.println("要发送的数据是"+stringList.get(index));
            if(!stringList.get(index).equals("null")) {
                this.collector.emit(new Values(stringList.get(index)));
            }
            index++;
            System.out.println("数据发送成功");
        }
        else
        {
            System.out.println("未获取到数据...");
        }
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        outputFieldsDeclarer.declare(new Fields("TestDate"));
        System.out.println("发送数据定义结束");
    }
}

BoltTest.class

计算流数据

package Strom;

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Map;

public class BoltTest  extends BaseRichBolt {
    private OutputCollector collector;
    @Override
    public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
        this.collector=outputCollector;
    }

    @Override
    public void execute(Tuple tuple) {
        System.out.println("计算开始");
        String sentence = tuple.getStringByField("TestDate");
        System.out.println(sentence);//这里不进行任何逻辑处理直接打印出接受的数据
        System.out.println("计算结束");

    }

    @Override
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {

    }
    public void cleanup(){
        System.out.println("---------- FINAL COUNTS -----------");

        System.out.println("计算结束部分，这里也不进行任何逻辑操作");

        System.out.println("----------------------------");
    }

}

CountTest.class

测试主程序

package com.shby.storm.test;

import Strom.BoltTest;
import Strom.SpoutTest;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.generated.StormTopology;
import backtype.storm.topology.TopologyBuilder;

public class CountTest {
    private static final String SPOUT_TEST_ID = "SpoutTest";
    private static final String SPLIT_TEST = "BoltTest";
    private static final String TOPOLOGY_NAME = "count-topology";
    public static void main(String[] args) throws InterruptedException {
        // TODO Auto-generated method stub
        System.out.println("计算测试Demo程序开始");
        SpoutTest spoutTest = new SpoutTest();
        BoltTest boltTest = new BoltTest();
        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout(SPOUT_TEST_ID, spoutTest);
        builder.setBolt(SPLIT_TEST, boltTest).setNumTasks(1).globalGrouping(SPOUT_TEST_ID);

        StormTopology topology = builder.createTopology();
        
        LocalCluster localCluster = new LocalCluster();
        Config config = new Config();
        localCluster.submitTopology("WordCountTopology", config, topology);

        System.out.println("测试主程序结束");
    }

}

最后成功结果如下