在线时间:8:00-16:00
迪恩网络APP
随时随地掌握行业动态
扫描二维码
关注迪恩网络微信公众号
在nginx这一层,接收到访问请求的时候,就把请求的流量上报发送给kafka storm才能去消费kafka中的实时的访问日志,然后去进行缓存热数据的统计 从lua脚本直接创建一个kafka producer,发送数据到kafka wget https://github.com/doujiang24/lua-resty-kafka/archive/master.zip yum install -y unzip unzip lua-resty-kafka-master.zip cp -rf /usr/local/lua-resty-kafka-master/lib/resty /usr/hello/lualib lua脚本: local cjson = require("cjson") local producer = require("resty.kafka.producer") local broker_list = { { host = "192.168.31.187", port = 9092 }, { host = "192.168.31.19", port = 9092 }, { host = "192.168.31.227", port = 9092 } } local log_json = {} log_json["headers"] = ngx.req.get_headers() log_json["uri_args"] = ngx.req.get_uri_args() log_json["body"] = ngx.req.read_body() log_json["http_version"] = ngx.req.http_version() log_json["method"] =ngx.req.get_method() log_json["raw_reader"] = ngx.req.raw_header() log_json["body_data"] = ngx.req.get_body_data() local message = cjson.encode(log_json); local productId = ngx.req.get_uri_args()["productId"] local async_producer = producer:new(broker_list, { producer_type = "async" }) local ok, err = async_producer:send("access-log", productId, message) if not ok then ngx.log(ngx.ERR, "kafka send err:", err) return end
两台机器上都这样做,才能统一上报流量到kafka bin/kafka-topics.sh --zookeeper 192.168.31.187:2181,192.168.31.19:2181,192.168.31.227:2181 --topic access-log --replication-factor 1 --partitions 1 --create bin/kafka-console-consumer.sh --zookeeper 192.168.31.187:2181,192.168.31.19:2181,192.168.31.227:2181 --topic access-log --from-beginning (1)kafka在187上的节点死掉了,可能是虚拟机的问题,杀掉进程,重新启动一下 nohup bin/kafka-server-start.sh config/server.properties & (2)需要在nginx.conf中,http部分,加入resolver 8.8.8.8; (3)需要在kafka中加入advertised.host.name = 192.168.31.187,重启三个kafka进程 (4)需要启动eshop-cache缓存服务,因为nginx中的本地缓存可能不在了
基于storm+kafka完成商品访问次数实时统计拓扑的开发: 1、kafka consumer spout 单独的线程消费,写入队列 nextTuple,每次都是判断队列有没有数据,有的话再去获取并发射出去,不能阻塞 2、日志解析bolt 3、商品访问次数统计bolt 4、基于LRUMap完成统计 import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.StormSubmitter; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.tuple.Fields; import org.apache.storm.utils.Utils; import com.roncoo.eshop.storm.bolt.LogParseBolt; import com.roncoo.eshop.storm.bolt.ProductCountBolt; import com.roncoo.eshop.storm.spout.AccessLogKafkaSpout; /** * 热数据统计拓扑 * @author Administrator * */ public class HotProductTopology { public static void main(String[] args) { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("AccessLogKafkaSpout", new AccessLogKafkaSpout(), 1); builder.setBolt("LogParseBolt", new LogParseBolt(), 5) .setNumTasks(5) .shuffleGrouping("AccessLogKafkaSpout"); builder.setBolt("ProductCountBolt", new ProductCountBolt(), 5) .setNumTasks(10) .fieldsGrouping("LogParseBolt", new Fields("productId")); Config config = new Config(); if(args != null && args.length > 1) { config.setNumWorkers(3); try { StormSubmitter.submitTopology(args[0], config, builder.createTopology()); } catch (Exception e) { e.printStackTrace(); } } else { LocalCluster cluster = new LocalCluster(); cluster.submitTopology("HotProductTopology", config, builder.createTopology()); Utils.sleep(30000); cluster.shutdown(); } } }
import java.util.Map; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; import com.alibaba.fastjson.JSONObject; /** * 日志解析的bolt * @author Administrator * */ public class LogParseBolt extends BaseRichBolt { private static final long serialVersionUID = -8017609899644290359L; private OutputCollector collector; @SuppressWarnings("rawtypes") public void prepare(Map conf, TopologyContext context, OutputCollector collector) { this.collector = collector; } public void execute(Tuple tuple) { String message = tuple.getStringByField("message"); JSONObject messageJSON = JSONObject.parseObject(message); JSONObject uriArgsJSON = messageJSON.getJSONObject("uri_args"); Long productId = uriArgsJSON.getLong("productId"); if(productId != null) { collector.emit(new Values(productId)); } } public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("productId")); } }
import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.storm.shade.org.json.simple.JSONArray; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.trident.util.LRUMap; import org.apache.storm.tuple.Tuple; import org.apache.storm.utils.Utils; import com.roncoo.eshop.storm.zk.ZooKeeperSession; /** * 商品访问次数统计bolt * @author Administrator * */ public class ProductCountBolt extends BaseRichBolt { private static final long serialVersionUID = -8761807561458126413L; private LRUMap<Long, Long> productCountMap = new LRUMap<Long, Long>(1000); private ZooKeeperSession zkSession; private int taskid; @SuppressWarnings("rawtypes") public void prepare(Map conf, TopologyContext context, OutputCollector collector) { this.zkSession = ZooKeeperSession.getInstance(); this.taskid = context.getThisTaskId(); new Thread(new ProductCountThread()).start(); // 1、将自己的taskid写入一个zookeeper node中,形成taskid的列表 // 2、然后每次都将自己的热门商品列表,写入自己的taskid对应的zookeeper节点 // 3、然后这样的话,并行的预热程序才能从第一步中知道,有哪些taskid // 4、然后并行预热程序根据每个taskid去获取一个锁,然后再从对应的znode中拿到热门商品列表 initTaskId(context.getThisTaskId()); } private void initTaskId(int taskid) { // ProductCountBolt所有的task启动的时候, 都会将自己的taskid写到同一个node的值中 // 格式就是逗号分隔,拼接成一个列表 // 111,211,355 zkSession.acquireDistributedLock(); String taskidList = zkSession.getNodeData(); if(!"".equals(taskidList)) { taskidList += "," + taskid; } else { taskidList += taskid; } zkSession.setNodeData("/taskid-list", taskidList); zkSession.releaseDistributedLock(); } private class ProductCountThread implements Runnable { public void run() { List<Map.Entry<Long, Long>> topnProductList = new ArrayList<Map.Entry<Long, Long>>(); while(true) { topnProductList.clear(); int topn = 3; if(productCountMap.size() == 0) { Utils.sleep(100); continue; } for(Map.Entry<Long, Long> productCountEntry : productCountMap.entrySet()) { if(topnProductList.size() == 0) { topnProductList.add(productCountEntry); } else { // 比较大小,生成最热topn的算法有很多种 // 但是我这里为了简化起见,不想引入过多的数据结构和算法的的东西 // 很有可能还是会有漏洞,但是我已经反复推演了一下了,而且也画图分析过这个算法的运行流程了 boolean bigger = false; for(int i = 0; i < topnProductList.size(); i++){ Map.Entry<Long, Long> topnProductCountEntry = topnProductList.get(i); if(productCountEntry.getValue() > topnProductCountEntry.getValue()) { int lastIndex = topnProductList.size() < topn ? topnProductList.size() - 1 : topn - 2; for(int j = lastIndex; j >= i; j--) { topnProductList.set(j + 1, topnProductList.get(j)); } topnProductList.set(i, productCountEntry); bigger = true; break; } } if(!bigger) { if(topnProductList.size() < topn) { topnProductList.add(productCountEntry); } } } } // 获取到一个topn list String topnProductListJSON = JSONArray.toJSONString(topnProductList); zkSession.setNodeData("/task-hot-product-list-" + taskid, topnProductListJSON); Utils.sleep(5000); } } } public void execute(Tuple tuple) { Long productId = tuple.getLongByField("productId"); Long count = productCountMap.get(productId); if(count == null) { count = 0L; } count++; productCountMap.put(productId, count); } public void declareOutputFields(OutputFieldsDeclarer declarer) { } }
import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.ArrayBlockingQueue; import kafka.consumer.Consumer; import kafka.consumer.ConsumerConfig; import kafka.consumer.ConsumerIterator; import kafka.consumer.KafkaStream; import kafka.javaapi.consumer.ConsumerConnector; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.base.BaseRichSpout; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.Values; import org.apache.storm.utils.Utils; /** * kafka消费数据的spout */ public class AccessLogKafkaSpout extends BaseRichSpout { private static final long serialVersionUID = 8698470299234327074L; private ArrayBlockingQueue<String> queue = new ArrayBlockingQueue<String>(1000); private SpoutOutputCollector collector; @SuppressWarnings("rawtypes") public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { this.collector = collector; startKafkaConsumer(); } @SuppressWarnings("rawtypes") private void startKafkaConsumer() { Properties props = new Properties(); props.put("zookeeper.connect", "192.168.31.187:2181,192.168.31.19:2181,192.168.31.227:2181"); props.put("group.id", "eshop-cache-group"); props.put("zookeeper.session.timeout.ms", "40000"); props.put("zookeeper.sync.time.ms", "200"); props.put("auto.commit.interval.ms", "1000"); ConsumerConfig consumerConfig = new ConsumerConfig(props); ConsumerConnector consumerConnector = Consumer. createJavaConsumerConnector(consumerConfig); String topic = "access-log"; Map<String, Integer> topicCountMap = new HashMap<String, Integer>(); topicCountMap.put(topic, 1); Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumerConnector.createMessageStreams(topicCountMap); List<KafkaStream<byte[], byte[]>> streams = consumerMap.get(topic); for (KafkaStream stream : streams) { new Thread(new KafkaMessageProcessor(stream)).start(); } } private class KafkaMessageProcessor implements Runnable { @SuppressWarnings("rawtypes") private KafkaStream kafkaStream; @SuppressWarnings("rawtypes") public KafkaMessageProcessor(KafkaStream kafkaStream) { this.kafkaStream = kafkaStream; } @SuppressWarnings("unchecked") public void run() { ConsumerIterator<byte[], byte[]> it = kafkaStream.iterator(); while (it.hasNext()) { String message = new String(it.next().message()); try { queue.put(message); } catch (InterruptedException e) { e.printStackTrace(); } } } } public void nextTuple() { if(queue.size() > 0) { try { String message = queue.take(); collector.emit(new Values(message)); } catch (Exception e) { e.printStackTrace(); } } else { Utils.sleep(100); } } public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("message")); } }
import java.util.concurrent.CountDownLatch; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; import org.apache.zookeeper.Watcher.Event.KeeperState; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.Stat; /** * ZooKeeperSession * @author Administrator * */ public class ZooKeeperSession { private static CountDownLatch connectedSemaphore = new CountDownLatch(1); private ZooKeeper zookeeper; public ZooKeeperSession() { // 去连接zookeeper server,创建会话的时候,是异步去进行的 // 所以要给一个监听器,说告诉我们什么时候才是真正完成了跟zk server的连接 try { this.zookeeper = new ZooKeeper( "192.168.31.187:2181,192.168.31.19:2181,192.168.31.227:2181", 50000, new ZooKeeperWatcher()); // 给一个状态CONNECTING,连接中 System.out.println(zookeeper.getState()); try { // CountDownLatch // java多线程并发同步的一个工具类 // 会传递进去一些数字,比如说1,2 ,3 都可以 // 然后await(),如果数字不是0,那么久卡住,等待 // 其他的线程可以调用coutnDown(),减1 // 如果数字减到0,那么之前所有在await的线程,都会逃出阻塞的状态 // 继续向下运行 connectedSemaphore.await(); } catch(InterruptedException e) { e.printStackTrace(); } System.out.println("ZooKeeper session established......"); } catch (Exception e) { e.printStackTrace(); } } /** * 获取分布式锁 * @param productId */ public void acquireDistributedLock() { String path = "/taskid-list-lock"; try { zookeeper.create(path, "".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); System.out.println("success to acquire lock for taskid-list-lock"); } catch (Exception e) { // 如果那个商品对应的锁的node,已经存在了,就是已经被别人加锁了,那么就这里就会报错 // NodeExistsException int count = 0; while(true) { try { Thread.sleep(1000); zookeeper.create(path, "".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); } catch (Exception e2) { count++; System.out.println("the " + count + " times try to acquire lock for taskid-list-lock......"); continue; } System.out.println("success to acquire lock for taskid-list-lock after " + count + " times try......"); break; } } } /** * 释放掉一个分布式锁 * @param productId */ public void releaseDistributedLock() { String path = "/taskid-list-lock"; try { zookeeper.delete(path, -1); System.out.println("release the lock for taskid-list-lock......"); } catch (Exception e) { e.printStackTrace(); } } public String getNodeData() { try { return new String(zookeeper.getData("/taskid-list", false, new Stat())); } catch (Exception e) { e.printStackTrace(); } return ""; } public void setNodeData(String path, String data) { try { zookeeper.setData(path, data.getBytes(), -1); } catch (Exception e) { e.printStackTrace(); } } /** * 建立zk session的watcher * @author Administrator * */ private class ZooKeeperWatcher implements Watcher { public void process(WatchedEvent event) { System.out.println("Receive watched event: " + event.getState()); if(KeeperState.SyncConnected == event.getState()) { connectedSemaphore.countDown(); } } } /** * 封装单例的静态内部类 * @author Administrator * */ private static class Singleton { private static ZooKeeperSession instance; static { instance = new ZooKeeperSession(); } public static ZooKeeperSession getInstance() { return instance; } } /** * 获取单例 * @return */ public static ZooKeeperSession getInstance() { return Singleton.getInstance(); } /** * 初始化单例的便捷方法 */ public static void init() { getInstance(); } } 于双重zookeeper分布式锁完成分布式并行缓存预热:
2、从zk中读取taskid列表 3、依次遍历每个taskid,尝试获取分布式锁,如果获取不到,快速报错,不要等待,因为说明已经有其他服务实例在预热了 4、直接尝试获取下一个taskid的分布式锁 5、即使获取到了分布式锁,也要检查一下这个taskid的预热状态,如果已经被预热过了,就不再预热了 6、执行预热操作,遍历productid列表,查询数据,然后写ehcache和redis 7、预热完成后,设置taskid对应的预热状态 ZKsession重载两个方法: /** * 获取分布式锁 * @param productId */ public boolean acquireFastFailedDistributedLock(String path) { try { zookeeper.create(path, "".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); System.out.println("success to acquire lock for " + path); return true; } catch (Exception e) { System.out.println("fail to acquire lock for " + path); } return false; } /** * 释放掉一个分布式锁 * @param productId */ public void releaseDistributedLock(String path) { try { zookeeper.delete(path, -1); System.out.println("release the lock for " + path + "......"); } catch (Exception e) { e.printStackTrace(); } } public String getNodeData(String path) { try { return new String(zookeeper.getData(path, false, new Stat())); } catch (Exception e) { e.printStackTrace(); } return ""; } public void setNodeData(String path, String data) { try { zookeeper.setData(path, data.getBytes(), -1); } catch (Exception e) { e.printStackTrace(); } }
/** * 获取分布式锁 */ public void acquireDistributedLock(String path) { try { zookeeper.create(path, "".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); System.out.println("success to acquire lock for " + path); } catch (Exception e) { // 如果那个商品对应的锁的node,已经存在了,就是已经被别人加锁了,那么就这里就会报错 // NodeExistsException int count = 0; while(true) { try { Thread.sleep(1000); zookeeper.create(path, "".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); } catch (Exception e2) { count++; System.out.println("the " + count + " times try to acquire lock for " + path + "......"); continue; } System.out.println("success to acquire lock for " + path + " after " + count + " times try......"); break; } } }
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.roncoo.eshop.cache.model.ProductInfo; import com.roncoo.eshop.cache.service.CacheService; import com.roncoo.eshop.cache.spring.SpringContext; import com.roncoo.eshop.cache.zk.ZooKeeperSession; /** * 缓存预热线程 */ public class CachePrewarmThread extends Thread { @Override public void run() { CacheService cacheService = (CacheService) SpringContext. getApplicationContext().getBean("cacheService"); ZooKeeperSession zkSession = ZooKeeperSession.getInstance(); // 获取storm taskid列表 String taskidList = zkSession.getNodeData("/taskid-list"); if(taskidList != null && !"".equals(taskidList)) { String[] taskidListSplited = taskidList.split(","); for(String taskid : taskidListSplited) { String taskidLockPath = "/taskid-lock-" + taskid; boolean result = zkSession.acquireFastFailedDistributedLock(taskidLockPath); if(!result) { continue; } String taskidStatusLockPath = "/taskid-status-lock-" + taskid; zkSession.acquireDistributedLock(taskidStatusLockPath); //检查越热的状态 String taskidStatus = zkSession.getNodeData("/taskid-status-" + taskid); if("".equals(taskidStatus)) { String productidList = zkSession.getNodeData("/task-hot-product-list-" + taskid); JSONArray productidJSONArray = JSONArray.parseArray(productidList); for(int i = 0; i < productidJSONArray.size(); i++) { Long productId = productidJSONArray.getLong(i); String productInfoJSON = "{\"id\": " + productId + ", \"name\": \"iphone7手机\", \"price\": 5599, \"pictureList\":\"a.jpg,b.jpg\", \"specification\": \"iphone7的规格\", \"service\": \"iphone7的售后服务\", \"color\": \"红色,白色,黑色\", \"size\": \"5.5\", \"shopId\": 1, \"modifiedTime\": \"2017-01-01 12:00:00\"}"; ProductInfo productInfo = JSONObject.parseObject(productInfoJSON, ProductInfo.class); cacheService.saveProductInfo2LocalCache(productInfo); cacheService.saveProductInfo2ReidsCache(productInfo); } zkSession.setNodeData(taskidStatusLockPath, "success"); } zkSession.releaseDistributedLock(taskidStatusLockPath); zkSession.releaseDistributedLock(taskidLockPath); } } } }
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.roncoo.eshop.cache.model.ProductInfo; import com.roncoo.eshop.cache.service.CacheService; import com.roncoo.eshop.cache.spring.SpringContext; import com.roncoo.eshop.cache.zk.ZooKeeperSession; /** * 缓存预热线程 */ public class CachePrewarmThread extends Thread { @Override public void run() { CacheService cacheService = (CacheService) SpringContext. getApplicationContext().getBean("cacheService"); ZooKeeperSession zkSession = ZooKeeperSession.getInstance(); // 获取storm taskid列表 String taskidList = zkSession.getNodeData("/taskid-list"); if(taskidList != null && !"".equals(taskidList)) { String[] taskidListSplited = taskidList.split(","); for(String taskid : taskidListSplited) { String taskidLockPath = "/taskid-lock-" + taskid; boolean result = zkSession.acquireFastFailedDistributedLock(taskidLockPath); if(!result) { continue; } String taskidStatusLockPath = "/taskid-status-lock-" + taskid; zkSession.acquireDistributedLock(taskidStatusLockPath); //检查越热的状态 String taskidStatus = zkSession.getNodeData("/taskid-status-" + taskid); if("".equals(taskidStatus)) { String productidList = zkSession.getNodeData("/task-hot-product-list-" + taskid); JSONArray productidJSONArray = JSONArray.parseArray(productidList); for(int i = 0; i < productidJSONArray.size(); i++) { Long productId = productidJSONArray.getLong(i); String productInfoJSON = "{\"id\": " + productId + ", \"name\": \"iphone7手机\", \"price\": 5599, \"pictureList\":\"a.jpg,b.jpg\", \"specification\": \"iphone7的规格\", \"service\": \"iphone7的售后服务\", \"color\": \"红色,白色,黑色\", \"size\": \"5.5\", \"shopId\": 1, \"modifiedTime\": \"2017-01-01 12:00:00\"}"; ProductInfo productInfo = JSONObject.parseObject(productInfoJSON, ProductInfo.class); cacheService.saveProductInfo2LocalCache(productInfo); cacheService.saveProductInfo2ReidsCache(productInfo); } zkSession.setNodeData(taskidStatusLockPath, "success"); } zkSession.releaseDistributedLock(taskidStatusLockPath); zkSession.releaseDistributedLock(taskidLockPath); } } } }
|
请发表评论