一：JS SDK
1.修改配置workspace
2.导入
3.Demo.html
4.效果
5.产生新的日志
tail -f access.log
二：重点
1.关于js的产生
参考程序analytics.js的JS SDK
1 (function() { 2 var CookieUtil = { 3 // get the cookie of the key is name 4 get : function(name) { 5 var cookieName = encodeURIComponent(name) + "=", cookieStart = document.cookie 6 .indexOf(cookieName), cookieValue = null; 7 if (cookieStart > -1) { 8 var cookieEnd = document.cookie.indexOf(";", cookieStart); 9 if (cookieEnd == -1) { 10 cookieEnd = document.cookie.length; 11 } 12 cookieValue = decodeURIComponent(document.cookie.substring( 13 cookieStart + cookieName.length, cookieEnd)); 14 } 15 return cookieValue; 16 }, 17 // set the name/value pair to browser cookie 18 set : function(name, value, expires, path, domain, secure) { 19 var cookieText = encodeURIComponent(name) + "=" 20 + encodeURIComponent(value); 21 22 if (expires) { 23 // set the expires time 24 var expiresTime = new Date(); 25 expiresTime.setTime(expires); 26 cookieText += ";expires=" + expiresTime.toGMTString(); 27 } 28 29 if (path) { 30 cookieText += ";path=" + path; 31 } 32 33 if (domain) { 34 cookieText += ";domain=" + domain; 35 } 36 37 if (secure) { 38 cookieText += ";secure"; 39 } 40 41 document.cookie = cookieText; 42 }, 43 setExt : function(name, value) { 44 this.set(name, value, new Date().getTime() + 315360000000, "/"); 45 } 46 }; 47 48 // 主体，其实就是tracker js 49 var tracker = { 50 // config 51 clientConfig : { 52 // TODO 这里的url需要传入具体的地址 53 serverUrl : "http://linux-hadoop3.ibeifeng.com/BEIfeng.gif", 54 sessionTimeout : 360, // 360s -> 6min 指定会话的过期时间，指的是操作停留最多的时间 55 maxWaitTime : 3600, // 3600s -> 60min -> 1h 指定的是单页面的最多停留时间，当前这个参数无效 56 ver : "1" 57 }, 58 59 cookieExpiresTime : 315360000000, // cookie过期时间，10年 60 61 columns : { 62 // 发送到服务器的列名称 63 eventName : "en", 64 version : "ver", 65 platform : "pl", 66 sdk : "sdk", 67 uuid : "u_ud", 68 memberId : "u_mid", 69 sessionId : "u_sd", 70 clientTime : "c_time", 71 language : "l", 72 userAgent : "b_iev", 73 resolution : "b_rst", 74 currentUrl : "p_url", 75 referrerUrl : "p_ref", 76 title : "tt", 77 orderId : "oid", 78 orderName : "on", 79 currencyAmount : "cua", 80 currencyType : "cut", 81 paymentType : "pt", 82 category : "ca", 83 action : "ac", 84 kv : "kv_", 85 duration : "du" 86 }, 87 88 keys : { 89 pageView : "e_pv", 90 chargeRequestEvent : "e_crt", 91 launch : "e_l", 92 eventDurationEvent : "e_e", 93 sid : "bftrack_sid", 94 uuid : "bftrack_uuid", 95 mid : "bftrack_mid", 96 preVisitTime : "bftrack_previsit", 97 98 }, 99 100 /**101 * 获取会话id102 */103 getSid : function() {104 return CookieUtil.get(this.keys.sid);105 },106 107 /**108 * 保存会话id到cookie109 */110 setSid : function(sid) {111 if (sid) {112 CookieUtil.setExt(this.keys.sid, sid);113 }114 },115 116 /**117 * 获取uuid，从cookie中118 */119 getUuid : function() {120 return CookieUtil.get(this.keys.uuid);121 },122 123 /**124 * 保存uuid到cookie125 */126 setUuid : function(uuid) {127 if (uuid) {128 CookieUtil.setExt(this.keys.uuid, uuid);129 }130 },131 132 /**133 * 获取memberID134 */135 getMemberId : function() {136 return CookieUtil.get(this.keys.mid);137 },138 139 /**140 * 设置mid141 */142 setMemberId : function(mid) {143 if (mid) {144 CookieUtil.setExt(this.keys.mid, mid);145 }146 },147 148 // 入口方法149 startSession : function() {150 // 加载js就触发的方法151 if (this.getSid()) {152 // 会话id存在，表示uuid也存在153 if (this.isSessionTimeout()) {154 // 会话过期,产生新的会话155 this.createNewSession();156 } else {157 // 会话没有过期，更新最近访问时间158 this.updatePreVisitTime(new Date().getTime());159 }160 } else {161 // 会话id不存在，表示uuid也不存在162 this.createNewSession();163 }164 this.onPageView();165 },166 167 onLaunch : function() {168 // 触发launch事件169 var launch = {};170 launch[this.columns.eventName] = this.keys.launch; // 设置事件名称171 this.setCommonColumns(launch); // 设置公用columns172 this.sendDataToServer(this.parseParam(launch)); // 最终发送编码后的数据173 },174 175 onPageView : function() {176 // 触发page view事件177 if (this.preCallApi()) {178 var time = new Date().getTime();179 var pageviewEvent = {};180 pageviewEvent[this.columns.eventName] = this.keys.pageView;181 pageviewEvent[this.columns.currentUrl] = window.location.href; // 设置当前url182 pageviewEvent[this.columns.referrerUrl] = document.referrer; // 设置前一个页面的url183 pageviewEvent[this.columns.title] = document.title; // 设置title184 this.setCommonColumns(pageviewEvent); // 设置公用columns185 this.sendDataToServer(this.parseParam(pageviewEvent)); // 最终发送编码后的数据186 this.updatePreVisitTime(time); // 更新最近访问时间187 }188 },189 190 onChargeRequest : function(orderId, name, currencyAmount, currencyType,191 paymentType) {192 // 触发订单产生事件193 if (this.preCallApi()) {194 if (!orderId || !currencyType || !paymentType) {195 this.log("订单id、货币类型以及支付方式不能为空");196 return;197 }198 199 if (typeof (currencyAmount) == "number") {200 // 金额必须是数字201 var time = new Date().getTime();202 var chargeRequestEvent = {};203 chargeRequestEvent[this.columns.eventName] = this.keys.chargeRequestEvent;204 chargeRequestEvent[this.columns.orderId] = orderId;205 chargeRequestEvent[this.columns.orderName] = name;206 chargeRequestEvent[this.columns.currencyAmount] = currencyAmount;207 chargeRequestEvent[this.columns.currencyType] = currencyType;208 chargeRequestEvent[this.columns.paymentType] = paymentType;209 this.setCommonColumns(chargeRequestEvent); // 设置公用columns210 this.sendDataToServer(this.parseParam(chargeRequestEvent)); // 最终发送编码后的数据ss211 this.updatePreVisitTime(time);212 } else {213 this.log("订单金额必须是数字");214 return;215 }216 }217 },218 219 onEventDuration : function(category, action, map, duration) {220 // 触发event事件221 if (this.preCallApi()) {222 if (category && action) {223 var time = new Date().getTime();224 var event = {};225 event[this.columns.eventName] = this.keys.eventDurationEvent;226 event[this.columns.category] = category;227 event[this.columns.action] = action;228 if (map) {229 // map如果不为空，进行内容的添加230 for ( var k in map) {231 // 循环key232 if (k && map[k]) {233 // 当key和value不为空的时候，进行添加操作234 event[this.columns.kv + k] = map[k]; // key添加前缀"kv_"235 }236 }237 }238 if (duration) {239 event[this.columns.duration] = duration; // 当duration不为0的时候进行添加240 }241 this.setCommonColumns(event); // 设置公用columns242 this.sendDataToServer(this.parseParam(event)); // 最终发送编码后的数据ss243 this.updatePreVisitTime(time);244 } else {245 this.log("category和action不能为空");246 }247 }248 },249 250 /**251 * 执行对外方法前必须执行的方法252 */253 preCallApi : function() {254 if (this.isSessionTimeout()) {255 // 如果为true，表示需要新建256 this.startSession();257 } else {258 this.updatePreVisitTime(new Date().getTime());259 }260 return true;261 },262 263 sendDataToServer : function(data) {264 // 发送数据data到服务器，其中data是一个字符串265 // TODO：发送以前发送失败的数据266 var that = this;267 var i2 = new Image(1, 1);268 i2.onerror = function() {269 // 这里可以进行重试操作270 // 当请求失败的情况下，执行这块的代码，可以将数据保存到local stroage中，下次再重新发送数据271 };272 // 给定图片的请求url273 i2.src = this.clientConfig.serverUrl + "?" + data;274 },275 276 /**277 * 往data中添加发送到日志收集服务器的公用部分278 */279 setCommonColumns : function(data) {280 data[this.columns.version] = this.clientConfig.ver;281 data[this.columns.platform] = "website";282 data[this.columns.sdk] = "js";283 data[this.columns.uuid] = this.getUuid(); // 设置用户id284 data[this.columns.memberId] = this.getMemberId(); // 设置会员id285 data[this.columns.sessionId] = this.getSid(); // 设置sid286 data[this.columns.clientTime] = new Date().getTime(); // 设置客户端时间287 data[this.columns.language] = window.navigator.language; // 设置浏览器语言288 data[this.columns.userAgent] = window.navigator.userAgent; // 设置浏览器类型289 data[this.columns.resolution] = screen.width + "*" + screen.height; // 设置浏览器分辨率290 },291 292 /**293 * 创建新的会员，并判断是否是第一次访问页面，如果是，进行launch事件的发送。294 */295 createNewSession : function() {296 var time = new Date().getTime(); // 获取当前操作时间297 // 1. 进行会话更新操作298 var sid = this.generateId(); // 产生一个session id299 this.setSid(sid);300 this.updatePreVisitTime(time); // 更新最近访问时间301 // 2. 进行uuid查看操作302 if (!this.getUuid()) {303 // uuid不存在，先创建uuid，然后保存到cookie，最后触发launch事件304 var uuid = this.generateId(); // 产品uuid305 this.setUuid(uuid);306 this.onLaunch(); // 触发launch事件307 }308 },309 310 /**311 * 参数编码返回字符串312 */313 parseParam : function(data) {314 var params = "";315 for ( var e in data) {316 if (e && data[e]) {317 // 对key和value进行编码操作318 params += encodeURIComponent(e) + "="319 + encodeURIComponent(data[e]) + "&";320 }321 }322 if (params) {323 return params.substring(0, params.length - 1);324 } else {325 return params;326 }327 },328 329 /**330 * 产生uuid331 * UUID的产生逻辑，可以参考Java中UUID的生产代码332 */333 generateId : function() {334 var chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';335 var tmpid = [];336 var r;337 tmpid[8] = tmpid[13] = tmpid[18] = tmpid[23] = '-';338 tmpid[14] = '4';339 340 for (i = 0; i < 36; i++) {341 if (!tmpid[i]) {342 r = 0 | Math.random() * 16;343 tmpid[i] = chars[(i == 19) ? (r & 0x3) | 0x8 : r];344 }345 }346 return tmpid.join('');347 },348 349 /**350 * 判断这个会话是否过期，查看当前时间和最近访问时间间隔时间是否小于this.clientConfig.sessionTimeout351 * 如果是小于，返回false;否则返回true。352 */353 isSessionTimeout : function() {354 var time = new Date().getTime();355 var preTime = CookieUtil.get(this.keys.preVisitTime);356 if (preTime) {357 // 最近访问时间存在,那么进行区间判断358 return time - preTime > this.clientConfig.sessionTimeout * 1000;359 }360 return true;361 },362 363 /**364 * 更新最近访问时间365 */366 updatePreVisitTime : function(time) {367 CookieUtil.setExt(this.keys.preVisitTime, time);368 },369 370 /**371 * 打印日志372 */373 log : function(msg) {374 console.log(msg);375 },376 377 };378 379 // 对外暴露的方法名称380 window.__AE__ = {381 startSession : function() {382 tracker.startSession();383 },384 onPageView : function() {385 tracker.onPageView();386 },387 onChargeRequest : function(orderId, name, currencyAmount, currencyType,388 paymentType) {389 tracker.onChargeRequest(orderId, name, currencyAmount,390 currencyType, paymentType);391 },392 onEventDuration : function(category, action, map, duration) {393 tracker.onEventDuration(category, action, map, duration);394 },395 setMemberId : function(mid) {396 tracker.setMemberId(mid);397 }398 };399 400 // 自动加载方法401 var autoLoad = function() {402 // 进行参数设置403 var _aelog_ = _aelog_ || window._aelog_ || [];404 var memberId = null;405 for (i = 0; i < _aelog_.length; i++) {406 _aelog_[i][0] === "memberId" && (memberId = _aelog_[i][1]);407 }408 // 根据是给定memberid，设置memberid的值409 memberId && __AE__.setMemberId(memberId);410 // 启动session411 __AE__.startSession();412 };413 414 // 调用415 autoLoad();416 })();
2.同时在access.log收集日志
三：JAVA SDK
1.原理图
2.程序的重点
将日志数据发送到队列
取数据，将数据发送到nginx服务器
3.程序AnalyticsEngineSDK
1 package com.ibeifeng.sdk.java.logmake; 2 3 import java.io.UnsupportedEncodingException; 4 import java.net.URLEncoder; 5 import java.util.HashMap; 6 import java.util.Map; 7 import java.util.logging.Level; 8 import java.util.logging.Logger; 9 10 /** 11 * 分析引擎sdk java服务器端数据收集 12 * 13 * @author ibeifeng 14 * @version 1.0 15 * 16 */ 17 public class AnalyticsEngineSDK { 18 // 日志打印对象 19 private static final Logger log = Logger.getGlobal(); 20 // 请求url的主体部分 21 public static final String accessUrl = "http://linux-hadoop3.ibeifeng.com/BEIfeng.gif"; 22 private static final String platformName = "java_server"; 23 private static final String sdkName = "jdk"; 24 private static final String version = "1"; 25 26 /** 27 * 触发订单支付成功事件，发送事件数据到服务器 28 * 29 * @param orderId 30 * 订单支付id 31 * @param memberId 32 * 订单支付会员id 33 * @return 如果发送数据成功(加入到发送队列中)，那么返回true；否则返回false(参数异常&添加到发送队列失败). 34 */ 35 public static boolean onChargeSuccess(String orderId, String memberId) { 36 try { 37 if (isEmpty(orderId) || isEmpty(memberId)) { 38 // 订单id或者memberid为空 39 log.log(Level.WARNING, "订单id和会员id不能为空"); 40 return false; 41 } 42 // 代码执行到这儿，表示订单id和会员id都不为空。 43 Map data = new HashMap (); 44 data.put("u_mid", memberId); 45 data.put("oid", orderId); 46 data.put("c_time", String.valueOf(System.currentTimeMillis())); 47 data.put("ver", version); 48 data.put("en", "e_cs"); 49 data.put("pl", platformName); 50 data.put("sdk", sdkName); 51 // 创建url 52 String url = buildUrl(data); 53 // 发送url&将url加入到队列 54 SendDataMonitor.addSendUrl(url); 55 return true; 56 } catch (Throwable e) { 57 log.log(Level.WARNING, "发送数据异常", e); 58 } 59 return false; 60 } 61 62 /** 63 * 触发订单退款事件，发送退款数据到服务器 64 * 65 * @param orderId 66 * 退款订单id 67 * @param memberId 68 * 退款会员id 69 * @return 如果发送数据成功，返回true。否则返回false。 70 */ 71 public static boolean onChargeRefund(String orderId, String memberId) { 72 try { 73 if (isEmpty(orderId) || isEmpty(memberId)) { 74 // 订单id或者memberid为空 75 log.log(Level.WARNING, "订单id和会员id不能为空"); 76 return false; 77 } 78 // 代码执行到这儿，表示订单id和会员id都不为空。 79 Map data = new HashMap (); 80 data.put("u_mid", memberId); 81 data.put("oid", orderId); 82 data.put("c_time", String.valueOf(System.currentTimeMillis())); 83 data.put("ver", version); 84 data.put("en", "e_cr"); 85 data.put("pl", platformName); 86 data.put("sdk", sdkName); 87 // 构建url 88 String url = buildUrl(data); 89 // 发送url&将url添加到队列中 90 SendDataMonitor.addSendUrl(url); 91 return true; 92 } catch (Throwable e) { 93 log.log(Level.WARNING, "发送数据异常", e); 94 } 95 return false; 96 } 97 98 /** 99 * 根据传入的参数构建url100 * 101 * @param data102 * @return103 * @throws UnsupportedEncodingException104 */105 private static String buildUrl(Map data) throws UnsupportedEncodingException {106 StringBuilder sb = new StringBuilder();107 sb.append(accessUrl).append("?");108 for (Map.Entry entry : data.entrySet()) {109 if (isNotEmpty(entry.getKey()) && isNotEmpty(entry.getValue())) {110 // key和value不为空111 sb.append(entry.getKey().trim()).append("=").append(URLEncoder.encode(entry.getValue().trim(), "utf-8"))112 .append("&");113 // 解码114 // URLDecoder.decode("需要解码的内容", "utf-8");115 }116 }117 return sb.substring(0, sb.length() - 1);// 去掉最后&118 }119 120 /**121 * 判断字符串是否为空，如果为空，返回true。否则返回false。122 * 123 * @param value124 * @return125 */126 private static boolean isEmpty(String value) {127 return value == null || value.trim().isEmpty();128 }129 130 /**131 * 判断字符串是否非空，如果不是空，返回true。如果是空，返回false。132 * 133 * @param value134 * @return135 */136 private static boolean isNotEmpty(String value) {137 return !isEmpty(value);138 }139 }
4.程序SendDataMonitor
1 package com.ibeifeng.sdk.java.logmake; 2 3 import java.io.BufferedReader; 4 import java.io.IOException; 5 import java.io.InputStreamReader; 6 import java.net.HttpURLConnection; 7 import java.net.URL; 8 import java.util.concurrent.BlockingQueue; 9 import java.util.concurrent.LinkedBlockingQueue; 10 import java.util.logging.Level; 11 import java.util.logging.Logger; 12 13 /** 14 * 发送url数据的监控者，用于启动一个单独的线程来发送数据 15 * 16 * @author ibeifeng 17 * 18 */ 19 public class SendDataMonitor { 20 // 日志记录对象 21 private static final Logger log = Logger.getGlobal(); 22 // 队列，用户存储发送url， 并发控制的Int.maxSize大小的阻塞队列 23 private BlockingQueue queue = new LinkedBlockingQueue (); 24 // 用于单列的一个类对象 25 private static SendDataMonitor monitor = null; 26 27 private SendDataMonitor() { 28 // 私有构造方法，进行单列模式的创建 29 } 30 31 /** 32 * 获取单例的monitor对象实例 33 * 34 * @return 35 */ 36 public static SendDataMonitor getSendDataMonitor() { 37 if (monitor == null) { 38 synchronized (SendDataMonitor.class) { 39 if (monitor == null) { 40 monitor = new SendDataMonitor(); 41 42 Thread thread = new Thread(new Runnable() { 43 44 @Override 45 public void run() { 46 // 线程中调用具体的处理方法 47 SendDataMonitor.monitor.run(); 48 } 49 }); 50 // 测试的时候，不设置为守护模式 51 // thread.setDaemon(true); 52 thread.start(); 53 } 54 } 55 } 56 return monitor; 57 } 58 59 /** 60 * 添加一个url到队列中去 61 * 62 * @param url 63 * @throws InterruptedException 64 */ 65 public static void addSendUrl(String url) throws InterruptedException { 66 getSendDataMonitor().queue.put(url); 67 } 68 69 /** 70 * 具体执行发送url的方法 71 * 72 */ 73 private void run() { 74 while (true) { 75 try { 76 // take 方法是阻塞方法，队列上有数据则取出，队列上没有数据则等待 77 String url = this.queue.take(); 78 // 正式的发送url 79 HttpRequestUtil.sendData(url); 80 } catch (Throwable e) { 81 log.log(Level.WARNING, "发送url异常", e); 82 } 83 } 84 } 85 86 /** 87 * 内部类，用户发送数据的http工具类 88 * 89 * @author ibeifeng 90 * 91 */ 92 public static class HttpRequestUtil { 93 /** 94 * 具体发送url的方法 95 * 96 * @param url 97 * @throws IOException 98 */ 99 public static void sendData(String url) throws IOException {100 HttpURLConnection con = null;101 BufferedReader in = null;102 103 try {104 URL obj = new URL(url); // 创建url对象105 con = (HttpURLConnection) obj.openConnection(); // 打开url连接106 // 设置连接参数107 con.setConnectTimeout(5000); // 连接过期时间108 con.setReadTimeout(5000); // 读取数据过期时间109 con.setRequestMethod("GET"); // 设置请求类型为get110 111 System.out.println("发送url:" + url);112 // 发送连接请求113 in = new BufferedReader(new InputStreamReader(con.getInputStream()));114 // TODO: 这里考虑是否可以115 } finally {116 try {117 if (in != null) {118 in.close();119 }120 } catch (Throwable e) {121 // nothing122 }123 try {124 con.disconnect();125 } catch (Throwable e) {126 // nothing127 }128 }129 }130 }131 }
5.测试
6.结果
7.程序Test
1 package com.ibeifeng.sdk.java.logmake.test; 2 3 import java.util.HashSet; 4 import java.util.Random; 5 import java.util.Set; 6 7 import com.ibeifeng.sdk.java.logmake.AnalyticsEngineSDK; 8 9 public class Test {10 private static Random random = new Random(System.currentTimeMillis());11 private static Set orders = new HashSet<>();12 13 public static void main(String[] args) throws InterruptedException {14 Order order = null; 15 while (true) {16 order = getSuccessOrder();17 // 发送订单付款行为数据18 AnalyticsEngineSDK.onChargeSuccess(order.orderId, order.memberId);19 Thread.sleep(random.nextInt(500));20 if (random.nextInt(100) > 75) {21 // 25%的订单发生退款行为22 order = getRefundOrder();23 if (order != null) {24 // 发送订单退款行为数据25 AnalyticsEngineSDK.onChargeRefund(order.orderId, order.memberId);26 Thread.sleep(random.nextInt(500));27 }28 }29 }30 }31 32 private static Order getSuccessOrder() {33 while (true) {34 int orderId = random.nextInt(Math.max(200000, orders.size() * 2));35 Order order = new Order();36 order.orderId = "orderid" + orderId;37 if (!orders.contains(order)) {38 // 该order是一个新的order对象39 order.memberId = "ibeifeng" + random.nextInt(1000);40 orders.add(order);41 return order;42 }43 }44 }45 46 private static Order getRefundOrder() {47 int count = 0;48 Order[] os = orders.toArray(new Order[0]);49 while (true) {50 count++;51 int index = random.nextInt(os.length); // 获取下标位置52 Order order = os[index]; // 获取对应下标位置的数据53 if (!order.refund) {54 order.refund = true; // 设置为已经退款操作55 return order;56 } else if (count >= os.length) {57 // 设置最多重试次数58 return null;59 }60 }61 }62 63 static class Order {64 public String orderId;65 public String memberId;66 public boolean refund = false;67 68 @Override69 public int hashCode() {70 final int prime = 31;71 int result = 1;72 result = prime * result + ((orderId == null) ? 0 : orderId.hashCode());73 return result;74 }75 76 @Override77 public boolean equals(Object obj) {78 if (this == obj)79 return true;80 if (obj == null)81 return false;82 if (getClass() != obj.getClass())83 return false;84 Order other = (Order) obj;85 if (orderId == null) {86 if (other.orderId != null)87 return false;88 } else if (!orderId.equals(other.orderId))89 return false;90 return true;91 }92 }93 }
8.测试
