1e56d7b71SSascha Leib"use strict"; 2e56d7b71SSascha Leib/* DokuWiki BotMon Plugin Script file */ 3e56d7b71SSascha Leib/* 14.10.2025 - 0.5.0 - pre-release */ 4e56d7b71SSascha Leib/* Author: Sascha Leib <ad@hominem.info> */ 5e56d7b71SSascha Leib 6e56d7b71SSascha Leib// enumeration of user types: 7e56d7b71SSascha Leibconst BM_USERTYPE = Object.freeze({ 8e56d7b71SSascha Leib 'UNKNOWN': 'unknown', 9e56d7b71SSascha Leib 'KNOWN_USER': 'user', 10e56d7b71SSascha Leib 'PROBABLY_HUMAN': 'human', 11e56d7b71SSascha Leib 'LIKELY_BOT': 'likely_bot', 12e56d7b71SSascha Leib 'KNOWN_BOT': 'known_bot' 13e56d7b71SSascha Leib}); 14e56d7b71SSascha Leib 15e56d7b71SSascha Leib// enumeration of log types: 16e56d7b71SSascha Leibconst BM_LOGTYPE = Object.freeze({ 17e56d7b71SSascha Leib 'SERVER': 'srv', 18e56d7b71SSascha Leib 'CLIENT': 'log', 19e56d7b71SSascha Leib 'TICKER': 'tck' 20e56d7b71SSascha Leib}); 21e56d7b71SSascha Leib 22e56d7b71SSascha Leib// enumeration of IP versions: 23e56d7b71SSascha Leibconst BM_IPVERSION = Object.freeze({ 24e56d7b71SSascha Leib 'IPv4': 4, 25e56d7b71SSascha Leib 'IPv6': 6 26e56d7b71SSascha Leib}); 27e56d7b71SSascha Leib 28e56d7b71SSascha Leib/* BotMon root object */ 29e56d7b71SSascha Leibconst BotMon = { 30e56d7b71SSascha Leib 31e56d7b71SSascha Leib init: function() { 32e56d7b71SSascha Leib //console.info('BotMon.init()'); 33e56d7b71SSascha Leib 34e56d7b71SSascha Leib // find the plugin basedir: 350edf1a56SSascha Leib this._baseDir = document.currentScript.src.substring(0, document.currentScript.src.lastIndexOf('/')+1); 36e56d7b71SSascha Leib 37e56d7b71SSascha Leib // read the page language from the DOM: 38e56d7b71SSascha Leib this._lang = document.getRootNode().documentElement.lang || this._lang; 39e56d7b71SSascha Leib 40e56d7b71SSascha Leib // get the time offset: 41e56d7b71SSascha Leib this._timeDiff = BotMon.t._getTimeOffset(); 42e56d7b71SSascha Leib 43e56d7b71SSascha Leib // get yesterday's date: 44e56d7b71SSascha Leib let d = new Date(); 45e56d7b71SSascha Leib d.setDate(d.getDate() - 1); 46e56d7b71SSascha Leib this._datestr = d.toISOString().slice(0, 10); 47e56d7b71SSascha Leib 48e56d7b71SSascha Leib // init the sub-objects: 49e56d7b71SSascha Leib BotMon.t._callInit(this); 50e56d7b71SSascha Leib }, 51e56d7b71SSascha Leib 52e56d7b71SSascha Leib _baseDir: null, 53e56d7b71SSascha Leib _lang: 'en', 54e56d7b71SSascha Leib _datestr: '', 55e56d7b71SSascha Leib _timeDiff: '', 56e56d7b71SSascha Leib 57e56d7b71SSascha Leib /* internal tools */ 58e56d7b71SSascha Leib t: { 59e56d7b71SSascha Leib /* helper function to call inits of sub-objects */ 60e56d7b71SSascha Leib _callInit: function(obj) { 61e56d7b71SSascha Leib //console.info('BotMon.t._callInit(obj=',obj,')'); 62e56d7b71SSascha Leib 63e56d7b71SSascha Leib /* call init / _init on each sub-object: */ 64e56d7b71SSascha Leib Object.keys(obj).forEach( (key,i) => { 65e56d7b71SSascha Leib const sub = obj[key]; 66e56d7b71SSascha Leib let init = null; 67e56d7b71SSascha Leib if (typeof sub === 'object' && sub.init) { 68e56d7b71SSascha Leib init = sub.init; 69e56d7b71SSascha Leib } 70e56d7b71SSascha Leib 71e56d7b71SSascha Leib // bind to object 72e56d7b71SSascha Leib if (typeof init == 'function') { 73e56d7b71SSascha Leib const init2 = init.bind(sub); 74e56d7b71SSascha Leib init2(obj); 75e56d7b71SSascha Leib } 76e56d7b71SSascha Leib }); 77e56d7b71SSascha Leib }, 78e56d7b71SSascha Leib 79e56d7b71SSascha Leib /* helper function to calculate the time difference to UTC: */ 80e56d7b71SSascha Leib _getTimeOffset: function() { 81e56d7b71SSascha Leib const now = new Date(); 82e56d7b71SSascha Leib let offset = now.getTimezoneOffset(); // in minutes 83e56d7b71SSascha Leib const sign = Math.sign(offset); // +1 or -1 84e56d7b71SSascha Leib offset = Math.abs(offset); // always positive 85e56d7b71SSascha Leib 86e56d7b71SSascha Leib let hours = 0; 87e56d7b71SSascha Leib while (offset >= 60) { 88e56d7b71SSascha Leib hours += 1; 89e56d7b71SSascha Leib offset -= 60; 90e56d7b71SSascha Leib } 91e56d7b71SSascha Leib return ( hours > 0 ? sign * hours + ' h' : '') + (offset > 0 ? ` ${offset} min` : ''); 92e56d7b71SSascha Leib }, 93e56d7b71SSascha Leib 94e56d7b71SSascha Leib /* helper function to create a new element with all attributes and text content */ 95e56d7b71SSascha Leib _makeElement: function(name, atlist = undefined, text = undefined) { 96e56d7b71SSascha Leib var r = null; 97e56d7b71SSascha Leib try { 98e56d7b71SSascha Leib r = document.createElement(name); 99e56d7b71SSascha Leib if (atlist) { 100e56d7b71SSascha Leib for (let attr in atlist) { 101e56d7b71SSascha Leib r.setAttribute(attr, atlist[attr]); 102e56d7b71SSascha Leib } 103e56d7b71SSascha Leib } 104e56d7b71SSascha Leib if (text) { 105e56d7b71SSascha Leib r.textContent = text.toString(); 106e56d7b71SSascha Leib } 107e56d7b71SSascha Leib } catch(e) { 108e56d7b71SSascha Leib console.error(e); 109e56d7b71SSascha Leib } 110e56d7b71SSascha Leib return r; 111e56d7b71SSascha Leib }, 112e56d7b71SSascha Leib 113e56d7b71SSascha Leib /* helper to convert an ip address string to a normalised format: */ 114e56d7b71SSascha Leib _ip2Num: function(ip) { 115e56d7b71SSascha Leib if (!ip) { 116e56d7b71SSascha Leib return 'null'; 117e56d7b71SSascha Leib } else if (ip.indexOf(':') > 0) { /* IP6 */ 118e56d7b71SSascha Leib return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(':')); 119e56d7b71SSascha Leib } else { /* IP4 */ 120e56d7b71SSascha Leib return ip.split('.').map(d => ('000'+d).slice(-3) ).join('.'); 121e56d7b71SSascha Leib } 122e56d7b71SSascha Leib }, 123e56d7b71SSascha Leib 124e56d7b71SSascha Leib /* helper function to format a Date object to show only the time. */ 125e56d7b71SSascha Leib /* returns String */ 126e56d7b71SSascha Leib _formatTime: function(date) { 127e56d7b71SSascha Leib 128e56d7b71SSascha Leib if (date) { 129e56d7b71SSascha Leib return date.getHours() + ':' + ('0'+date.getMinutes()).slice(-2) + ':' + ('0'+date.getSeconds()).slice(-2); 130e56d7b71SSascha Leib } else { 131e56d7b71SSascha Leib return null; 132e56d7b71SSascha Leib } 133e56d7b71SSascha Leib 134e56d7b71SSascha Leib }, 135e56d7b71SSascha Leib 136e56d7b71SSascha Leib /* helper function to show a time difference in seconds or minutes */ 137e56d7b71SSascha Leib /* returns String */ 138e56d7b71SSascha Leib _formatTimeDiff: function(dateA, dateB) { 139e56d7b71SSascha Leib 140e56d7b71SSascha Leib // if the second date is ealier, swap them: 141e56d7b71SSascha Leib if (dateA > dateB) dateB = [dateA, dateA = dateB][0]; 142e56d7b71SSascha Leib 143e56d7b71SSascha Leib // get the difference in milliseconds: 144e56d7b71SSascha Leib let ms = dateB - dateA; 145e56d7b71SSascha Leib 146e56d7b71SSascha Leib if (ms > 50) { /* ignore small time spans */ 147e56d7b71SSascha Leib const h = Math.floor((ms / (1000 * 60 * 60)) % 24); 148e56d7b71SSascha Leib const m = Math.floor((ms / (1000 * 60)) % 60); 149e56d7b71SSascha Leib const s = Math.floor((ms / 1000) % 60); 150e56d7b71SSascha Leib 151e56d7b71SSascha Leib return ( h>0 ? h + 'h ': '') + ( m>0 ? m + 'm ': '') + ( s>0 ? s + 's': ''); 152e56d7b71SSascha Leib } 153e56d7b71SSascha Leib 154e56d7b71SSascha Leib return null; 155e56d7b71SSascha Leib 156e56d7b71SSascha Leib }, 157e56d7b71SSascha Leib 158e56d7b71SSascha Leib // calcualte a reduced ration between two numbers 159e56d7b71SSascha Leib // adapted from https://stackoverflow.com/questions/3946373/math-in-js-how-do-i-get-a-ratio-from-a-percentage 160e56d7b71SSascha Leib _getRatio: function(a, b, tolerance) { 161e56d7b71SSascha Leib 162e56d7b71SSascha Leib var bg = b; 163e56d7b71SSascha Leib var sm = a; 164e56d7b71SSascha Leib if (a > b) { 165e56d7b71SSascha Leib var bg = a; 166e56d7b71SSascha Leib var sm = b; 167e56d7b71SSascha Leib } 168e56d7b71SSascha Leib 169e56d7b71SSascha Leib for (var i = 1; i < 1000000; i++) { 170e56d7b71SSascha Leib var d = sm / i; 171e56d7b71SSascha Leib var res = bg / d; 172e56d7b71SSascha Leib var howClose = Math.abs(res - res.toFixed(0)); 173e56d7b71SSascha Leib if (howClose < tolerance) { 174e56d7b71SSascha Leib if (a > b) { 175e56d7b71SSascha Leib return res.toFixed(0) + ':' + i; 176e56d7b71SSascha Leib } else { 177e56d7b71SSascha Leib return i + ':' + res.toFixed(0); 178e56d7b71SSascha Leib } 179e56d7b71SSascha Leib } 180e56d7b71SSascha Leib } 181e56d7b71SSascha Leib } 182e56d7b71SSascha Leib } 183e56d7b71SSascha Leib}; 184e56d7b71SSascha Leib 185e56d7b71SSascha Leib/* everything specific to the "Latest" tab is self-contained in the "live" object: */ 186e56d7b71SSascha LeibBotMon.live = { 187e56d7b71SSascha Leib init: function() { 1880edf1a56SSascha Leib console.info('BotMon.live.init()'); 189e56d7b71SSascha Leib 190e56d7b71SSascha Leib // set the title: 191e56d7b71SSascha Leib const tDiff = '<abbr title="Coordinated Universal Time">UTC</abbr> ' + (BotMon._timeDiff != '' ? ` (offset: ${BotMon._timeDiff}` : '' ) + ')'; 192e56d7b71SSascha Leib BotMon.live.gui.status.setTitle(`Data for <time datetime="${BotMon._datestr}">${BotMon._datestr}</time> ${tDiff}`); 193e56d7b71SSascha Leib 194e56d7b71SSascha Leib // init sub-objects: 195e56d7b71SSascha Leib BotMon.t._callInit(this); 196e56d7b71SSascha Leib }, 197e56d7b71SSascha Leib 198e56d7b71SSascha Leib data: { 199e56d7b71SSascha Leib init: function() { 200e56d7b71SSascha Leib //console.info('BotMon.live.data.init()'); 201e56d7b71SSascha Leib 202e56d7b71SSascha Leib // call sub-inits: 203e56d7b71SSascha Leib BotMon.t._callInit(this); 204e56d7b71SSascha Leib }, 205e56d7b71SSascha Leib 206e56d7b71SSascha Leib // this will be called when the known json files are done loading: 207e56d7b71SSascha Leib _dispatch: function(file) { 208e56d7b71SSascha Leib //console.info('BotMon.live.data._dispatch(,',file,')'); 209e56d7b71SSascha Leib 210e56d7b71SSascha Leib // shortcut to make code more readable: 211e56d7b71SSascha Leib const data = BotMon.live.data; 212e56d7b71SSascha Leib 213e56d7b71SSascha Leib // set the flags: 214e56d7b71SSascha Leib switch(file) { 215e56d7b71SSascha Leib case 'rules': 216e56d7b71SSascha Leib data._dispatchRulesLoaded = true; 217e56d7b71SSascha Leib break; 218e56d7b71SSascha Leib case 'ipranges': 219e56d7b71SSascha Leib data._dispatchIPRangesLoaded = true; 220e56d7b71SSascha Leib break; 221e56d7b71SSascha Leib case 'bots': 222e56d7b71SSascha Leib data._dispatchBotsLoaded = true; 223e56d7b71SSascha Leib break; 224e56d7b71SSascha Leib case 'clients': 225e56d7b71SSascha Leib data._dispatchClientsLoaded = true; 226e56d7b71SSascha Leib break; 227e56d7b71SSascha Leib case 'platforms': 228e56d7b71SSascha Leib data._dispatchPlatformsLoaded = true; 229e56d7b71SSascha Leib break; 230e56d7b71SSascha Leib default: 231e56d7b71SSascha Leib // ignore 232e56d7b71SSascha Leib } 233e56d7b71SSascha Leib 234e56d7b71SSascha Leib // are all the flags set? 235e56d7b71SSascha Leib if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded && data._dispatchIPRangesLoaded) { 236e56d7b71SSascha Leib // chain the log files loading: 237e56d7b71SSascha Leib BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded); 238e56d7b71SSascha Leib } 239e56d7b71SSascha Leib }, 240e56d7b71SSascha Leib // flags to track which data files have been loaded: 241e56d7b71SSascha Leib _dispatchBotsLoaded: false, 242e56d7b71SSascha Leib _dispatchClientsLoaded: false, 243e56d7b71SSascha Leib _dispatchPlatformsLoaded: false, 244e56d7b71SSascha Leib _dispatchIPRangesLoaded: false, 245e56d7b71SSascha Leib _dispatchRulesLoaded: false, 246e56d7b71SSascha Leib 247e56d7b71SSascha Leib // event callback, after the server log has been loaded: 248e56d7b71SSascha Leib _onServerLogLoaded: function() { 249e56d7b71SSascha Leib //console.info('BotMon.live.data._onServerLogLoaded()'); 250e56d7b71SSascha Leib 251e56d7b71SSascha Leib // chain the client log file to load: 252e56d7b71SSascha Leib BotMon.live.data.loadLogFile(BM_LOGTYPE.CLIENT, BotMon.live.data._onClientLogLoaded); 253e56d7b71SSascha Leib }, 254e56d7b71SSascha Leib 255e56d7b71SSascha Leib // event callback, after the client log has been loaded: 256e56d7b71SSascha Leib _onClientLogLoaded: function() { 257e56d7b71SSascha Leib //console.info('BotMon.live.data._onClientLogLoaded()'); 258e56d7b71SSascha Leib 259e56d7b71SSascha Leib // chain the ticks file to load: 260e56d7b71SSascha Leib BotMon.live.data.loadLogFile(BM_LOGTYPE.TICKER, BotMon.live.data._onTicksLogLoaded); 261e56d7b71SSascha Leib 262e56d7b71SSascha Leib }, 263e56d7b71SSascha Leib 264e56d7b71SSascha Leib // event callback, after the tiker log has been loaded: 265e56d7b71SSascha Leib _onTicksLogLoaded: function() { 266e56d7b71SSascha Leib //console.info('BotMon.live.data._onTicksLogLoaded()'); 267e56d7b71SSascha Leib 268e56d7b71SSascha Leib // analyse the data: 269e56d7b71SSascha Leib BotMon.live.data.analytics.analyseAll(); 270e56d7b71SSascha Leib 271e56d7b71SSascha Leib // sort the data: 272e56d7b71SSascha Leib // #TODO 273e56d7b71SSascha Leib 274e56d7b71SSascha Leib // display the data: 275e56d7b71SSascha Leib BotMon.live.gui.overview.make(); 276e56d7b71SSascha Leib 277e56d7b71SSascha Leib //console.log(BotMon.live.data.model._visitors); 278e56d7b71SSascha Leib 279e56d7b71SSascha Leib }, 280e56d7b71SSascha Leib 281e56d7b71SSascha Leib // the data model: 282e56d7b71SSascha Leib model: { 283e56d7b71SSascha Leib // visitors storage: 284e56d7b71SSascha Leib _visitors: [], 285e56d7b71SSascha Leib 286e56d7b71SSascha Leib // find an already existing visitor record: 287e56d7b71SSascha Leib findVisitor: function(visitor, type) { 288e56d7b71SSascha Leib //console.info('BotMon.live.data.model.findVisitor()', type); 289e56d7b71SSascha Leib //console.log(visitor); 290e56d7b71SSascha Leib 291e56d7b71SSascha Leib // shortcut to make code more readable: 292e56d7b71SSascha Leib const model = BotMon.live.data.model; 293e56d7b71SSascha Leib 294e56d7b71SSascha Leib const timeout = 60 * 60 * 1000; // session timeout: One hour 295e56d7b71SSascha Leib 296e56d7b71SSascha Leib if (visitor._type == BM_USERTYPE.KNOWN_BOT) { // known bots match by their bot ID: 297e56d7b71SSascha Leib 298e56d7b71SSascha Leib for (let i=0; i<model._visitors.length; i++) { 299e56d7b71SSascha Leib const v = model._visitors[i]; 300e56d7b71SSascha Leib 301e56d7b71SSascha Leib // bots match when their ID matches: 302e56d7b71SSascha Leib if (v._bot && v._bot.id == visitor._bot.id) { 303e56d7b71SSascha Leib return v; 304e56d7b71SSascha Leib } 305e56d7b71SSascha Leib } 306e56d7b71SSascha Leib } else { // other types match by their DW/PHPIDs: 307e56d7b71SSascha Leib 308e56d7b71SSascha Leib // loop over all visitors already registered and check for ID matches: 309e56d7b71SSascha Leib for (let i=0; i<model._visitors.length; i++) { 310e56d7b71SSascha Leib const v = model._visitors[i]; 311e56d7b71SSascha Leib 312e56d7b71SSascha Leib if ( v.id == visitor.id) { // match the DW/PHP IDs 313e56d7b71SSascha Leib return v; 314e56d7b71SSascha Leib } 315e56d7b71SSascha Leib } 316e56d7b71SSascha Leib 317e56d7b71SSascha Leib // if not found, try to match IP address and user agent: 318e56d7b71SSascha Leib for (let i=0; i<model._visitors.length; i++) { 319e56d7b71SSascha Leib const v = model._visitors[i]; 320e56d7b71SSascha Leib if ( v.ip == visitor.ip && v.agent == visitor.agent) { 321e56d7b71SSascha Leib return v; 322e56d7b71SSascha Leib } 323e56d7b71SSascha Leib } 324e56d7b71SSascha Leib } 325e56d7b71SSascha Leib 326e56d7b71SSascha Leib return null; // nothing found 327e56d7b71SSascha Leib }, 328e56d7b71SSascha Leib 329e56d7b71SSascha Leib /* if there is already this visit registered, return the page view item */ 330e56d7b71SSascha Leib _getPageView: function(visit, view) { 331e56d7b71SSascha Leib 332e56d7b71SSascha Leib // shortcut to make code more readable: 333e56d7b71SSascha Leib const model = BotMon.live.data.model; 334e56d7b71SSascha Leib 335e56d7b71SSascha Leib for (let i=0; i<visit._pageViews.length; i++) { 336e56d7b71SSascha Leib const pv = visit._pageViews[i]; 337e56d7b71SSascha Leib if (pv.pg == view.pg) { 338e56d7b71SSascha Leib return pv; 339e56d7b71SSascha Leib } 340e56d7b71SSascha Leib } 341e56d7b71SSascha Leib return null; // not found 342e56d7b71SSascha Leib }, 343e56d7b71SSascha Leib 344e56d7b71SSascha Leib // register a new visitor (or update if already exists) 345e56d7b71SSascha Leib registerVisit: function(nv, type) { 346e56d7b71SSascha Leib //console.info('registerVisit', nv, type); 347e56d7b71SSascha Leib 348e56d7b71SSascha Leib // shortcut to make code more readable: 349e56d7b71SSascha Leib const model = BotMon.live.data.model; 350e56d7b71SSascha Leib 351e56d7b71SSascha Leib // is it a known bot? 352e56d7b71SSascha Leib const bot = BotMon.live.data.bots.match(nv.agent); 353e56d7b71SSascha Leib 354e56d7b71SSascha Leib // enrich new visitor with relevant data: 355e56d7b71SSascha Leib if (!nv._bot) nv._bot = bot ?? null; // bot info 356e56d7b71SSascha Leib nv._type = ( bot ? BM_USERTYPE.KNOWN_BOT : ( nv.usr && nv.usr !== '' ? BM_USERTYPE.KNOWN_USER : BM_USERTYPE.UNKNOWN ) ); // user type 357e56d7b71SSascha Leib if (bot && bot.geo) { 358e56d7b71SSascha Leib if (!nv.geo || nv.geo == '' || nv.geo == 'ZZ') nv.geo = bot.geo; 359e56d7b71SSascha Leib } else if (!nv.geo ||nv.geo == '') { 360e56d7b71SSascha Leib nv.geo = 'ZZ'; 361e56d7b71SSascha Leib } 362e56d7b71SSascha Leib 363e56d7b71SSascha Leib // update first and last seen: 364e56d7b71SSascha Leib if (!nv._firstSeen) nv._firstSeen = nv.ts; // first-seen 365e56d7b71SSascha Leib nv._lastSeen = nv.ts; // last-seen 366e56d7b71SSascha Leib 367e56d7b71SSascha Leib // country name: 368e56d7b71SSascha Leib try { 369e56d7b71SSascha Leib nv._country = ( nv.geo == 'local' ? "localhost" : "Unknown" ); 370e56d7b71SSascha Leib if (nv.geo && nv.geo !== '' && nv.geo !== 'ZZ' && nv.geo !== 'local') { 371e56d7b71SSascha Leib const countryName = new Intl.DisplayNames(['en', BotMon._lang], {type: 'region'}); 372e56d7b71SSascha Leib nv._country = countryName.of(nv.geo.substring(0,2)) ?? nv.geo; 373e56d7b71SSascha Leib } 374e56d7b71SSascha Leib } catch (err) { 375e56d7b71SSascha Leib console.error(err); 376e56d7b71SSascha Leib nv._country = 'Error'; 377e56d7b71SSascha Leib } 378e56d7b71SSascha Leib 379e56d7b71SSascha Leib // check if it already exists: 380e56d7b71SSascha Leib let visitor = model.findVisitor(nv, type); 381e56d7b71SSascha Leib if (!visitor) { 382e56d7b71SSascha Leib visitor = nv; 383e56d7b71SSascha Leib visitor._seenBy = [type]; 384e56d7b71SSascha Leib visitor._pageViews = []; // array of page views 385e56d7b71SSascha Leib visitor._hasReferrer = false; // has at least one referrer 386e56d7b71SSascha Leib visitor._jsClient = false; // visitor has been seen logged by client js as well 387e56d7b71SSascha Leib visitor._client = BotMon.live.data.clients.match(nv.agent) ?? null; // client info 388e56d7b71SSascha Leib visitor._platform = BotMon.live.data.platforms.match(nv.agent); // platform info 389e56d7b71SSascha Leib model._visitors.push(visitor); 390e56d7b71SSascha Leib } else { // update existing 391e56d7b71SSascha Leib if (visitor._firstSeen > nv.ts) { 392e56d7b71SSascha Leib visitor._firstSeen = nv.ts; 393e56d7b71SSascha Leib } 394e56d7b71SSascha Leib } 395e56d7b71SSascha Leib 396e56d7b71SSascha Leib // find browser 397e56d7b71SSascha Leib 398e56d7b71SSascha Leib // is this visit already registered? 399e56d7b71SSascha Leib let prereg = model._getPageView(visitor, nv); 400e56d7b71SSascha Leib if (!prereg) { 401e56d7b71SSascha Leib // add new page view: 402e56d7b71SSascha Leib prereg = model._makePageView(nv, type); 403e56d7b71SSascha Leib visitor._pageViews.push(prereg); 404e56d7b71SSascha Leib } else { 405e56d7b71SSascha Leib // update last seen date 406e56d7b71SSascha Leib prereg._lastSeen = nv.ts; 407e56d7b71SSascha Leib // increase view count: 408e56d7b71SSascha Leib prereg._viewCount += 1; 409e56d7b71SSascha Leib prereg._tickCount += 1; 410e56d7b71SSascha Leib } 411e56d7b71SSascha Leib 412e56d7b71SSascha Leib // update referrer state: 413e56d7b71SSascha Leib visitor._hasReferrer = visitor._hasReferrer || 414e56d7b71SSascha Leib (prereg.ref !== undefined && prereg.ref !== ''); 415e56d7b71SSascha Leib 416e56d7b71SSascha Leib // update time stamp for last-seen: 417e56d7b71SSascha Leib if (visitor._lastSeen < nv.ts) { 418e56d7b71SSascha Leib visitor._lastSeen = nv.ts; 419e56d7b71SSascha Leib } 420e56d7b71SSascha Leib 421e56d7b71SSascha Leib // if needed: 422e56d7b71SSascha Leib return visitor; 423e56d7b71SSascha Leib }, 424e56d7b71SSascha Leib 425e56d7b71SSascha Leib // updating visit data from the client-side log: 426e56d7b71SSascha Leib updateVisit: function(dat) { 427e56d7b71SSascha Leib //console.info('updateVisit', dat); 428e56d7b71SSascha Leib 429e56d7b71SSascha Leib // shortcut to make code more readable: 430e56d7b71SSascha Leib const model = BotMon.live.data.model; 431e56d7b71SSascha Leib 432e56d7b71SSascha Leib const type = BM_LOGTYPE.CLIENT; 433e56d7b71SSascha Leib 434e56d7b71SSascha Leib let visitor = BotMon.live.data.model.findVisitor(dat, type); 435e56d7b71SSascha Leib if (!visitor) { 436e56d7b71SSascha Leib visitor = model.registerVisit(dat, type); 437e56d7b71SSascha Leib } 438e56d7b71SSascha Leib if (visitor) { 439e56d7b71SSascha Leib 440e56d7b71SSascha Leib if (visitor._lastSeen < dat.ts) { 441e56d7b71SSascha Leib visitor._lastSeen = dat.ts; 442e56d7b71SSascha Leib } 443e56d7b71SSascha Leib if (!visitor._seenBy.includes(type)) { 444e56d7b71SSascha Leib visitor._seenBy.push(type); 445e56d7b71SSascha Leib } 446e56d7b71SSascha Leib visitor._jsClient = true; // seen by client js 447e56d7b71SSascha Leib } 448e56d7b71SSascha Leib 449e56d7b71SSascha Leib // find the page view: 450e56d7b71SSascha Leib let prereg = BotMon.live.data.model._getPageView(visitor, dat); 451e56d7b71SSascha Leib if (prereg) { 452e56d7b71SSascha Leib // update the page view: 453e56d7b71SSascha Leib prereg._lastSeen = dat.ts; 454e56d7b71SSascha Leib if (!prereg._seenBy.includes(type)) prereg._seenBy.push(type); 455e56d7b71SSascha Leib prereg._jsClient = true; // seen by client js 456e56d7b71SSascha Leib } else { 457e56d7b71SSascha Leib // add the page view to the visitor: 458e56d7b71SSascha Leib prereg = model._makePageView(dat, type); 459e56d7b71SSascha Leib visitor._pageViews.push(prereg); 460e56d7b71SSascha Leib } 461e56d7b71SSascha Leib prereg._tickCount += 1; 462e56d7b71SSascha Leib }, 463e56d7b71SSascha Leib 464e56d7b71SSascha Leib // updating visit data from the ticker log: 465e56d7b71SSascha Leib updateTicks: function(dat) { 466e56d7b71SSascha Leib //console.info('updateTicks', dat); 467e56d7b71SSascha Leib 468e56d7b71SSascha Leib // shortcut to make code more readable: 469e56d7b71SSascha Leib const model = BotMon.live.data.model; 470e56d7b71SSascha Leib 471e56d7b71SSascha Leib const type = BM_LOGTYPE.TICKER; 472e56d7b71SSascha Leib 473e56d7b71SSascha Leib // find the visit info: 474e56d7b71SSascha Leib let visitor = model.findVisitor(dat, type); 475e56d7b71SSascha Leib if (!visitor) { 476e56d7b71SSascha Leib console.info(`No visitor with ID “${dat.id}” found, registering as a new one.`); 477e56d7b71SSascha Leib visitor = model.registerVisit(dat, type); 478e56d7b71SSascha Leib } 479e56d7b71SSascha Leib if (visitor) { 480e56d7b71SSascha Leib // update visitor: 481e56d7b71SSascha Leib if (visitor._lastSeen < dat.ts) visitor._lastSeen = dat.ts; 482e56d7b71SSascha Leib if (!visitor._seenBy.includes(type)) visitor._seenBy.push(type); 483e56d7b71SSascha Leib 484e56d7b71SSascha Leib // get the page view info: 485e56d7b71SSascha Leib let pv = model._getPageView(visitor, dat); 486e56d7b71SSascha Leib if (!pv) { 487e56d7b71SSascha Leib console.info(`No page view for visit ID “${dat.id}”, page “${dat.pg}”, registering a new one.`); 488e56d7b71SSascha Leib pv = model._makePageView(dat, type); 489e56d7b71SSascha Leib visitor._pageViews.push(pv); 490e56d7b71SSascha Leib } 491e56d7b71SSascha Leib 492e56d7b71SSascha Leib // update the page view info: 493e56d7b71SSascha Leib if (!pv._seenBy.includes(type)) pv._seenBy.push(type); 494e56d7b71SSascha Leib if (pv._lastSeen.getTime() < dat.ts.getTime()) pv._lastSeen = dat.ts; 495e56d7b71SSascha Leib pv._tickCount += 1; 496e56d7b71SSascha Leib 497e56d7b71SSascha Leib } 498e56d7b71SSascha Leib }, 499e56d7b71SSascha Leib 500e56d7b71SSascha Leib // helper function to create a new "page view" item: 501e56d7b71SSascha Leib _makePageView: function(data, type) { 502e56d7b71SSascha Leib // console.info('_makePageView', data); 503e56d7b71SSascha Leib 504e56d7b71SSascha Leib // try to parse the referrer: 505e56d7b71SSascha Leib let rUrl = null; 506e56d7b71SSascha Leib try { 507e56d7b71SSascha Leib rUrl = ( data.ref && data.ref !== '' ? new URL(data.ref) : null ); 508e56d7b71SSascha Leib } catch (e) { 509e56d7b71SSascha Leib console.warn(`Invalid referer: “${data.ref}”.`); 510e56d7b71SSascha Leib console.info(data); 511e56d7b71SSascha Leib } 512e56d7b71SSascha Leib 513e56d7b71SSascha Leib return { 514e56d7b71SSascha Leib _by: type, 515e56d7b71SSascha Leib ip: data.ip, 516e56d7b71SSascha Leib pg: data.pg, 517e56d7b71SSascha Leib lang: data.lang || '??', 518e56d7b71SSascha Leib _ref: rUrl, 519e56d7b71SSascha Leib _firstSeen: data.ts, 520e56d7b71SSascha Leib _lastSeen: data.ts, 521e56d7b71SSascha Leib _seenBy: [type], 522e56d7b71SSascha Leib _jsClient: ( type !== BM_LOGTYPE.SERVER), 523e56d7b71SSascha Leib _viewCount: 1, 524e56d7b71SSascha Leib _tickCount: 0 525e56d7b71SSascha Leib }; 526e56d7b71SSascha Leib } 527e56d7b71SSascha Leib }, 528e56d7b71SSascha Leib 529e56d7b71SSascha Leib // functions to analyse the data: 530e56d7b71SSascha Leib analytics: { 531e56d7b71SSascha Leib 532e56d7b71SSascha Leib /** 533e56d7b71SSascha Leib * Initializes the analytics data storage object: 534e56d7b71SSascha Leib */ 535e56d7b71SSascha Leib init: function() { 536e56d7b71SSascha Leib //console.info('BotMon.live.data.analytics.init()'); 537e56d7b71SSascha Leib }, 538e56d7b71SSascha Leib 539e56d7b71SSascha Leib // data storage: 540e56d7b71SSascha Leib data: { 541e56d7b71SSascha Leib totalVisits: 0, 542e56d7b71SSascha Leib totalPageViews: 0, 543e56d7b71SSascha Leib humanPageViews: 0, 544e56d7b71SSascha Leib bots: { 545e56d7b71SSascha Leib known: 0, 546e56d7b71SSascha Leib suspected: 0, 547e56d7b71SSascha Leib human: 0, 548e56d7b71SSascha Leib users: 0 549e56d7b71SSascha Leib } 550e56d7b71SSascha Leib }, 551e56d7b71SSascha Leib 552e56d7b71SSascha Leib // sort the visits by type: 553e56d7b71SSascha Leib groups: { 554e56d7b71SSascha Leib knownBots: [], 555e56d7b71SSascha Leib suspectedBots: [], 556e56d7b71SSascha Leib humans: [], 557e56d7b71SSascha Leib users: [] 558e56d7b71SSascha Leib }, 559e56d7b71SSascha Leib 560e56d7b71SSascha Leib // all analytics 561e56d7b71SSascha Leib analyseAll: function() { 562e56d7b71SSascha Leib //console.info('BotMon.live.data.analytics.analyseAll()'); 563e56d7b71SSascha Leib 564e56d7b71SSascha Leib // shortcut to make code more readable: 565e56d7b71SSascha Leib const model = BotMon.live.data.model; 566e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 567e56d7b71SSascha Leib 568e56d7b71SSascha Leib BotMon.live.gui.status.showBusy("Analysing data …"); 569e56d7b71SSascha Leib 570e56d7b71SSascha Leib // loop over all visitors: 571e56d7b71SSascha Leib model._visitors.forEach( (v) => { 572e56d7b71SSascha Leib 573e56d7b71SSascha Leib // count visits and page views: 574e56d7b71SSascha Leib this.data.totalVisits += 1; 575e56d7b71SSascha Leib this.data.totalPageViews += v._pageViews.length; 576e56d7b71SSascha Leib 577e56d7b71SSascha Leib // check for typical bot aspects: 578e56d7b71SSascha Leib let botScore = 0; 579e56d7b71SSascha Leib 580e56d7b71SSascha Leib if (v._type == BM_USERTYPE.KNOWN_BOT) { // known bots 581e56d7b71SSascha Leib 582e56d7b71SSascha Leib this.data.bots.known += v._pageViews.length; 583e56d7b71SSascha Leib this.groups.knownBots.push(v); 584e56d7b71SSascha Leib 585e56d7b71SSascha Leib } else if (v._type == BM_USERTYPE.KNOWN_USER) { // known users */ 586e56d7b71SSascha Leib 587e56d7b71SSascha Leib this.data.bots.users += v._pageViews.length; 588e56d7b71SSascha Leib this.groups.users.push(v); 589e56d7b71SSascha Leib 590e56d7b71SSascha Leib } else { 591e56d7b71SSascha Leib 592e56d7b71SSascha Leib // get evaluation: 593e56d7b71SSascha Leib const e = BotMon.live.data.rules.evaluate(v); 594e56d7b71SSascha Leib v._eval = e.rules; 595e56d7b71SSascha Leib v._botVal = e.val; 596e56d7b71SSascha Leib 597e56d7b71SSascha Leib if (e.isBot) { // likely bots 598e56d7b71SSascha Leib v._type = BM_USERTYPE.LIKELY_BOT; 599e56d7b71SSascha Leib this.data.bots.suspected += v._pageViews.length; 600e56d7b71SSascha Leib this.groups.suspectedBots.push(v); 601e56d7b71SSascha Leib } else { // probably humans 602e56d7b71SSascha Leib v._type = BM_USERTYPE.PROBABLY_HUMAN; 603e56d7b71SSascha Leib this.data.bots.human += v._pageViews.length; 604e56d7b71SSascha Leib this.groups.humans.push(v); 605e56d7b71SSascha Leib } 606e56d7b71SSascha Leib } 607e56d7b71SSascha Leib 608e56d7b71SSascha Leib // perform actions depending on the visitor type: 609e56d7b71SSascha Leib if (v._type == BM_USERTYPE.KNOWN_BOT ) { /* known bots only */ 610e56d7b71SSascha Leib 611e56d7b71SSascha Leib } else if (v._type == BM_USERTYPE.LIKELY_BOT) { /* probable bots only */ 612e56d7b71SSascha Leib 613e56d7b71SSascha Leib // add bot views to IP range information: 614e56d7b71SSascha Leib me.addToIpRanges(v); 615e56d7b71SSascha Leib 616e56d7b71SSascha Leib } else { /* humans only */ 617e56d7b71SSascha Leib 618e56d7b71SSascha Leib // add browser and platform statistics: 619e56d7b71SSascha Leib me.addBrowserPlatform(v); 620e56d7b71SSascha Leib 621e56d7b71SSascha Leib // add 622e56d7b71SSascha Leib v._pageViews.forEach( pv => { 623e56d7b71SSascha Leib me.addToRefererList(pv._ref); 624e56d7b71SSascha Leib me.addToPagesList(pv.pg); 625e56d7b71SSascha Leib }); 626e56d7b71SSascha Leib } 627e56d7b71SSascha Leib 628e56d7b71SSascha Leib // add to the country lists: 629e56d7b71SSascha Leib me.addToCountries(v.geo, v._country, v._type); 630e56d7b71SSascha Leib 631e56d7b71SSascha Leib }); 632e56d7b71SSascha Leib 633e56d7b71SSascha Leib BotMon.live.gui.status.hideBusy('Done.'); 634e56d7b71SSascha Leib }, 635e56d7b71SSascha Leib 636e56d7b71SSascha Leib // get a list of known bots: 637e56d7b71SSascha Leib getTopBots: function(max) { 638e56d7b71SSascha Leib //console.info('BotMon.live.data.analytics.getTopBots('+max+')'); 639e56d7b71SSascha Leib 640e56d7b71SSascha Leib //console.log(BotMon.live.data.analytics.groups.knownBots); 641e56d7b71SSascha Leib 642e56d7b71SSascha Leib let botsList = BotMon.live.data.analytics.groups.knownBots.toSorted( (a, b) => { 643e56d7b71SSascha Leib return b._pageViews.length - a._pageViews.length; 644e56d7b71SSascha Leib }); 645e56d7b71SSascha Leib 646e56d7b71SSascha Leib const other = { 647e56d7b71SSascha Leib 'id': 'other', 648e56d7b71SSascha Leib 'name': "Others", 649e56d7b71SSascha Leib 'count': 0 650e56d7b71SSascha Leib }; 651e56d7b71SSascha Leib 652e56d7b71SSascha Leib const rList = []; 653e56d7b71SSascha Leib const max2 = ( botsList.length > max ? max-1 : botsList.length ); 654e56d7b71SSascha Leib let total = 0; // adding up the items 655e56d7b71SSascha Leib for (let i=0; i<botsList.length; i++) { 656e56d7b71SSascha Leib const it = botsList[i]; 657e56d7b71SSascha Leib if (it && it._bot) { 658e56d7b71SSascha Leib if (i < max2) { 659e56d7b71SSascha Leib rList.push({ 660e56d7b71SSascha Leib id: it._bot.id, 661e56d7b71SSascha Leib name: (it._bot.n ? it._bot.n : it._bot.id), 662e56d7b71SSascha Leib count: it._pageViews.length 663e56d7b71SSascha Leib }); 664e56d7b71SSascha Leib } else { 665e56d7b71SSascha Leib other.count += it._pageViews.length; 666e56d7b71SSascha Leib }; 667e56d7b71SSascha Leib total += it._pageViews.length; 668e56d7b71SSascha Leib } 669e56d7b71SSascha Leib }; 670e56d7b71SSascha Leib 671e56d7b71SSascha Leib // add the "other" item, if needed: 672e56d7b71SSascha Leib if (botsList.length > max2) { 673e56d7b71SSascha Leib rList.push(other); 674e56d7b71SSascha Leib }; 675e56d7b71SSascha Leib 676e56d7b71SSascha Leib rList.forEach( it => { 677e56d7b71SSascha Leib it.pct = (it.count * 100 / total); 678e56d7b71SSascha Leib }); 679e56d7b71SSascha Leib 680e56d7b71SSascha Leib return rList; 681e56d7b71SSascha Leib }, 682e56d7b71SSascha Leib 683e56d7b71SSascha Leib // most visited pages list: 684e56d7b71SSascha Leib _pagesList: [], 685e56d7b71SSascha Leib 686e56d7b71SSascha Leib /** 687e56d7b71SSascha Leib * Add a page view to the list of most visited pages. 688e56d7b71SSascha Leib * @param {string} pageId - The page ID to add to the list. 689e56d7b71SSascha Leib * @example 690e56d7b71SSascha Leib * BotMon.live.data.analytics.addToPagesList('1234567890'); 691e56d7b71SSascha Leib */ 692e56d7b71SSascha Leib addToPagesList: function(pageId) { 693e56d7b71SSascha Leib //console.log('BotMon.live.data.analytics.addToPagesList', pageId); 694e56d7b71SSascha Leib 695e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 696e56d7b71SSascha Leib 697e56d7b71SSascha Leib // already exists? 698e56d7b71SSascha Leib let pgObj = null; 699e56d7b71SSascha Leib for (let i = 0; i < me._pagesList.length; i++) { 700e56d7b71SSascha Leib if (me._pagesList[i].id == pageId) { 701e56d7b71SSascha Leib pgObj = me._pagesList[i]; 702e56d7b71SSascha Leib break; 703e56d7b71SSascha Leib } 704e56d7b71SSascha Leib } 705e56d7b71SSascha Leib 706e56d7b71SSascha Leib // if not exists, create it: 707e56d7b71SSascha Leib if (!pgObj) { 708e56d7b71SSascha Leib pgObj = { 709e56d7b71SSascha Leib id: pageId, 710e56d7b71SSascha Leib count: 1 711e56d7b71SSascha Leib }; 712e56d7b71SSascha Leib me._pagesList.push(pgObj); 713e56d7b71SSascha Leib } else { 714e56d7b71SSascha Leib pgObj.count += 1; 715e56d7b71SSascha Leib } 716e56d7b71SSascha Leib }, 717e56d7b71SSascha Leib 718e56d7b71SSascha Leib getTopPages: function(max) { 719e56d7b71SSascha Leib //console.info('BotMon.live.data.analytics.getTopPages('+max+')'); 720e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 721e56d7b71SSascha Leib return me._pagesList.toSorted( (a, b) => { 722e56d7b71SSascha Leib return b.count - a.count; 723e56d7b71SSascha Leib }).slice(0,max); 724e56d7b71SSascha Leib }, 725e56d7b71SSascha Leib 726e56d7b71SSascha Leib // Referer List: 727e56d7b71SSascha Leib _refererList: [], 728e56d7b71SSascha Leib 729e56d7b71SSascha Leib addToRefererList: function(ref) { 730e56d7b71SSascha Leib //console.log('BotMon.live.data.analytics.addToRefererList',ref); 731e56d7b71SSascha Leib 732e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 733e56d7b71SSascha Leib 734e56d7b71SSascha Leib // ignore internal references: 735e56d7b71SSascha Leib if (ref && ref.host == window.location.host) { 736e56d7b71SSascha Leib return; 737e56d7b71SSascha Leib } 738e56d7b71SSascha Leib 739e56d7b71SSascha Leib const refInfo = me.getRefererInfo(ref); 740e56d7b71SSascha Leib 741e56d7b71SSascha Leib // already exists? 742e56d7b71SSascha Leib let refObj = null; 743e56d7b71SSascha Leib for (let i = 0; i < me._refererList.length; i++) { 744e56d7b71SSascha Leib if (me._refererList[i].id == refInfo.id) { 745e56d7b71SSascha Leib refObj = me._refererList[i]; 746e56d7b71SSascha Leib break; 747e56d7b71SSascha Leib } 748e56d7b71SSascha Leib } 749e56d7b71SSascha Leib 750e56d7b71SSascha Leib // if not exists, create it: 751e56d7b71SSascha Leib if (!refObj) { 752e56d7b71SSascha Leib refObj = refInfo; 753e56d7b71SSascha Leib refObj.count = 1; 754e56d7b71SSascha Leib me._refererList.push(refObj); 755e56d7b71SSascha Leib } else { 756e56d7b71SSascha Leib refObj.count += 1; 757e56d7b71SSascha Leib } 758e56d7b71SSascha Leib }, 759e56d7b71SSascha Leib 760e56d7b71SSascha Leib getRefererInfo: function(url) { 761e56d7b71SSascha Leib //console.log('BotMon.live.data.analytics.getRefererInfo',url); 762e56d7b71SSascha Leib try { 763e56d7b71SSascha Leib url = new URL(url); 764e56d7b71SSascha Leib } catch (e) { 765e56d7b71SSascha Leib return { 766e56d7b71SSascha Leib 'id': 'null', 767e56d7b71SSascha Leib 'n': 'Invalid Referer' 768e56d7b71SSascha Leib }; 769e56d7b71SSascha Leib } 770e56d7b71SSascha Leib 771e56d7b71SSascha Leib // find the referer ID: 772e56d7b71SSascha Leib let refId = 'null'; 773e56d7b71SSascha Leib let refName = 'No Referer'; 774e56d7b71SSascha Leib if (url && url.host) { 775e56d7b71SSascha Leib const hArr = url.host.split('.'); 776e56d7b71SSascha Leib const tld = hArr[hArr.length-1]; 777e56d7b71SSascha Leib refId = ( tld == 'localhost' ? tld : hArr[hArr.length-2]); 778e56d7b71SSascha Leib refName = hArr[hArr.length-2] + '.' + tld; 779e56d7b71SSascha Leib } 780e56d7b71SSascha Leib 781e56d7b71SSascha Leib return { 782e56d7b71SSascha Leib 'id': refId, 783e56d7b71SSascha Leib 'n': refName 784e56d7b71SSascha Leib }; 785e56d7b71SSascha Leib }, 786e56d7b71SSascha Leib 787e56d7b71SSascha Leib /** 788e56d7b71SSascha Leib * Get a sorted list of the top referers. 789e56d7b71SSascha Leib * The list is sorted in descending order of count. 790e56d7b71SSascha Leib * If the array has more items than the given maximum, the rest of the items are added to an "other" item. 791e56d7b71SSascha Leib * Each item in the list has a "pct" property, which is the percentage of the total count. 792e56d7b71SSascha Leib * @param {number} max - The maximum number of items to return. 793e56d7b71SSascha Leib * @return {Array} The sorted list of top referers. 794e56d7b71SSascha Leib */ 795e56d7b71SSascha Leib getTopReferers: function(max) { 796e56d7b71SSascha Leib //console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')')); 797e56d7b71SSascha Leib 798e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 799e56d7b71SSascha Leib 800e56d7b71SSascha Leib return me._makeTopList(me._refererList, max); 801e56d7b71SSascha Leib }, 802e56d7b71SSascha Leib 803e56d7b71SSascha Leib /** 804e56d7b71SSascha Leib * Create a sorted list of top items from a given array. 805e56d7b71SSascha Leib * The list is sorted in descending order of count. 806e56d7b71SSascha Leib * If the array has more items than the given maximum, the rest of the items are added to an "other" item. 807e56d7b71SSascha Leib * Each item in the list has a "pct" property, which is the percentage of the total count. 808e56d7b71SSascha Leib * @param {Array} arr - The array to sort and truncate. 809e56d7b71SSascha Leib * @param {number} max - The maximum number of items to return. 810e56d7b71SSascha Leib * @return {Array} The sorted list of top items. 811e56d7b71SSascha Leib */ 812e56d7b71SSascha Leib _makeTopList: function(arr, max) { 813e56d7b71SSascha Leib //console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')')); 814e56d7b71SSascha Leib 815e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 816e56d7b71SSascha Leib 817e56d7b71SSascha Leib // sort the list: 818e56d7b71SSascha Leib arr.sort( (a,b) => { 819e56d7b71SSascha Leib return b.count - a.count; 820e56d7b71SSascha Leib }); 821e56d7b71SSascha Leib 822e56d7b71SSascha Leib const rList = []; // return array 823e56d7b71SSascha Leib const max2 = ( arr.length >= max ? max-1 : arr.length ); 824e56d7b71SSascha Leib const other = { 825e56d7b71SSascha Leib 'id': 'other', 826e56d7b71SSascha Leib 'name': "Others", 827*20997f50SSascha Leib 'typ': 'other', 828e56d7b71SSascha Leib 'count': 0 829e56d7b71SSascha Leib }; 830e56d7b71SSascha Leib let total = 0; // adding up the items 831e56d7b71SSascha Leib for (let i=0; Math.min(max, arr.length) > i; i++) { 832e56d7b71SSascha Leib const it = arr[i]; 833e56d7b71SSascha Leib if (it) { 834e56d7b71SSascha Leib if (i < max2) { 835e56d7b71SSascha Leib const rIt = { 836e56d7b71SSascha Leib id: it.id, 837e56d7b71SSascha Leib name: (it.n ? it.n : it.id), 838*20997f50SSascha Leib typ: it.typ || it.id, 839e56d7b71SSascha Leib count: it.count 840e56d7b71SSascha Leib }; 841e56d7b71SSascha Leib rList.push(rIt); 842e56d7b71SSascha Leib } else { 843e56d7b71SSascha Leib other.count += it.count; 844e56d7b71SSascha Leib } 845e56d7b71SSascha Leib total += it.count; 846e56d7b71SSascha Leib } 847e56d7b71SSascha Leib } 848e56d7b71SSascha Leib 849e56d7b71SSascha Leib // add the "other" item, if needed: 850e56d7b71SSascha Leib if (arr.length > max2) { 851e56d7b71SSascha Leib rList.push(other); 852e56d7b71SSascha Leib }; 853e56d7b71SSascha Leib 854e56d7b71SSascha Leib rList.forEach( it => { 855e56d7b71SSascha Leib it.pct = (it.count * 100 / total); 856e56d7b71SSascha Leib }); 857e56d7b71SSascha Leib 858e56d7b71SSascha Leib return rList; 859e56d7b71SSascha Leib }, 860e56d7b71SSascha Leib 861e56d7b71SSascha Leib /* countries of visits */ 862e56d7b71SSascha Leib _countries: { 863e56d7b71SSascha Leib 'human': [], 864e56d7b71SSascha Leib 'bot': [] 865e56d7b71SSascha Leib }, 866e56d7b71SSascha Leib /** 867e56d7b71SSascha Leib * Adds a country code to the statistics. 868e56d7b71SSascha Leib * 869e56d7b71SSascha Leib * @param {string} iso The ISO 3166-1 alpha-2 country code. 870e56d7b71SSascha Leib */ 871e56d7b71SSascha Leib addToCountries: function(iso, name, type) { 872e56d7b71SSascha Leib 873e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 874e56d7b71SSascha Leib 875e56d7b71SSascha Leib // find the correct array: 876e56d7b71SSascha Leib let arr = null; 877e56d7b71SSascha Leib switch (type) { 878e56d7b71SSascha Leib 879e56d7b71SSascha Leib case BM_USERTYPE.KNOWN_USER: 880e56d7b71SSascha Leib case BM_USERTYPE.PROBABLY_HUMAN: 881e56d7b71SSascha Leib arr = me._countries.human; 882e56d7b71SSascha Leib break; 883e56d7b71SSascha Leib case BM_USERTYPE.LIKELY_BOT: 884e56d7b71SSascha Leib case BM_USERTYPE.KNOWN_BOT: 885e56d7b71SSascha Leib arr = me._countries.bot; 886e56d7b71SSascha Leib break; 887e56d7b71SSascha Leib default: 888e56d7b71SSascha Leib console.warn(`Unknown user type ${type} in function addToCountries.`); 889e56d7b71SSascha Leib } 890e56d7b71SSascha Leib 891e56d7b71SSascha Leib if (arr) { 892e56d7b71SSascha Leib let cRec = arr.find( it => it.id == iso); 893e56d7b71SSascha Leib if (!cRec) { 894e56d7b71SSascha Leib cRec = { 895e56d7b71SSascha Leib 'id': iso, 896e56d7b71SSascha Leib 'n': name, 897e56d7b71SSascha Leib 'count': 1 898e56d7b71SSascha Leib }; 899e56d7b71SSascha Leib arr.push(cRec); 900e56d7b71SSascha Leib } else { 901e56d7b71SSascha Leib cRec.count += 1; 902e56d7b71SSascha Leib } 903e56d7b71SSascha Leib } 904e56d7b71SSascha Leib }, 905e56d7b71SSascha Leib 906e56d7b71SSascha Leib /** 907e56d7b71SSascha Leib * Returns a list of countries with visit counts, sorted by visit count in descending order. 908e56d7b71SSascha Leib * 909e56d7b71SSascha Leib * @param {BM_USERTYPE} type array of types type of visitors to return. 910e56d7b71SSascha Leib * @param {number} max The maximum number of entries to return. 911e56d7b71SSascha Leib * @return {Array} A list of objects with properties 'iso' (ISO 3166-1 alpha-2 country code) and 'count' (visit count). 912e56d7b71SSascha Leib */ 913e56d7b71SSascha Leib getCountryList: function(type, max) { 914e56d7b71SSascha Leib 915e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 916e56d7b71SSascha Leib 917e56d7b71SSascha Leib // find the correct array: 918e56d7b71SSascha Leib let arr = null; 919e56d7b71SSascha Leib switch (type) { 920e56d7b71SSascha Leib 921e56d7b71SSascha Leib case 'human': 922e56d7b71SSascha Leib arr = me._countries.human; 923e56d7b71SSascha Leib break; 924e56d7b71SSascha Leib case 'bot': 925e56d7b71SSascha Leib arr = me._countries.bot; 926e56d7b71SSascha Leib break; 927e56d7b71SSascha Leib default: 928e56d7b71SSascha Leib console.warn(`Unknown user type ${type} in function getCountryList.`); 929e56d7b71SSascha Leib return; 930e56d7b71SSascha Leib } 931e56d7b71SSascha Leib 932e56d7b71SSascha Leib return me._makeTopList(arr, max); 933e56d7b71SSascha Leib }, 934e56d7b71SSascha Leib 935e56d7b71SSascha Leib /* browser and platform of human visitors */ 936e56d7b71SSascha Leib _browsers: [], 937e56d7b71SSascha Leib _platforms: [], 938e56d7b71SSascha Leib 939e56d7b71SSascha Leib addBrowserPlatform: function(visitor) { 940e56d7b71SSascha Leib //console.info('addBrowserPlatform', visitor); 941e56d7b71SSascha Leib 942e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 943e56d7b71SSascha Leib 944e56d7b71SSascha Leib // add to browsers list: 945e56d7b71SSascha Leib let browserRec = ( visitor._client ? visitor._client : {'id': 'unknown'}); 946e56d7b71SSascha Leib if (visitor._client) { 947e56d7b71SSascha Leib let bRec = me._browsers.find( it => it.id == browserRec.id); 948e56d7b71SSascha Leib if (!bRec) { 949e56d7b71SSascha Leib bRec = { 950e56d7b71SSascha Leib id: browserRec.id, 951e56d7b71SSascha Leib n: browserRec.n, 952e56d7b71SSascha Leib count: 1 953e56d7b71SSascha Leib }; 954e56d7b71SSascha Leib me._browsers.push(bRec); 955e56d7b71SSascha Leib } else { 956e56d7b71SSascha Leib bRec.count += 1; 957e56d7b71SSascha Leib } 958e56d7b71SSascha Leib } 959e56d7b71SSascha Leib 960e56d7b71SSascha Leib // add to platforms list: 961e56d7b71SSascha Leib let platformRec = ( visitor._platform ? visitor._platform : {'id': 'unknown'}); 962e56d7b71SSascha Leib if (visitor._platform) { 963e56d7b71SSascha Leib let pRec = me._platforms.find( it => it.id == platformRec.id); 964e56d7b71SSascha Leib if (!pRec) { 965e56d7b71SSascha Leib pRec = { 966e56d7b71SSascha Leib id: platformRec.id, 967e56d7b71SSascha Leib n: platformRec.n, 968e56d7b71SSascha Leib count: 1 969e56d7b71SSascha Leib }; 970e56d7b71SSascha Leib me._platforms.push(pRec); 971e56d7b71SSascha Leib } else { 972e56d7b71SSascha Leib pRec.count += 1; 973e56d7b71SSascha Leib } 974e56d7b71SSascha Leib } 975e56d7b71SSascha Leib 976e56d7b71SSascha Leib }, 977e56d7b71SSascha Leib 978e56d7b71SSascha Leib getTopBrowsers: function(max) { 979e56d7b71SSascha Leib 980e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 981e56d7b71SSascha Leib 982e56d7b71SSascha Leib return me._makeTopList(me._browsers, max); 983e56d7b71SSascha Leib }, 984e56d7b71SSascha Leib 985e56d7b71SSascha Leib getTopPlatforms: function(max) { 986e56d7b71SSascha Leib 987e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 988e56d7b71SSascha Leib 989e56d7b71SSascha Leib return me._makeTopList(me._platforms, max); 990e56d7b71SSascha Leib }, 991e56d7b71SSascha Leib 992e56d7b71SSascha Leib /* bounces are counted, not calculates: */ 993e56d7b71SSascha Leib getBounceCount: function(type) { 994e56d7b71SSascha Leib 995e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 996e56d7b71SSascha Leib var bounces = 0; 997e56d7b71SSascha Leib const list = me.groups[type]; 998e56d7b71SSascha Leib 999e56d7b71SSascha Leib list.forEach(it => { 1000e56d7b71SSascha Leib bounces += (it._pageViews.length <= 1 ? 1 : 0); 1001e56d7b71SSascha Leib }); 1002e56d7b71SSascha Leib 1003e56d7b71SSascha Leib return bounces; 1004e56d7b71SSascha Leib }, 1005e56d7b71SSascha Leib 1006*20997f50SSascha Leib _ipRanges: [], 1007e56d7b71SSascha Leib 1008e56d7b71SSascha Leib /* adds a visit to the ip ranges arrays */ 1009e56d7b71SSascha Leib addToIpRanges: function(v) { 1010e56d7b71SSascha Leib //console.info('addToIpRanges', v.ip); 1011e56d7b71SSascha Leib 1012e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 1013e56d7b71SSascha Leib const ipRanges = BotMon.live.data.ipRanges; 1014e56d7b71SSascha Leib 10150edf1a56SSascha Leib // Number of IP address segments to look at: 10160edf1a56SSascha Leib const kIP4Segments = 1; 1017*20997f50SSascha Leib const kIP6Segments = 3; 10180edf1a56SSascha Leib 1019*20997f50SSascha Leib let ipGroup = ''; // group name 1020*20997f50SSascha Leib let ipSeg = []; // IP segment array 1021*20997f50SSascha Leib let rawIP = ''; // raw ip prefix 1022*20997f50SSascha Leib let ipName = ''; // IP group display name 1023*20997f50SSascha Leib 1024*20997f50SSascha Leib const ipType = v.ip.indexOf(':') > 0 ? BM_IPVERSION.IPv6 : BM_IPVERSION.IPv4; 1025*20997f50SSascha Leib const ipAddr = BotMon.t._ip2Num(v.ip); 1026e56d7b71SSascha Leib 1027e56d7b71SSascha Leib // is there already a known IP range assigned? 1028e56d7b71SSascha Leib if (v._ipRange) { 1029*20997f50SSascha Leib 1030*20997f50SSascha Leib ipGroup = v._ipRange.g; // group name 1031*20997f50SSascha Leib ipName = ipRanges.getOwner( v._ipRange.g ) || "Unknown"; 1032*20997f50SSascha Leib 1033*20997f50SSascha Leib } else { // no known IP range, let's collect necessary information: 1034*20997f50SSascha Leib 1035*20997f50SSascha Leib 1036*20997f50SSascha Leib // collect basic IP address info: 1037*20997f50SSascha Leib if (ipType == BM_IPVERSION.IPv6) { 1038*20997f50SSascha Leib ipSeg = ipAddr.split(':'); 1039*20997f50SSascha Leib const prefix = v.ip.split(':').slice(0, kIP6Segments).join(':'); 1040*20997f50SSascha Leib rawIP = ipSeg.slice(0, kIP6Segments).join(':'); 1041*20997f50SSascha Leib ipGroup = 'ip6-' + rawIP.replaceAll(':', '-'); 1042*20997f50SSascha Leib ipName = prefix + '::' + '/' + (16 * kIP6Segments); 1043*20997f50SSascha Leib } else { 1044*20997f50SSascha Leib ipSeg = ipAddr.split('.'); 1045*20997f50SSascha Leib const prefix = v.ip.split('.').slice(0, kIP4Segments).join('.'); 1046*20997f50SSascha Leib rawIP = ipSeg.slice(0, kIP4Segments).join('.') ; 1047*20997f50SSascha Leib ipGroup = 'ip4-' + rawIP.replaceAll('.', '-'); 1048*20997f50SSascha Leib ipName = prefix + '.0.0.0'.substring(0, 1+(4-kIP4Segments)*2) + '/' + (8 * kIP4Segments); 1049*20997f50SSascha Leib } 1050e56d7b71SSascha Leib } 1051e56d7b71SSascha Leib 1052*20997f50SSascha Leib // check if record already exists: 1053*20997f50SSascha Leib let ipRec = me._ipRanges.find( it => it.g == ipGroup); 1054*20997f50SSascha Leib if (!ipRec) { 1055*20997f50SSascha Leib 1056*20997f50SSascha Leib // ip info record initialised: 1057*20997f50SSascha Leib ipRec = { 1058*20997f50SSascha Leib g: ipGroup, 1059*20997f50SSascha Leib n: ipName, 1060*20997f50SSascha Leib count: 0 1061e56d7b71SSascha Leib } 1062*20997f50SSascha Leib 1063*20997f50SSascha Leib // existing record? 1064*20997f50SSascha Leib if (v._ipRange) { 1065*20997f50SSascha Leib 1066*20997f50SSascha Leib ipRec.from = v._ipRange.from; 1067*20997f50SSascha Leib ipRec.to = v._ipRange.to; 1068*20997f50SSascha Leib ipRec.typ = 'net'; 1069*20997f50SSascha Leib 1070*20997f50SSascha Leib } else { // no known IP range, let's collect necessary information: 1071*20997f50SSascha Leib 1072*20997f50SSascha Leib // complete the ip info record: 1073*20997f50SSascha Leib if (ipType == BM_IPVERSION.IPv6) { 1074*20997f50SSascha Leib ipRec.from = rawIP + ':0000:0000:0000:0000:0000:0000:0000'.substring(0, (8-kIP6Segments)*5); 1075*20997f50SSascha Leib ipRec.to = rawIP + ':FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF'.substring(0, (8-kIP6Segments)*5); 1076*20997f50SSascha Leib ipRec.typ = '6'; 1077*20997f50SSascha Leib } else { 1078*20997f50SSascha Leib ipRec.from = rawIP + '.000.000.000.000'.substring(0, (4-kIP4Segments)*4); 1079*20997f50SSascha Leib ipRec.to = rawIP + '.255.255.255.254'.substring(0, (4-kIP4Segments)*4); 1080*20997f50SSascha Leib ipRec.typ = '4'; 1081*20997f50SSascha Leib } 1082*20997f50SSascha Leib } 1083*20997f50SSascha Leib 1084*20997f50SSascha Leib me._ipRanges.push(ipRec); 1085*20997f50SSascha Leib } 1086*20997f50SSascha Leib 1087*20997f50SSascha Leib // add to counter: 1088*20997f50SSascha Leib ipRec.count += v._pageViews.length; 1089*20997f50SSascha Leib 1090e56d7b71SSascha Leib }, 1091e56d7b71SSascha Leib 1092e56d7b71SSascha Leib getTopBotISPs: function(max) { 1093e56d7b71SSascha Leib 1094e56d7b71SSascha Leib const me = BotMon.live.data.analytics; 1095e56d7b71SSascha Leib 1096*20997f50SSascha Leib return me._makeTopList(me._ipRanges, max); 1097e56d7b71SSascha Leib }, 1098e56d7b71SSascha Leib }, 1099e56d7b71SSascha Leib 1100e56d7b71SSascha Leib // information on "known bots": 1101e56d7b71SSascha Leib bots: { 1102e56d7b71SSascha Leib // loads the list of known bots from a JSON file: 1103e56d7b71SSascha Leib init: async function() { 1104e56d7b71SSascha Leib //console.info('BotMon.live.data.bots.init()'); 1105e56d7b71SSascha Leib 1106e56d7b71SSascha Leib // Load the list of known bots: 1107e56d7b71SSascha Leib BotMon.live.gui.status.showBusy("Loading known bots …"); 1108e56d7b71SSascha Leib const url = BotMon._baseDir + 'config/known-bots.json'; 1109e56d7b71SSascha Leib try { 1110e56d7b71SSascha Leib const response = await fetch(url); 1111e56d7b71SSascha Leib if (!response.ok) { 1112e56d7b71SSascha Leib throw new Error(`${response.status} ${response.statusText}`); 1113e56d7b71SSascha Leib } 1114e56d7b71SSascha Leib 1115e56d7b71SSascha Leib this._list = await response.json(); 1116e56d7b71SSascha Leib this._ready = true; 1117e56d7b71SSascha Leib 1118e56d7b71SSascha Leib } catch (error) { 1119e56d7b71SSascha Leib BotMon.live.gui.status.setError("Error while loading the known bots file:", error.message); 1120e56d7b71SSascha Leib } finally { 1121e56d7b71SSascha Leib BotMon.live.gui.status.hideBusy("Status: Done."); 1122e56d7b71SSascha Leib BotMon.live.data._dispatch('bots') 1123e56d7b71SSascha Leib } 1124e56d7b71SSascha Leib }, 1125e56d7b71SSascha Leib 1126e56d7b71SSascha Leib // returns bot info if the clientId matches a known bot, null otherwise: 1127e56d7b71SSascha Leib match: function(agent) { 1128e56d7b71SSascha Leib //console.info('BotMon.live.data.bots.match(',agent,')'); 1129e56d7b71SSascha Leib 1130e56d7b71SSascha Leib const BotList = BotMon.live.data.bots._list; 1131e56d7b71SSascha Leib 1132e56d7b71SSascha Leib // default is: not found! 1133e56d7b71SSascha Leib let botInfo = null; 1134e56d7b71SSascha Leib 1135e56d7b71SSascha Leib if (!agent) return null; 1136e56d7b71SSascha Leib 1137e56d7b71SSascha Leib // check for known bots: 1138e56d7b71SSascha Leib BotList.find(bot => { 1139e56d7b71SSascha Leib let r = false; 1140e56d7b71SSascha Leib for (let j=0; j<bot.rx.length; j++) { 1141e56d7b71SSascha Leib const rxr = agent.match(new RegExp(bot.rx[j])); 1142e56d7b71SSascha Leib if (rxr) { 1143e56d7b71SSascha Leib botInfo = { 1144e56d7b71SSascha Leib n : bot.n, 1145e56d7b71SSascha Leib id: bot.id, 1146e56d7b71SSascha Leib geo: (bot.geo ? bot.geo : null), 1147e56d7b71SSascha Leib url: bot.url, 1148e56d7b71SSascha Leib v: (rxr.length > 1 ? rxr[1] : -1) 1149e56d7b71SSascha Leib }; 1150e56d7b71SSascha Leib r = true; 1151e56d7b71SSascha Leib break; 1152e56d7b71SSascha Leib }; 1153e56d7b71SSascha Leib }; 1154e56d7b71SSascha Leib return r; 1155e56d7b71SSascha Leib }); 1156e56d7b71SSascha Leib 1157e56d7b71SSascha Leib // check for unknown bots: 1158e56d7b71SSascha Leib if (!botInfo) { 1159e56d7b71SSascha Leib const botmatch = agent.match(/([\s\d\w\-]*bot|[\s\d\w\-]*crawler|[\s\d\w\-]*spider)[\/\s;\),\\.$]/i); 1160e56d7b71SSascha Leib if(botmatch) { 1161e56d7b71SSascha Leib botInfo = {'id': ( botmatch[1] || "other_" ), 'n': "Other" + ( botmatch[1] ? " (" + botmatch[1] + ")" : "" ) , "bot": botmatch[1] }; 1162e56d7b71SSascha Leib } 1163e56d7b71SSascha Leib } 1164e56d7b71SSascha Leib 1165e56d7b71SSascha Leib //console.log("botInfo:", botInfo); 1166e56d7b71SSascha Leib return botInfo; 1167e56d7b71SSascha Leib }, 1168e56d7b71SSascha Leib 1169e56d7b71SSascha Leib 1170e56d7b71SSascha Leib // indicates if the list is loaded and ready to use: 1171e56d7b71SSascha Leib _ready: false, 1172e56d7b71SSascha Leib 1173e56d7b71SSascha Leib // the actual bot list is stored here: 1174e56d7b71SSascha Leib _list: [] 1175e56d7b71SSascha Leib }, 1176e56d7b71SSascha Leib 1177e56d7b71SSascha Leib // information on known clients (browsers): 1178e56d7b71SSascha Leib clients: { 1179e56d7b71SSascha Leib // loads the list of known clients from a JSON file: 1180e56d7b71SSascha Leib init: async function() { 1181e56d7b71SSascha Leib //console.info('BotMon.live.data.clients.init()'); 1182e56d7b71SSascha Leib 1183e56d7b71SSascha Leib // Load the list of known bots: 1184e56d7b71SSascha Leib BotMon.live.gui.status.showBusy("Loading known clients"); 1185e56d7b71SSascha Leib const url = BotMon._baseDir + 'config/known-clients.json'; 1186e56d7b71SSascha Leib try { 1187e56d7b71SSascha Leib const response = await fetch(url); 1188e56d7b71SSascha Leib if (!response.ok) { 1189e56d7b71SSascha Leib throw new Error(`${response.status} ${response.statusText}`); 1190e56d7b71SSascha Leib } 1191e56d7b71SSascha Leib 1192e56d7b71SSascha Leib BotMon.live.data.clients._list = await response.json(); 1193e56d7b71SSascha Leib BotMon.live.data.clients._ready = true; 1194e56d7b71SSascha Leib 1195e56d7b71SSascha Leib } catch (error) { 1196e56d7b71SSascha Leib BotMon.live.gui.status.setError("Error while loading the known clients file: " + error.message); 1197e56d7b71SSascha Leib } finally { 1198e56d7b71SSascha Leib BotMon.live.gui.status.hideBusy("Status: Done."); 1199e56d7b71SSascha Leib BotMon.live.data._dispatch('clients') 1200e56d7b71SSascha Leib } 1201e56d7b71SSascha Leib }, 1202e56d7b71SSascha Leib 1203e56d7b71SSascha Leib // returns bot info if the user-agent matches a known bot, null otherwise: 1204e56d7b71SSascha Leib match: function(agent) { 1205e56d7b71SSascha Leib //console.info('BotMon.live.data.clients.match(',agent,')'); 1206e56d7b71SSascha Leib 1207e56d7b71SSascha Leib let match = {"n": "Unknown", "v": -1, "id": 'null'}; 1208e56d7b71SSascha Leib 1209e56d7b71SSascha Leib if (agent) { 1210e56d7b71SSascha Leib BotMon.live.data.clients._list.find(client => { 1211e56d7b71SSascha Leib let r = false; 1212e56d7b71SSascha Leib for (let j=0; j<client.rx.length; j++) { 1213e56d7b71SSascha Leib const rxr = agent.match(new RegExp(client.rx[j])); 1214e56d7b71SSascha Leib if (rxr) { 1215e56d7b71SSascha Leib match.n = client.n; 1216e56d7b71SSascha Leib match.v = (rxr.length > 1 ? rxr[1] : -1); 1217e56d7b71SSascha Leib match.id = client.id || null; 1218e56d7b71SSascha Leib r = true; 1219e56d7b71SSascha Leib break; 1220e56d7b71SSascha Leib } 1221e56d7b71SSascha Leib } 1222e56d7b71SSascha Leib return r; 1223e56d7b71SSascha Leib }); 1224e56d7b71SSascha Leib } 1225e56d7b71SSascha Leib 1226e56d7b71SSascha Leib //console.log(match) 1227e56d7b71SSascha Leib return match; 1228e56d7b71SSascha Leib }, 1229e56d7b71SSascha Leib 1230e56d7b71SSascha Leib // return the browser name for a browser ID: 1231e56d7b71SSascha Leib getName: function(id) { 1232e56d7b71SSascha Leib const it = BotMon.live.data.clients._list.find(client => client.id == id); 1233e56d7b71SSascha Leib return ( it && it.n ? it.n : "Unknown"); //it.n; 1234e56d7b71SSascha Leib }, 1235e56d7b71SSascha Leib 1236e56d7b71SSascha Leib // indicates if the list is loaded and ready to use: 1237e56d7b71SSascha Leib _ready: false, 1238e56d7b71SSascha Leib 1239e56d7b71SSascha Leib // the actual bot list is stored here: 1240e56d7b71SSascha Leib _list: [] 1241e56d7b71SSascha Leib 1242e56d7b71SSascha Leib }, 1243e56d7b71SSascha Leib 1244e56d7b71SSascha Leib // information on known platforms (operating systems): 1245e56d7b71SSascha Leib platforms: { 1246e56d7b71SSascha Leib // loads the list of known platforms from a JSON file: 1247e56d7b71SSascha Leib init: async function() { 1248e56d7b71SSascha Leib //console.info('BotMon.live.data.platforms.init()'); 1249e56d7b71SSascha Leib 1250e56d7b71SSascha Leib // Load the list of known bots: 1251e56d7b71SSascha Leib BotMon.live.gui.status.showBusy("Loading known platforms"); 1252e56d7b71SSascha Leib const url = BotMon._baseDir + 'config/known-platforms.json'; 1253e56d7b71SSascha Leib try { 1254e56d7b71SSascha Leib const response = await fetch(url); 1255e56d7b71SSascha Leib if (!response.ok) { 1256e56d7b71SSascha Leib throw new Error(`${response.status} ${response.statusText}`); 1257e56d7b71SSascha Leib } 1258e56d7b71SSascha Leib 1259e56d7b71SSascha Leib BotMon.live.data.platforms._list = await response.json(); 1260e56d7b71SSascha Leib BotMon.live.data.platforms._ready = true; 1261e56d7b71SSascha Leib 1262e56d7b71SSascha Leib } catch (error) { 1263e56d7b71SSascha Leib BotMon.live.gui.status.setError("Error while loading the known platforms file: " + error.message); 1264e56d7b71SSascha Leib } finally { 1265e56d7b71SSascha Leib BotMon.live.gui.status.hideBusy("Status: Done."); 1266e56d7b71SSascha Leib BotMon.live.data._dispatch('platforms') 1267e56d7b71SSascha Leib } 1268e56d7b71SSascha Leib }, 1269e56d7b71SSascha Leib 1270e56d7b71SSascha Leib // returns bot info if the browser id matches a known platform: 1271e56d7b71SSascha Leib match: function(cid) { 1272e56d7b71SSascha Leib //console.info('BotMon.live.data.platforms.match(',cid,')'); 1273e56d7b71SSascha Leib 1274e56d7b71SSascha Leib let match = {"n": "Unknown", "id": 'null'}; 1275e56d7b71SSascha Leib 1276e56d7b71SSascha Leib if (cid) { 1277e56d7b71SSascha Leib BotMon.live.data.platforms._list.find(platform => { 1278e56d7b71SSascha Leib let r = false; 1279e56d7b71SSascha Leib for (let j=0; j<platform.rx.length; j++) { 1280e56d7b71SSascha Leib const rxr = cid.match(new RegExp(platform.rx[j])); 1281e56d7b71SSascha Leib if (rxr) { 1282e56d7b71SSascha Leib match.n = platform.n; 1283e56d7b71SSascha Leib match.v = (rxr.length > 1 ? rxr[1] : -1); 1284e56d7b71SSascha Leib match.id = platform.id || null; 1285e56d7b71SSascha Leib r = true; 1286e56d7b71SSascha Leib break; 1287e56d7b71SSascha Leib } 1288e56d7b71SSascha Leib } 1289e56d7b71SSascha Leib return r; 1290e56d7b71SSascha Leib }); 1291e56d7b71SSascha Leib } 1292e56d7b71SSascha Leib 1293e56d7b71SSascha Leib return match; 1294e56d7b71SSascha Leib }, 1295e56d7b71SSascha Leib 1296e56d7b71SSascha Leib // return the platform name for a given ID: 1297e56d7b71SSascha Leib getName: function(id) { 1298e56d7b71SSascha Leib const it = BotMon.live.data.platforms._list.find( pf => pf.id == id); 1299e56d7b71SSascha Leib return ( it ? it.n : 'Unknown' ); 1300e56d7b71SSascha Leib }, 1301e56d7b71SSascha Leib 1302e56d7b71SSascha Leib 1303e56d7b71SSascha Leib // indicates if the list is loaded and ready to use: 1304e56d7b71SSascha Leib _ready: false, 1305e56d7b71SSascha Leib 1306e56d7b71SSascha Leib // the actual bot list is stored here: 1307e56d7b71SSascha Leib _list: [] 1308e56d7b71SSascha Leib 1309e56d7b71SSascha Leib }, 1310e56d7b71SSascha Leib 1311e56d7b71SSascha Leib // storage and functions for the known bot IP-Ranges: 1312e56d7b71SSascha Leib ipRanges: { 1313e56d7b71SSascha Leib 1314e56d7b71SSascha Leib init: function() { 1315e56d7b71SSascha Leib //console.log('BotMon.live.data.ipRanges.init()'); 1316e56d7b71SSascha Leib // #TODO: Load from separate IP-Ranges file 1317e56d7b71SSascha Leib // load the rules file: 1318e56d7b71SSascha Leib const me = BotMon.live.data; 1319e56d7b71SSascha Leib 1320e56d7b71SSascha Leib try { 1321e56d7b71SSascha Leib BotMon.live.data._loadSettingsFile(['user-ipranges', 'known-ipranges'], 1322e56d7b71SSascha Leib (json) => { 1323e56d7b71SSascha Leib 1324e56d7b71SSascha Leib // groups can be just saved in the data structure: 1325e56d7b71SSascha Leib if (json.groups && json.groups.constructor.name == 'Array') { 1326e56d7b71SSascha Leib me.ipRanges._groups = json.groups; 1327e56d7b71SSascha Leib } 1328e56d7b71SSascha Leib 1329e56d7b71SSascha Leib // groups can be just saved in the data structure: 1330e56d7b71SSascha Leib if (json.ranges && json.ranges.constructor.name == 'Array') { 1331e56d7b71SSascha Leib json.ranges.forEach(range => { 1332e56d7b71SSascha Leib me.ipRanges.add(range); 1333e56d7b71SSascha Leib }) 1334e56d7b71SSascha Leib } 1335e56d7b71SSascha Leib 1336e56d7b71SSascha Leib // finished loading 1337e56d7b71SSascha Leib BotMon.live.gui.status.hideBusy("Status: Done."); 1338e56d7b71SSascha Leib BotMon.live.data._dispatch('ipranges') 1339e56d7b71SSascha Leib }); 1340e56d7b71SSascha Leib } catch (error) { 1341e56d7b71SSascha Leib BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); 1342e56d7b71SSascha Leib } 1343e56d7b71SSascha Leib }, 1344e56d7b71SSascha Leib 1345e56d7b71SSascha Leib // the actual bot list is stored here: 1346e56d7b71SSascha Leib _list: [], 1347e56d7b71SSascha Leib _groups: [], 1348e56d7b71SSascha Leib 1349e56d7b71SSascha Leib add: function(data) { 1350e56d7b71SSascha Leib //console.log('BotMon.live.data.ipRanges.add(',data,')'); 1351e56d7b71SSascha Leib 1352e56d7b71SSascha Leib const me = BotMon.live.data.ipRanges; 1353e56d7b71SSascha Leib 1354e56d7b71SSascha Leib // convert IP address to easier comparable form: 1355e56d7b71SSascha Leib const ip2Num = BotMon.t._ip2Num; 1356e56d7b71SSascha Leib 1357e56d7b71SSascha Leib let item = { 1358e56d7b71SSascha Leib 'cidr': data.from.replaceAll(/::+/g, '::') + '/' + ( data.m ? data.m : '??' ), 1359e56d7b71SSascha Leib 'from': ip2Num(data.from), 1360e56d7b71SSascha Leib 'to': ip2Num(data.to), 1361e56d7b71SSascha Leib 'm': data.m, 1362e56d7b71SSascha Leib 'g': data.g 1363e56d7b71SSascha Leib }; 1364e56d7b71SSascha Leib me._list.push(item); 1365e56d7b71SSascha Leib 1366e56d7b71SSascha Leib }, 1367e56d7b71SSascha Leib 1368e56d7b71SSascha Leib getOwner: function(gid) { 1369e56d7b71SSascha Leib 1370e56d7b71SSascha Leib const me = BotMon.live.data.ipRanges; 1371e56d7b71SSascha Leib 1372e56d7b71SSascha Leib for (let i=0; i < me._groups.length; i++) { 1373e56d7b71SSascha Leib const it = me._groups[i]; 1374e56d7b71SSascha Leib if (it.id == gid) { 1375e56d7b71SSascha Leib return it.name; 1376e56d7b71SSascha Leib } 1377e56d7b71SSascha Leib } 1378e56d7b71SSascha Leib return null; 1379e56d7b71SSascha Leib }, 1380e56d7b71SSascha Leib 1381e56d7b71SSascha Leib match: function(ip) { 1382e56d7b71SSascha Leib //console.log('BotMon.live.data.ipRanges.match(',ip,')'); 1383e56d7b71SSascha Leib 1384e56d7b71SSascha Leib const me = BotMon.live.data.ipRanges; 1385e56d7b71SSascha Leib 1386e56d7b71SSascha Leib // convert IP address to easier comparable form: 1387e56d7b71SSascha Leib const ipNum = BotMon.t._ip2Num(ip); 1388e56d7b71SSascha Leib 1389e56d7b71SSascha Leib for (let i=0; i < me._list.length; i++) { 1390e56d7b71SSascha Leib const ipRange = me._list[i]; 1391e56d7b71SSascha Leib 1392e56d7b71SSascha Leib if (ipNum >= ipRange.from && ipNum <= ipRange.to) { 1393e56d7b71SSascha Leib return ipRange; 1394e56d7b71SSascha Leib } 1395e56d7b71SSascha Leib 1396e56d7b71SSascha Leib }; 1397e56d7b71SSascha Leib return null; 1398e56d7b71SSascha Leib } 1399e56d7b71SSascha Leib }, 1400e56d7b71SSascha Leib 1401e56d7b71SSascha Leib // storage for the rules and related functions 1402e56d7b71SSascha Leib rules: { 1403e56d7b71SSascha Leib 1404e56d7b71SSascha Leib /** 1405e56d7b71SSascha Leib * Initializes the rules data. 1406e56d7b71SSascha Leib * 1407e56d7b71SSascha Leib * Loads the default config file and the user config file (if present). 1408e56d7b71SSascha Leib * The default config file is used if the user config file does not have a certain setting. 1409e56d7b71SSascha Leib * The user config file can override settings from the default config file. 1410e56d7b71SSascha Leib * 1411e56d7b71SSascha Leib * The rules are loaded from the `rules` property of the config files. 1412e56d7b71SSascha Leib * The IP ranges are loaded from the `ipRanges` property of the config files. 1413e56d7b71SSascha Leib * 1414e56d7b71SSascha Leib * If an error occurs while loading the config file, it is displayed in the status bar. 1415e56d7b71SSascha Leib * After the config file is loaded, the status bar is hidden. 1416e56d7b71SSascha Leib */ 1417e56d7b71SSascha Leib init: async function() { 1418e56d7b71SSascha Leib //console.info('BotMon.live.data.rules.init()'); 1419e56d7b71SSascha Leib 1420e56d7b71SSascha Leib // Load the list of known bots: 1421e56d7b71SSascha Leib BotMon.live.gui.status.showBusy("Loading list of rules …"); 1422e56d7b71SSascha Leib 1423e56d7b71SSascha Leib // load the rules file: 1424e56d7b71SSascha Leib const me = BotMon.live.data; 1425e56d7b71SSascha Leib 1426e56d7b71SSascha Leib try { 1427e56d7b71SSascha Leib BotMon.live.data._loadSettingsFile(['user-config', 'default-config'], 1428e56d7b71SSascha Leib (json) => { 1429e56d7b71SSascha Leib 1430e56d7b71SSascha Leib // override the threshold? 1431e56d7b71SSascha Leib if (json.threshold) me._threshold = json.threshold; 1432e56d7b71SSascha Leib 1433e56d7b71SSascha Leib // set the rules list: 1434e56d7b71SSascha Leib if (json.rules && json.rules.constructor.name == 'Array') { 1435e56d7b71SSascha Leib me.rules._rulesList = json.rules; 1436e56d7b71SSascha Leib } 1437e56d7b71SSascha Leib 1438e56d7b71SSascha Leib BotMon.live.gui.status.hideBusy("Status: Done."); 1439e56d7b71SSascha Leib BotMon.live.data._dispatch('rules') 1440e56d7b71SSascha Leib } 1441e56d7b71SSascha Leib ); 1442e56d7b71SSascha Leib } catch (error) { 1443e56d7b71SSascha Leib BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); 1444e56d7b71SSascha Leib } 1445e56d7b71SSascha Leib }, 1446e56d7b71SSascha Leib 1447e56d7b71SSascha Leib _rulesList: [], // list of rules to find out if a visitor is a bot 1448e56d7b71SSascha Leib _threshold: 100, // above this, it is considered a bot. 1449e56d7b71SSascha Leib 1450e56d7b71SSascha Leib // returns a descriptive text for a rule id 1451e56d7b71SSascha Leib getRuleInfo: function(ruleId) { 1452e56d7b71SSascha Leib // console.info('getRuleInfo', ruleId); 1453e56d7b71SSascha Leib 1454e56d7b71SSascha Leib // shortcut for neater code: 1455e56d7b71SSascha Leib const me = BotMon.live.data.rules; 1456e56d7b71SSascha Leib 1457e56d7b71SSascha Leib for (let i=0; i<me._rulesList.length; i++) { 1458e56d7b71SSascha Leib const rule = me._rulesList[i]; 1459e56d7b71SSascha Leib if (rule.id == ruleId) { 1460e56d7b71SSascha Leib return rule; 1461e56d7b71SSascha Leib } 1462e56d7b71SSascha Leib } 1463e56d7b71SSascha Leib return null; 1464e56d7b71SSascha Leib 1465e56d7b71SSascha Leib }, 1466e56d7b71SSascha Leib 1467e56d7b71SSascha Leib // evaluate a visitor for lkikelihood of being a bot 1468e56d7b71SSascha Leib evaluate: function(visitor) { 1469e56d7b71SSascha Leib 1470e56d7b71SSascha Leib // shortcut for neater code: 1471e56d7b71SSascha Leib const me = BotMon.live.data.rules; 1472e56d7b71SSascha Leib 1473e56d7b71SSascha Leib let r = { // evaluation result 1474e56d7b71SSascha Leib 'val': 0, 1475e56d7b71SSascha Leib 'rules': [], 1476e56d7b71SSascha Leib 'isBot': false 1477e56d7b71SSascha Leib }; 1478e56d7b71SSascha Leib 1479e56d7b71SSascha Leib for (let i=0; i<me._rulesList.length; i++) { 1480e56d7b71SSascha Leib const rule = me._rulesList[i]; 1481e56d7b71SSascha Leib const params = ( rule.params ? rule.params : [] ); 1482e56d7b71SSascha Leib 1483e56d7b71SSascha Leib if (rule.func) { // rule is calling a function 1484e56d7b71SSascha Leib if (me.func[rule.func]) { 1485e56d7b71SSascha Leib if(me.func[rule.func](visitor, ...params)) { 1486e56d7b71SSascha Leib r.val += rule.bot; 1487e56d7b71SSascha Leib r.rules.push(rule.id) 1488e56d7b71SSascha Leib } 1489e56d7b71SSascha Leib } else { 1490e56d7b71SSascha Leib //console.warn("Unknown rule function: “${rule.func}”. Ignoring rule.") 1491e56d7b71SSascha Leib } 1492e56d7b71SSascha Leib } 1493e56d7b71SSascha Leib } 1494e56d7b71SSascha Leib 1495e56d7b71SSascha Leib // is a bot? 1496e56d7b71SSascha Leib r.isBot = (r.val >= me._threshold); 1497e56d7b71SSascha Leib 1498e56d7b71SSascha Leib return r; 1499e56d7b71SSascha Leib }, 1500e56d7b71SSascha Leib 1501e56d7b71SSascha Leib // list of functions that can be called by the rules list to evaluate a visitor: 1502e56d7b71SSascha Leib func: { 1503e56d7b71SSascha Leib 1504e56d7b71SSascha Leib // check if client is on the list passed as parameter: 1505e56d7b71SSascha Leib matchesClient: function(visitor, ...clients) { 1506e56d7b71SSascha Leib 1507e56d7b71SSascha Leib const clientId = ( visitor._client ? visitor._client.id : ''); 1508e56d7b71SSascha Leib return clients.includes(clientId); 1509e56d7b71SSascha Leib }, 1510e56d7b71SSascha Leib 1511e56d7b71SSascha Leib // check if OS/Platform is one of the obsolete ones: 1512e56d7b71SSascha Leib matchesPlatform: function(visitor, ...platforms) { 1513e56d7b71SSascha Leib 1514e56d7b71SSascha Leib const pId = ( visitor._platform ? visitor._platform.id : ''); 1515e56d7b71SSascha Leib 1516e56d7b71SSascha Leib if (visitor._platform.id == null) console.log(visitor._platform); 1517e56d7b71SSascha Leib 1518e56d7b71SSascha Leib return platforms.includes(pId); 1519e56d7b71SSascha Leib }, 1520e56d7b71SSascha Leib 1521e56d7b71SSascha Leib // are there at lest num pages loaded? 1522e56d7b71SSascha Leib smallPageCount: function(visitor, num) { 1523e56d7b71SSascha Leib return (visitor._pageViews.length <= Number(num)); 1524e56d7b71SSascha Leib }, 1525e56d7b71SSascha Leib 1526e56d7b71SSascha Leib // There was no entry in a specific log file for this visitor: 1527e56d7b71SSascha Leib // note that this will also trigger the "noJavaScript" rule: 1528e56d7b71SSascha Leib noRecord: function(visitor, type) { 1529e56d7b71SSascha Leib return !visitor._seenBy.includes(type); 1530e56d7b71SSascha Leib }, 1531e56d7b71SSascha Leib 1532e56d7b71SSascha Leib // there are no referrers in any of the page visits: 1533e56d7b71SSascha Leib noReferrer: function(visitor) { 1534e56d7b71SSascha Leib 1535e56d7b71SSascha Leib let r = false; // return value 1536e56d7b71SSascha Leib for (let i = 0; i < visitor._pageViews.length; i++) { 1537e56d7b71SSascha Leib if (!visitor._pageViews[i]._ref) { 1538e56d7b71SSascha Leib r = true; 1539e56d7b71SSascha Leib break; 1540e56d7b71SSascha Leib } 1541e56d7b71SSascha Leib } 1542e56d7b71SSascha Leib return r; 1543e56d7b71SSascha Leib }, 1544e56d7b71SSascha Leib 1545e56d7b71SSascha Leib // test for specific client identifiers: 1546e56d7b71SSascha Leib /*matchesClients: function(visitor, ...list) { 1547e56d7b71SSascha Leib 1548e56d7b71SSascha Leib for (let i=0; i<list.length; i++) { 1549e56d7b71SSascha Leib if (visitor._client.id == list[i]) { 1550e56d7b71SSascha Leib return true 1551e56d7b71SSascha Leib } 1552e56d7b71SSascha Leib }; 1553e56d7b71SSascha Leib return false; 1554e56d7b71SSascha Leib },*/ 1555e56d7b71SSascha Leib 1556e56d7b71SSascha Leib // unusual combinations of Platform and Client: 1557e56d7b71SSascha Leib combinationTest: function(visitor, ...combinations) { 1558e56d7b71SSascha Leib 1559e56d7b71SSascha Leib for (let i=0; i<combinations.length; i++) { 1560e56d7b71SSascha Leib 1561e56d7b71SSascha Leib if (visitor._platform.id == combinations[i][0] 1562e56d7b71SSascha Leib && visitor._client.id == combinations[i][1]) { 1563e56d7b71SSascha Leib return true 1564e56d7b71SSascha Leib } 1565e56d7b71SSascha Leib }; 1566e56d7b71SSascha Leib 1567e56d7b71SSascha Leib return false; 1568e56d7b71SSascha Leib }, 1569e56d7b71SSascha Leib 1570e56d7b71SSascha Leib // is the IP address from a known bot network? 1571e56d7b71SSascha Leib fromKnownBotIP: function(visitor) { 1572e56d7b71SSascha Leib //console.info('fromKnownBotIP()', visitor.ip); 1573e56d7b71SSascha Leib 1574e56d7b71SSascha Leib const ipInfo = BotMon.live.data.ipRanges.match(visitor.ip); 1575e56d7b71SSascha Leib 1576e56d7b71SSascha Leib if (ipInfo) { 1577e56d7b71SSascha Leib visitor._ipInKnownBotRange = true; 1578e56d7b71SSascha Leib visitor._ipRange = ipInfo; 1579e56d7b71SSascha Leib } 1580e56d7b71SSascha Leib 1581e56d7b71SSascha Leib return (ipInfo !== null); 1582e56d7b71SSascha Leib }, 1583e56d7b71SSascha Leib 1584e56d7b71SSascha Leib // is the page language mentioned in the client's accepted languages? 1585e56d7b71SSascha Leib // the parameter holds an array of exceptions, i.e. page languages that should be ignored. 1586e56d7b71SSascha Leib matchLang: function(visitor, ...exceptions) { 1587e56d7b71SSascha Leib 1588e56d7b71SSascha Leib if (visitor.lang && visitor.accept && exceptions.indexOf(visitor.lang) < 0) { 1589e56d7b71SSascha Leib return (visitor.accept.split(',').indexOf(visitor.lang) < 0); 1590e56d7b71SSascha Leib } 1591e56d7b71SSascha Leib return false; 1592e56d7b71SSascha Leib }, 1593e56d7b71SSascha Leib 1594e56d7b71SSascha Leib // the "Accept language" header contains certain entries: 1595e56d7b71SSascha Leib clientAccepts: function(visitor, ...languages) { 1596e56d7b71SSascha Leib //console.info('clientAccepts', visitor.accept, languages); 1597e56d7b71SSascha Leib 1598e56d7b71SSascha Leib if (visitor.accept && languages) {; 1599e56d7b71SSascha Leib return ( visitor.accept.split(',').filter(lang => languages.includes(lang)).length > 0 ); 1600e56d7b71SSascha Leib } 1601e56d7b71SSascha Leib return false; 1602e56d7b71SSascha Leib }, 1603e56d7b71SSascha Leib 1604e56d7b71SSascha Leib // Is there an accept-language field defined at all? 1605e56d7b71SSascha Leib noAcceptLang: function(visitor) { 1606e56d7b71SSascha Leib 1607e56d7b71SSascha Leib if (!visitor.accept || visitor.accept.length <= 0) { // no accept-languages header 1608e56d7b71SSascha Leib return true; 1609e56d7b71SSascha Leib } 1610e56d7b71SSascha Leib // TODO: parametrize this! 1611e56d7b71SSascha Leib return false; 1612e56d7b71SSascha Leib }, 1613e56d7b71SSascha Leib // At least x page views were recorded, but they come within less than y seconds 1614e56d7b71SSascha Leib loadSpeed: function(visitor, minItems, maxTime) { 1615e56d7b71SSascha Leib 1616e56d7b71SSascha Leib if (visitor._pageViews.length >= minItems) { 1617e56d7b71SSascha Leib //console.log('loadSpeed', visitor._pageViews.length, minItems, maxTime); 1618e56d7b71SSascha Leib 1619e56d7b71SSascha Leib const pvArr = visitor._pageViews.map(pv => pv._lastSeen).sort(); 1620e56d7b71SSascha Leib 1621e56d7b71SSascha Leib let totalTime = 0; 1622e56d7b71SSascha Leib for (let i=1; i < pvArr.length; i++) { 1623e56d7b71SSascha Leib totalTime += (pvArr[i] - pvArr[i-1]); 1624e56d7b71SSascha Leib } 1625e56d7b71SSascha Leib 1626e56d7b71SSascha Leib //console.log(' ', totalTime , Math.round(totalTime / (pvArr.length * 1000)), (( totalTime / pvArr.length ) <= maxTime * 1000), visitor.ip); 1627e56d7b71SSascha Leib 1628e56d7b71SSascha Leib return (( totalTime / pvArr.length ) <= maxTime * 1000); 1629e56d7b71SSascha Leib } 1630e56d7b71SSascha Leib }, 1631e56d7b71SSascha Leib 1632e56d7b71SSascha Leib // Country code matches one of those in the list: 1633e56d7b71SSascha Leib matchesCountry: function(visitor, ...countries) { 1634e56d7b71SSascha Leib 1635e56d7b71SSascha Leib // ingore if geoloc is not set or unknown: 1636e56d7b71SSascha Leib if (visitor.geo) { 1637e56d7b71SSascha Leib return (countries.indexOf(visitor.geo) >= 0); 1638e56d7b71SSascha Leib } 1639e56d7b71SSascha Leib return false; 1640e56d7b71SSascha Leib }, 1641e56d7b71SSascha Leib 1642e56d7b71SSascha Leib // Country does not match one of the given codes. 1643e56d7b71SSascha Leib notFromCountry: function(visitor, ...countries) { 1644e56d7b71SSascha Leib 1645e56d7b71SSascha Leib // ingore if geoloc is not set or unknown: 1646e56d7b71SSascha Leib if (visitor.geo && visitor.geo !== 'ZZ') { 1647e56d7b71SSascha Leib return (countries.indexOf(visitor.geo) < 0); 1648e56d7b71SSascha Leib } 1649e56d7b71SSascha Leib return false; 1650e56d7b71SSascha Leib } 1651e56d7b71SSascha Leib } 1652e56d7b71SSascha Leib }, 1653e56d7b71SSascha Leib 1654e56d7b71SSascha Leib /** 1655e56d7b71SSascha Leib * Loads a settings file from the specified list of filenames. 1656e56d7b71SSascha Leib * If the file is successfully loaded, it will call the callback function 1657e56d7b71SSascha Leib * with the loaded JSON data. 1658e56d7b71SSascha Leib * If no file can be loaded, it will display an error message. 1659e56d7b71SSascha Leib * 1660e56d7b71SSascha Leib * @param {string[]} fns - list of filenames to load 1661e56d7b71SSascha Leib * @param {function} callback - function to call with the loaded JSON data 1662e56d7b71SSascha Leib */ 1663e56d7b71SSascha Leib _loadSettingsFile: async function(fns, callback) { 1664e56d7b71SSascha Leib //console.info('BotMon.live.data._loadSettingsFile()', fns); 1665e56d7b71SSascha Leib 1666e56d7b71SSascha Leib const kJsonExt = '.json'; 1667e56d7b71SSascha Leib let loaded = false; // if successfully loaded file 1668e56d7b71SSascha Leib 1669e56d7b71SSascha Leib for (let i=0; i<fns.length; i++) { 1670e56d7b71SSascha Leib const filename = fns[i] +kJsonExt; 1671e56d7b71SSascha Leib try { 1672e56d7b71SSascha Leib const response = await fetch(DOKU_BASE + 'lib/plugins/botmon/config/' + filename); 1673e56d7b71SSascha Leib if (!response.ok) { 1674e56d7b71SSascha Leib continue; 1675e56d7b71SSascha Leib } else { 1676e56d7b71SSascha Leib loaded = true; 1677e56d7b71SSascha Leib } 1678e56d7b71SSascha Leib const json = await response.json(); 1679e56d7b71SSascha Leib if (callback && typeof callback === 'function') { 1680e56d7b71SSascha Leib callback(json); 1681e56d7b71SSascha Leib } 1682e56d7b71SSascha Leib break; 1683e56d7b71SSascha Leib } catch (e) { 1684e56d7b71SSascha Leib BotMon.live.gui.status.setError("Error while loading the config file: " + filename); 1685e56d7b71SSascha Leib } 1686e56d7b71SSascha Leib } 1687e56d7b71SSascha Leib 1688e56d7b71SSascha Leib if (!loaded) { 1689e56d7b71SSascha Leib BotMon.live.gui.status.setError("Could not load a config file."); 1690e56d7b71SSascha Leib } 1691e56d7b71SSascha Leib }, 1692e56d7b71SSascha Leib 1693e56d7b71SSascha Leib /** 1694e56d7b71SSascha Leib * Loads a log file (server, page load, or ticker) and parses it. 1695e56d7b71SSascha Leib * @param {String} type - the type of the log file to load (srv, log, or tck) 1696e56d7b71SSascha Leib * @param {Function} [onLoaded] - an optional callback function to call after loading is finished. 1697e56d7b71SSascha Leib */ 1698e56d7b71SSascha Leib loadLogFile: async function(type, onLoaded = undefined) { 1699e56d7b71SSascha Leib //console.info('BotMon.live.data.loadLogFile(',type,')'); 1700e56d7b71SSascha Leib 1701e56d7b71SSascha Leib let typeName = ''; 1702e56d7b71SSascha Leib let columns = []; 1703e56d7b71SSascha Leib 1704e56d7b71SSascha Leib switch (type) { 1705e56d7b71SSascha Leib case "srv": 1706e56d7b71SSascha Leib typeName = "Server"; 1707e56d7b71SSascha Leib columns = ['ts','ip','pg','id','typ','usr','agent','ref','lang','accept','geo']; 1708e56d7b71SSascha Leib break; 1709e56d7b71SSascha Leib case "log": 1710e56d7b71SSascha Leib typeName = "Page load"; 1711e56d7b71SSascha Leib columns = ['ts','ip','pg','id','usr','lt','ref','agent']; 1712e56d7b71SSascha Leib break; 1713e56d7b71SSascha Leib case "tck": 1714e56d7b71SSascha Leib typeName = "Ticker"; 1715e56d7b71SSascha Leib columns = ['ts','ip','pg','id','agent']; 1716e56d7b71SSascha Leib break; 1717e56d7b71SSascha Leib default: 1718e56d7b71SSascha Leib console.warn(`Unknown log type ${type}.`); 1719e56d7b71SSascha Leib return; 1720e56d7b71SSascha Leib } 1721e56d7b71SSascha Leib 1722e56d7b71SSascha Leib // Show the busy indicator and set the visible status: 1723e56d7b71SSascha Leib BotMon.live.gui.status.showBusy(`Loading ${typeName} log file …`); 1724e56d7b71SSascha Leib 1725e56d7b71SSascha Leib // compose the URL from which to load: 1726e56d7b71SSascha Leib const url = BotMon._baseDir + `logs/${BotMon._datestr}.${type}.txt`; 1727e56d7b71SSascha Leib //console.log("Loading:",url); 1728e56d7b71SSascha Leib 1729e56d7b71SSascha Leib // fetch the data: 1730e56d7b71SSascha Leib try { 1731e56d7b71SSascha Leib const response = await fetch(url); 1732e56d7b71SSascha Leib if (!response.ok) { 1733e56d7b71SSascha Leib 1734e56d7b71SSascha Leib throw new Error(`${response.status} ${response.statusText}`); 1735e56d7b71SSascha Leib 1736e56d7b71SSascha Leib } else { 1737e56d7b71SSascha Leib 1738e56d7b71SSascha Leib // parse the data: 1739e56d7b71SSascha Leib const logtxt = await response.text(); 1740e56d7b71SSascha Leib if (logtxt.length <= 0) { 1741e56d7b71SSascha Leib throw new Error(`Empty log file ${url}.`); 1742e56d7b71SSascha Leib } 1743e56d7b71SSascha Leib 1744e56d7b71SSascha Leib logtxt.split('\n').forEach((line) => { 1745e56d7b71SSascha Leib if (line.trim() === '') return; // skip empty lines 1746e56d7b71SSascha Leib const cols = line.split('\t'); 1747e56d7b71SSascha Leib 1748e56d7b71SSascha Leib // assign the columns to an object: 1749e56d7b71SSascha Leib const data = {}; 1750e56d7b71SSascha Leib cols.forEach( (colVal,i) => { 17510edf1a56SSascha Leib const colName = columns[i] || `col${i}`; 1752e56d7b71SSascha Leib const colValue = (colName == 'ts' ? new Date(colVal) : colVal.trim()); 1753e56d7b71SSascha Leib data[colName] = colValue; 1754e56d7b71SSascha Leib }); 1755e56d7b71SSascha Leib 1756e56d7b71SSascha Leib // register the visit in the model: 1757e56d7b71SSascha Leib switch(type) { 1758e56d7b71SSascha Leib case BM_LOGTYPE.SERVER: 1759e56d7b71SSascha Leib BotMon.live.data.model.registerVisit(data, type); 1760e56d7b71SSascha Leib break; 1761e56d7b71SSascha Leib case BM_LOGTYPE.CLIENT: 1762e56d7b71SSascha Leib data.typ = 'js'; 1763e56d7b71SSascha Leib BotMon.live.data.model.updateVisit(data); 1764e56d7b71SSascha Leib break; 1765e56d7b71SSascha Leib case BM_LOGTYPE.TICKER: 1766e56d7b71SSascha Leib data.typ = 'js'; 1767e56d7b71SSascha Leib BotMon.live.data.model.updateTicks(data); 1768e56d7b71SSascha Leib break; 1769e56d7b71SSascha Leib default: 1770e56d7b71SSascha Leib console.warn(`Unknown log type ${type}.`); 1771e56d7b71SSascha Leib return; 1772e56d7b71SSascha Leib } 1773e56d7b71SSascha Leib }); 1774e56d7b71SSascha Leib } 1775e56d7b71SSascha Leib 1776e56d7b71SSascha Leib } catch (error) { 1777e56d7b71SSascha Leib BotMon.live.gui.status.setError(`Error while loading the ${typeName} log file: ${error.message} – data may be incomplete.`); 1778e56d7b71SSascha Leib } finally { 1779e56d7b71SSascha Leib BotMon.live.gui.status.hideBusy("Status: Done."); 1780e56d7b71SSascha Leib if (onLoaded) { 1781e56d7b71SSascha Leib onLoaded(); // callback after loading is finished. 1782e56d7b71SSascha Leib } 1783e56d7b71SSascha Leib } 1784e56d7b71SSascha Leib } 1785e56d7b71SSascha Leib }, 1786e56d7b71SSascha Leib 1787e56d7b71SSascha Leib gui: { 1788e56d7b71SSascha Leib init: function() { 1789e2bc8f6fSSascha Leib //console.log('BotMon.live.gui.init()'); 1790e2bc8f6fSSascha Leib 17910edf1a56SSascha Leib // init sub-objects: 17920edf1a56SSascha Leib BotMon.t._callInit(this); 1793e56d7b71SSascha Leib }, 1794e56d7b71SSascha Leib 1795e2bc8f6fSSascha Leib tabs: { 1796e2bc8f6fSSascha Leib init: function() { 1797e2bc8f6fSSascha Leib //console.log('BotMon.live.gui.tabs.init()'); 1798e2bc8f6fSSascha Leib 1799e2bc8f6fSSascha Leib /* find and add all existing tabs */ 1800e2bc8f6fSSascha Leib document.querySelectorAll('#botmon__admin *[role=tablist]') 1801e2bc8f6fSSascha Leib .forEach((tablist) => { 1802e2bc8f6fSSascha Leib tablist.querySelectorAll('*[role=tab]') 1803e2bc8f6fSSascha Leib .forEach( t => t.addEventListener('click', this._onTabClick) ) 1804e2bc8f6fSSascha Leib }); 1805e2bc8f6fSSascha Leib }, 1806e2bc8f6fSSascha Leib 1807e2bc8f6fSSascha Leib /* callback for tab click */ 1808e2bc8f6fSSascha Leib _onTabClick: function(e) { 1809e2bc8f6fSSascha Leib //console.log('BotMon.live.gui.tabs._onTabClick()'); 1810e2bc8f6fSSascha Leib 1811e2bc8f6fSSascha Leib /* reusable constants: */ 1812e2bc8f6fSSascha Leib const kAriaSelected = 'aria-selected'; 1813e2bc8f6fSSascha Leib const kAriaControls = 'aria-controls'; 1814e2bc8f6fSSascha Leib const kTrue = 'true'; 1815e2bc8f6fSSascha Leib const kFalse = 'false'; 1816e2bc8f6fSSascha Leib const kHidden = 'hidden'; 1817e2bc8f6fSSascha Leib 1818e2bc8f6fSSascha Leib /* cancel default action */ 1819e2bc8f6fSSascha Leib e.preventDefault(); 1820e2bc8f6fSSascha Leib 1821e2bc8f6fSSascha Leib /* if the active tab is clicked, do nothing: */ 1822e2bc8f6fSSascha Leib let selState = this.getAttribute(kAriaSelected); 1823e2bc8f6fSSascha Leib if ( selState && selState == kTrue ) { 1824e2bc8f6fSSascha Leib return; 1825e2bc8f6fSSascha Leib } 1826e2bc8f6fSSascha Leib 1827e2bc8f6fSSascha Leib /* find the active tab element: */ 1828e2bc8f6fSSascha Leib var aItem = null; 1829e2bc8f6fSSascha Leib let tablist = this.parentNode; 1830e2bc8f6fSSascha Leib while (tablist.getAttribute('role') !== 'tablist') { 1831e2bc8f6fSSascha Leib tablist = tablist.parentNode; 1832e2bc8f6fSSascha Leib } 1833e2bc8f6fSSascha Leib 1834e2bc8f6fSSascha Leib if (tablist.getAttribute('role') == 'tablist') { 1835e2bc8f6fSSascha Leib let lis = tablist.querySelectorAll('*[role=tab]'); 1836e2bc8f6fSSascha Leib lis.forEach( (it) => { 1837e2bc8f6fSSascha Leib let selected = it.getAttribute(kAriaSelected); 1838e2bc8f6fSSascha Leib if ( selected && selected == kTrue ) { 1839e2bc8f6fSSascha Leib aItem = it; 1840e2bc8f6fSSascha Leib } 1841e2bc8f6fSSascha Leib }); 1842e2bc8f6fSSascha Leib } 1843e2bc8f6fSSascha Leib 1844e2bc8f6fSSascha Leib /* swap the active states: */ 1845e2bc8f6fSSascha Leib this.setAttribute(kAriaSelected, kTrue); 1846e2bc8f6fSSascha Leib if (aItem) { 1847e2bc8f6fSSascha Leib aItem.setAttribute(kAriaSelected, kFalse); 1848e2bc8f6fSSascha Leib let aId = aItem.getAttribute(kAriaControls); 1849e2bc8f6fSSascha Leib let aObj = document.getElementById(aId); 1850e2bc8f6fSSascha Leib if (aObj) aObj.hidden = true; 1851e2bc8f6fSSascha Leib } 1852e2bc8f6fSSascha Leib 1853e2bc8f6fSSascha Leib /* show the new panel: */ 1854e2bc8f6fSSascha Leib let nId = this.getAttribute(kAriaControls); 1855e2bc8f6fSSascha Leib let nObj = document.getElementById(nId); 1856e2bc8f6fSSascha Leib if (nObj) nObj.hidden = false; 1857e2bc8f6fSSascha Leib } 1858e2bc8f6fSSascha Leib }, 18590edf1a56SSascha Leib 1860e56d7b71SSascha Leib /* The Overview / web metrics section of the live tab */ 1861e56d7b71SSascha Leib overview: { 1862e56d7b71SSascha Leib /** 1863e56d7b71SSascha Leib * Populates the overview part of the today tab with the analytics data. 1864e56d7b71SSascha Leib * 1865e56d7b71SSascha Leib * @method make 1866e56d7b71SSascha Leib * @memberof BotMon.live.gui.overview 1867e56d7b71SSascha Leib */ 1868e56d7b71SSascha Leib make: function() { 1869e56d7b71SSascha Leib 1870e56d7b71SSascha Leib const data = BotMon.live.data.analytics.data; 1871e56d7b71SSascha Leib 1872e56d7b71SSascha Leib const maxItemsPerList = 5; // how many list items to show? 1873e56d7b71SSascha Leib 1874*20997f50SSascha Leib const kNoData = '–'; // shown when data is missing 1875*20997f50SSascha Leib 1876e56d7b71SSascha Leib // shortcut for neater code: 1877e56d7b71SSascha Leib const makeElement = BotMon.t._makeElement; 1878e56d7b71SSascha Leib 1879e56d7b71SSascha Leib const botsVsHumans = document.getElementById('botmon__today__botsvshumans'); 1880e56d7b71SSascha Leib if (botsVsHumans) { 1881e56d7b71SSascha Leib botsVsHumans.appendChild(makeElement('dt', {}, "Page views")); 1882e56d7b71SSascha Leib 1883e56d7b71SSascha Leib for (let i = 0; i <= 5; i++) { 1884e56d7b71SSascha Leib const dd = makeElement('dd'); 1885e56d7b71SSascha Leib let title = ''; 1886e56d7b71SSascha Leib let value = ''; 1887e56d7b71SSascha Leib switch(i) { 1888e56d7b71SSascha Leib case 0: 1889e56d7b71SSascha Leib title = "Known bots:"; 1890*20997f50SSascha Leib value = data.bots.known || kNoData; 1891e56d7b71SSascha Leib break; 1892e56d7b71SSascha Leib case 1: 1893e56d7b71SSascha Leib title = "Suspected bots:"; 1894*20997f50SSascha Leib value = data.bots.suspected || kNoData; 1895e56d7b71SSascha Leib break; 1896e56d7b71SSascha Leib case 2: 1897e56d7b71SSascha Leib title = "Probably humans:"; 1898*20997f50SSascha Leib value = data.bots.human || kNoData; 1899e56d7b71SSascha Leib break; 1900e56d7b71SSascha Leib case 3: 1901e56d7b71SSascha Leib title = "Registered users:"; 1902*20997f50SSascha Leib value = data.bots.users || kNoData; 1903e56d7b71SSascha Leib break; 1904e56d7b71SSascha Leib case 4: 1905e56d7b71SSascha Leib title = "Total:"; 1906*20997f50SSascha Leib value = data.totalPageViews || kNoData; 1907e56d7b71SSascha Leib break; 1908e56d7b71SSascha Leib case 5: 1909e56d7b71SSascha Leib title = "Bots-humans ratio:"; 1910e56d7b71SSascha Leib value = BotMon.t._getRatio(data.bots.suspected + data.bots.known, data.bots.users + data.bots.human, 100); 1911e56d7b71SSascha Leib break; 1912e56d7b71SSascha Leib default: 1913e56d7b71SSascha Leib console.warn(`Unknown list type ${i}.`); 1914e56d7b71SSascha Leib } 1915e56d7b71SSascha Leib dd.appendChild(makeElement('span', {}, title)); 1916e56d7b71SSascha Leib dd.appendChild(makeElement('strong', {}, value)); 1917e56d7b71SSascha Leib botsVsHumans.appendChild(dd); 1918e56d7b71SSascha Leib } 1919e56d7b71SSascha Leib } 1920e56d7b71SSascha Leib 1921e56d7b71SSascha Leib // update known bots list: 1922e56d7b71SSascha Leib const botElement = document.getElementById('botmon__botslist'); /* Known bots */ 1923e56d7b71SSascha Leib if (botElement) { 1924e56d7b71SSascha Leib botElement.appendChild(makeElement('dt', {}, `Top known bots`)); 1925e56d7b71SSascha Leib 1926e56d7b71SSascha Leib let botList = BotMon.live.data.analytics.getTopBots(maxItemsPerList); 1927e56d7b71SSascha Leib botList.forEach( (botInfo) => { 1928e56d7b71SSascha Leib const bli = makeElement('dd'); 1929e56d7b71SSascha Leib bli.appendChild(makeElement('span', {'class': 'has_icon bot bot_' + botInfo.id }, botInfo.name)); 1930e56d7b71SSascha Leib bli.appendChild(makeElement('span', {'class': 'count' }, botInfo.count)); 1931e56d7b71SSascha Leib botElement.append(bli) 1932e56d7b71SSascha Leib }); 1933e56d7b71SSascha Leib } 1934e56d7b71SSascha Leib 1935e56d7b71SSascha Leib // update the suspected bot IP ranges list: 1936e56d7b71SSascha Leib const botIps = document.getElementById('botmon__botips'); 1937e56d7b71SSascha Leib if (botIps) { 1938*20997f50SSascha Leib botIps.appendChild(makeElement('dt', {}, "Top bot Networks")); 1939e56d7b71SSascha Leib 1940e56d7b71SSascha Leib const ispList = BotMon.live.data.analytics.getTopBotISPs(5); 1941*20997f50SSascha Leib //console.log(ispList); 1942e56d7b71SSascha Leib ispList.forEach( (netInfo) => { 1943e56d7b71SSascha Leib const li = makeElement('dd'); 1944*20997f50SSascha Leib li.appendChild(makeElement('span', {'class': 'has_icon ipaddr ip' + netInfo.typ }, netInfo.name)); 1945e56d7b71SSascha Leib li.appendChild(makeElement('span', {'class': 'count' }, netInfo.count)); 1946e56d7b71SSascha Leib botIps.append(li) 1947e56d7b71SSascha Leib }); 1948e56d7b71SSascha Leib } 1949e56d7b71SSascha Leib 1950e56d7b71SSascha Leib // update the top bot countries list: 1951e56d7b71SSascha Leib const botCountries = document.getElementById('botmon__botcountries'); 1952e56d7b71SSascha Leib if (botCountries) { 1953e56d7b71SSascha Leib botCountries.appendChild(makeElement('dt', {}, `Top bot Countries`)); 1954e56d7b71SSascha Leib const countryList = BotMon.live.data.analytics.getCountryList('bot', 5); 1955e56d7b71SSascha Leib countryList.forEach( (cInfo) => { 1956e56d7b71SSascha Leib const cLi = makeElement('dd'); 1957e56d7b71SSascha Leib cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name)); 1958e56d7b71SSascha Leib cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count)); 1959e56d7b71SSascha Leib botCountries.appendChild(cLi); 1960e56d7b71SSascha Leib }); 1961e56d7b71SSascha Leib } 1962e56d7b71SSascha Leib 1963e56d7b71SSascha Leib // update the webmetrics overview: 1964e56d7b71SSascha Leib const wmoverview = document.getElementById('botmon__today__wm_overview'); 1965e56d7b71SSascha Leib if (wmoverview) { 1966e56d7b71SSascha Leib 1967e56d7b71SSascha Leib const humanVisits = BotMon.live.data.analytics.groups.users.length + BotMon.live.data.analytics.groups.humans.length; 1968e56d7b71SSascha Leib const bounceRate = Math.round(100 * (BotMon.live.data.analytics.getBounceCount('users') + BotMon.live.data.analytics.getBounceCount('humans')) / humanVisits); 1969e56d7b71SSascha Leib 1970e56d7b71SSascha Leib wmoverview.appendChild(makeElement('dt', {}, "Humans’ metrics")); 1971e56d7b71SSascha Leib for (let i = 0; i <= 4; i++) { 1972e56d7b71SSascha Leib const dd = makeElement('dd'); 1973e56d7b71SSascha Leib let title = ''; 1974e56d7b71SSascha Leib let value = ''; 1975e56d7b71SSascha Leib switch(i) { 1976e56d7b71SSascha Leib case 0: 1977*20997f50SSascha Leib title = "Registered users’ page views:"; 1978*20997f50SSascha Leib value = data.bots.users || kNoData; 1979e56d7b71SSascha Leib break; 1980e56d7b71SSascha Leib case 1: 1981*20997f50SSascha Leib title = "“Probably humans” page views:"; 1982*20997f50SSascha Leib value = data.bots.human || kNoData; 1983e56d7b71SSascha Leib break; 1984e56d7b71SSascha Leib case 2: 1985e56d7b71SSascha Leib title = "Total human page views:"; 1986*20997f50SSascha Leib value = (data.bots.users + data.bots.human) || kNoData; 1987e56d7b71SSascha Leib break; 1988e56d7b71SSascha Leib case 3: 1989e56d7b71SSascha Leib title = "Total human visits:"; 1990*20997f50SSascha Leib value = humanVisits || kNoData; 1991e56d7b71SSascha Leib break; 1992e56d7b71SSascha Leib case 4: 1993e56d7b71SSascha Leib title = "Humans’ bounce rate:"; 1994e56d7b71SSascha Leib value = bounceRate + '%'; 1995e56d7b71SSascha Leib break; 1996e56d7b71SSascha Leib default: 1997e56d7b71SSascha Leib console.warn(`Unknown list type ${i}.`); 1998e56d7b71SSascha Leib } 1999e56d7b71SSascha Leib dd.appendChild(makeElement('span', {}, title)); 2000e56d7b71SSascha Leib dd.appendChild(makeElement('strong', {}, value)); 2001e56d7b71SSascha Leib wmoverview.appendChild(dd); 2002e56d7b71SSascha Leib } 2003e56d7b71SSascha Leib } 2004e56d7b71SSascha Leib 2005e56d7b71SSascha Leib // update the webmetrics clients list: 2006e56d7b71SSascha Leib const wmclients = document.getElementById('botmon__today__wm_clients'); 2007e56d7b71SSascha Leib if (wmclients) { 2008e56d7b71SSascha Leib 2009e56d7b71SSascha Leib wmclients.appendChild(makeElement('dt', {}, "Browsers")); 2010e56d7b71SSascha Leib 2011e56d7b71SSascha Leib const clientList = BotMon.live.data.analytics.getTopBrowsers(maxItemsPerList); 2012e56d7b71SSascha Leib if (clientList) { 2013e56d7b71SSascha Leib clientList.forEach( (cInfo) => { 2014e56d7b71SSascha Leib const cDd = makeElement('dd'); 2015e56d7b71SSascha Leib cDd.appendChild(makeElement('span', {'class': 'has_icon client cl_' + cInfo.id }, ( cInfo.name ? cInfo.name : cInfo.id))); 2016e56d7b71SSascha Leib cDd.appendChild(makeElement('span', { 2017e56d7b71SSascha Leib 'class': 'count', 2018e56d7b71SSascha Leib 'title': cInfo.count + " page views" 2019e56d7b71SSascha Leib }, cInfo.pct.toFixed(1) + '%')); 2020e56d7b71SSascha Leib wmclients.appendChild(cDd); 2021e56d7b71SSascha Leib }); 2022e56d7b71SSascha Leib } 2023e56d7b71SSascha Leib } 2024e56d7b71SSascha Leib 2025e56d7b71SSascha Leib // update the webmetrics platforms list: 2026e56d7b71SSascha Leib const wmplatforms = document.getElementById('botmon__today__wm_platforms'); 2027e56d7b71SSascha Leib if (wmplatforms) { 2028e56d7b71SSascha Leib 2029e56d7b71SSascha Leib wmplatforms.appendChild(makeElement('dt', {}, "Platforms")); 2030e56d7b71SSascha Leib 2031e56d7b71SSascha Leib const pfList = BotMon.live.data.analytics.getTopPlatforms(maxItemsPerList); 2032e56d7b71SSascha Leib if (pfList) { 2033e56d7b71SSascha Leib pfList.forEach( (pInfo) => { 2034e56d7b71SSascha Leib const pDd = makeElement('dd'); 2035e56d7b71SSascha Leib pDd.appendChild(makeElement('span', {'class': 'has_icon platform pf_' + pInfo.id }, ( pInfo.name ? pInfo.name : pInfo.id))); 2036e56d7b71SSascha Leib pDd.appendChild(makeElement('span', { 2037e56d7b71SSascha Leib 'class': 'count', 2038e56d7b71SSascha Leib 'title': pInfo.count + " page views" 2039e56d7b71SSascha Leib }, pInfo.pct.toFixed(1) + '%')); 2040e56d7b71SSascha Leib wmplatforms.appendChild(pDd); 2041e56d7b71SSascha Leib }); 2042e56d7b71SSascha Leib } 2043e56d7b71SSascha Leib } 2044e56d7b71SSascha Leib 2045e56d7b71SSascha Leib // update the top bot countries list: 2046e56d7b71SSascha Leib const usrCountries = document.getElementById('botmon__today__wm_countries'); 2047e56d7b71SSascha Leib if (usrCountries) { 2048e56d7b71SSascha Leib usrCountries.appendChild(makeElement('dt', {}, `Top visitor Countries:`)); 2049e56d7b71SSascha Leib const usrCtryList = BotMon.live.data.analytics.getCountryList('human', 5); 2050e56d7b71SSascha Leib usrCtryList.forEach( (cInfo) => { 2051e56d7b71SSascha Leib const cLi = makeElement('dd'); 2052e56d7b71SSascha Leib cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name)); 2053e56d7b71SSascha Leib cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count)); 2054e56d7b71SSascha Leib usrCountries.appendChild(cLi); 2055e56d7b71SSascha Leib }); 2056e56d7b71SSascha Leib } 2057e56d7b71SSascha Leib 2058e56d7b71SSascha Leib // update the top pages; 2059e56d7b71SSascha Leib const wmpages = document.getElementById('botmon__today__wm_pages'); 2060e56d7b71SSascha Leib if (wmpages) { 2061e56d7b71SSascha Leib 2062e56d7b71SSascha Leib wmpages.appendChild(makeElement('dt', {}, "Top pages")); 2063e56d7b71SSascha Leib 2064e56d7b71SSascha Leib const pgList = BotMon.live.data.analytics.getTopPages(maxItemsPerList); 2065e56d7b71SSascha Leib if (pgList) { 2066e56d7b71SSascha Leib pgList.forEach( (pgInfo) => { 2067e56d7b71SSascha Leib const pgDd = makeElement('dd'); 2068e56d7b71SSascha Leib pgDd.appendChild(makeElement('a', { 2069e56d7b71SSascha Leib 'class': 'page_icon', 2070e56d7b71SSascha Leib 'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(pgInfo.id), 2071e56d7b71SSascha Leib 'target': 'preview', 2072e56d7b71SSascha Leib 'title': "PageID: " + pgInfo.id 2073e56d7b71SSascha Leib }, pgInfo.id)); 2074e56d7b71SSascha Leib pgDd.appendChild(makeElement('span', { 2075e56d7b71SSascha Leib 'class': 'count', 2076e56d7b71SSascha Leib 'title': pgInfo.count + " page views" 2077e56d7b71SSascha Leib }, pgInfo.count)); 2078e56d7b71SSascha Leib wmpages.appendChild(pgDd); 2079e56d7b71SSascha Leib }); 2080e56d7b71SSascha Leib } 2081e56d7b71SSascha Leib } 2082e56d7b71SSascha Leib 2083e56d7b71SSascha Leib // update the top referrers; 2084e56d7b71SSascha Leib const wmreferers = document.getElementById('botmon__today__wm_referers'); 2085e56d7b71SSascha Leib if (wmreferers) { 2086e56d7b71SSascha Leib 2087e56d7b71SSascha Leib wmreferers.appendChild(makeElement('dt', {}, "Referers")); 2088e56d7b71SSascha Leib 2089e56d7b71SSascha Leib const refList = BotMon.live.data.analytics.getTopReferers(maxItemsPerList); 2090e56d7b71SSascha Leib if (refList) { 2091e56d7b71SSascha Leib refList.forEach( (rInfo) => { 2092e56d7b71SSascha Leib const rDd = makeElement('dd'); 2093e56d7b71SSascha Leib rDd.appendChild(makeElement('span', {'class': 'has_icon referer ref_' + rInfo.id }, rInfo.name)); 2094e56d7b71SSascha Leib rDd.appendChild(makeElement('span', { 2095e56d7b71SSascha Leib 'class': 'count', 2096e56d7b71SSascha Leib 'title': rInfo.count + " references" 2097e56d7b71SSascha Leib }, rInfo.pct.toFixed(1) + '%')); 2098e56d7b71SSascha Leib wmreferers.appendChild(rDd); 2099e56d7b71SSascha Leib }); 2100e56d7b71SSascha Leib } 2101e56d7b71SSascha Leib } 2102e56d7b71SSascha Leib } 2103e56d7b71SSascha Leib }, 2104e56d7b71SSascha Leib 2105e56d7b71SSascha Leib status: { 2106e56d7b71SSascha Leib setText: function(txt) { 2107e56d7b71SSascha Leib const el = document.getElementById('botmon__today__status'); 2108e56d7b71SSascha Leib if (el && BotMon.live.gui.status._errorCount <= 0) { 2109e56d7b71SSascha Leib el.innerText = txt; 2110e56d7b71SSascha Leib } 2111e56d7b71SSascha Leib }, 2112e56d7b71SSascha Leib 2113e56d7b71SSascha Leib setTitle: function(html) { 2114e56d7b71SSascha Leib const el = document.getElementById('botmon__today__title'); 2115e56d7b71SSascha Leib if (el) { 2116e56d7b71SSascha Leib el.innerHTML = html; 2117e56d7b71SSascha Leib } 2118e56d7b71SSascha Leib }, 2119e56d7b71SSascha Leib 2120e56d7b71SSascha Leib setError: function(txt) { 2121e56d7b71SSascha Leib console.error(txt); 2122e56d7b71SSascha Leib BotMon.live.gui.status._errorCount += 1; 2123e56d7b71SSascha Leib const el = document.getElementById('botmon__today__status'); 2124e56d7b71SSascha Leib if (el) { 2125e56d7b71SSascha Leib el.innerText = "Data may be incomplete."; 2126e56d7b71SSascha Leib el.classList.add('error'); 2127e56d7b71SSascha Leib } 2128e56d7b71SSascha Leib }, 2129e56d7b71SSascha Leib _errorCount: 0, 2130e56d7b71SSascha Leib 2131e56d7b71SSascha Leib showBusy: function(txt = null) { 2132e56d7b71SSascha Leib BotMon.live.gui.status._busyCount += 1; 2133e56d7b71SSascha Leib const el = document.getElementById('botmon__today__busy'); 2134e56d7b71SSascha Leib if (el) { 2135e56d7b71SSascha Leib el.style.display = 'inline-block'; 2136e56d7b71SSascha Leib } 2137e56d7b71SSascha Leib if (txt) BotMon.live.gui.status.setText(txt); 2138e56d7b71SSascha Leib }, 2139e56d7b71SSascha Leib _busyCount: 0, 2140e56d7b71SSascha Leib 2141e56d7b71SSascha Leib hideBusy: function(txt = null) { 2142e56d7b71SSascha Leib const el = document.getElementById('botmon__today__busy'); 2143e56d7b71SSascha Leib BotMon.live.gui.status._busyCount -= 1; 2144e56d7b71SSascha Leib if (BotMon.live.gui.status._busyCount <= 0) { 2145e56d7b71SSascha Leib if (el) el.style.display = 'none'; 2146e56d7b71SSascha Leib if (txt) BotMon.live.gui.status.setText(txt); 2147e56d7b71SSascha Leib } 2148e56d7b71SSascha Leib } 2149e56d7b71SSascha Leib }, 2150e56d7b71SSascha Leib 2151e56d7b71SSascha Leib lists: { 2152e56d7b71SSascha Leib init: function() { 2153e56d7b71SSascha Leib 2154e56d7b71SSascha Leib // function shortcut: 2155e56d7b71SSascha Leib const makeElement = BotMon.t._makeElement; 2156e56d7b71SSascha Leib 2157e56d7b71SSascha Leib const parent = document.getElementById('botmon__today__visitorlists'); 2158e56d7b71SSascha Leib if (parent) { 2159e56d7b71SSascha Leib 2160e56d7b71SSascha Leib for (let i=0; i < 4; i++) { 2161e56d7b71SSascha Leib 2162e56d7b71SSascha Leib // change the id and title by number: 2163e56d7b71SSascha Leib let listTitle = ''; 2164e56d7b71SSascha Leib let listId = ''; 2165e56d7b71SSascha Leib let infolink = null; 2166e56d7b71SSascha Leib switch (i) { 2167e56d7b71SSascha Leib case 0: 2168e56d7b71SSascha Leib listTitle = "Registered users"; 2169e56d7b71SSascha Leib listId = 'users'; 2170e56d7b71SSascha Leib break; 2171e56d7b71SSascha Leib case 1: 2172e56d7b71SSascha Leib listTitle = "Probably humans"; 2173e56d7b71SSascha Leib listId = 'humans'; 2174e56d7b71SSascha Leib break; 2175e56d7b71SSascha Leib case 2: 2176e56d7b71SSascha Leib listTitle = "Suspected bots"; 2177e56d7b71SSascha Leib listId = 'suspectedBots'; 2178e56d7b71SSascha Leib infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/suspected_bots'; 2179e56d7b71SSascha Leib break; 2180e56d7b71SSascha Leib case 3: 2181e56d7b71SSascha Leib listTitle = "Known bots"; 2182e56d7b71SSascha Leib listId = 'knownBots'; 2183e56d7b71SSascha Leib infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/known_bots'; 2184e56d7b71SSascha Leib break; 2185e56d7b71SSascha Leib default: 2186e56d7b71SSascha Leib console.warn('Unknown list number.'); 2187e56d7b71SSascha Leib } 2188e56d7b71SSascha Leib 2189e56d7b71SSascha Leib const details = makeElement('details', { 2190e56d7b71SSascha Leib 'data-group': listId, 2191e56d7b71SSascha Leib 'data-loaded': false 2192e56d7b71SSascha Leib }); 2193e56d7b71SSascha Leib const title = details.appendChild(makeElement('summary')); 2194e56d7b71SSascha Leib title.appendChild(makeElement('span', {'class': 'title'}, listTitle)); 2195e56d7b71SSascha Leib if (infolink) { 2196e56d7b71SSascha Leib title.appendChild(makeElement('a', { 2197e56d7b71SSascha Leib 'class': 'ext_info', 2198e56d7b71SSascha Leib 'target': '_blank', 2199e56d7b71SSascha Leib 'href': infolink, 2200e56d7b71SSascha Leib 'title': "More information" 2201e56d7b71SSascha Leib }, "Info")); 2202e56d7b71SSascha Leib } 2203e56d7b71SSascha Leib details.addEventListener("toggle", this._onDetailsToggle); 2204e56d7b71SSascha Leib 2205e56d7b71SSascha Leib parent.appendChild(details); 2206e56d7b71SSascha Leib 2207e56d7b71SSascha Leib } 2208e56d7b71SSascha Leib } 2209e56d7b71SSascha Leib }, 2210e56d7b71SSascha Leib 2211e56d7b71SSascha Leib _onDetailsToggle: function(e) { 2212e56d7b71SSascha Leib //console.info('BotMon.live.gui.lists._onDetailsToggle()'); 2213e56d7b71SSascha Leib 2214e56d7b71SSascha Leib const target = e.target; 2215e56d7b71SSascha Leib 2216e56d7b71SSascha Leib if (target.getAttribute('data-loaded') == 'false') { // only if not loaded yet 2217e56d7b71SSascha Leib target.setAttribute('data-loaded', 'loading'); 2218e56d7b71SSascha Leib 2219e56d7b71SSascha Leib const fillType = target.getAttribute('data-group'); 2220e56d7b71SSascha Leib const fillList = BotMon.live.data.analytics.groups[fillType]; 2221e56d7b71SSascha Leib if (fillList && fillList.length > 0) { 2222e56d7b71SSascha Leib 2223e56d7b71SSascha Leib const ul = BotMon.t._makeElement('ul'); 2224e56d7b71SSascha Leib 2225e56d7b71SSascha Leib fillList.forEach( (it) => { 2226e56d7b71SSascha Leib ul.appendChild(BotMon.live.gui.lists._makeVisitorItem(it, fillType)); 2227e56d7b71SSascha Leib }); 2228e56d7b71SSascha Leib 2229e56d7b71SSascha Leib target.appendChild(ul); 2230e56d7b71SSascha Leib target.setAttribute('data-loaded', 'true'); 2231e56d7b71SSascha Leib } else { 2232e56d7b71SSascha Leib target.setAttribute('data-loaded', 'false'); 2233e56d7b71SSascha Leib } 2234e56d7b71SSascha Leib 2235e56d7b71SSascha Leib } 2236e56d7b71SSascha Leib }, 2237e56d7b71SSascha Leib 2238e56d7b71SSascha Leib _makeVisitorItem: function(data, type) { 2239e56d7b71SSascha Leib 2240e56d7b71SSascha Leib // shortcut for neater code: 2241e56d7b71SSascha Leib const make = BotMon.t._makeElement; 2242e56d7b71SSascha Leib 2243e56d7b71SSascha Leib let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' ); 2244e56d7b71SSascha Leib if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0'; 2245e56d7b71SSascha Leib 2246e56d7b71SSascha Leib const platformName = (data._platform ? data._platform.n : 'Unknown'); 2247e56d7b71SSascha Leib const clientName = (data._client ? data._client.n: 'Unknown'); 2248e56d7b71SSascha Leib 2249e56d7b71SSascha Leib const sumClass = ( !data._seenBy || data._seenBy.indexOf(BM_LOGTYPE.SERVER) < 0 ? 'noServer' : 'hasServer'); 2250e56d7b71SSascha Leib 2251e56d7b71SSascha Leib const li = make('li'); // root list item 2252e56d7b71SSascha Leib const details = make('details'); 2253e56d7b71SSascha Leib const summary = make('summary', { 2254e56d7b71SSascha Leib 'class': sumClass 2255e56d7b71SSascha Leib }); 2256e56d7b71SSascha Leib details.appendChild(summary); 2257e56d7b71SSascha Leib 2258e56d7b71SSascha Leib const span1 = make('span'); /* left-hand group */ 2259e56d7b71SSascha Leib 2260e56d7b71SSascha Leib if (data._type !== BM_USERTYPE.KNOWN_BOT) { /* No platform/client for bots */ 2261e56d7b71SSascha Leib span1.appendChild(make('span', { /* Platform */ 2262e56d7b71SSascha Leib 'class': 'icon_only platform pf_' + (data._platform ? data._platform.id : 'unknown'), 2263e56d7b71SSascha Leib 'title': "Platform: " + platformName 2264e56d7b71SSascha Leib }, platformName)); 2265e56d7b71SSascha Leib 2266e56d7b71SSascha Leib span1.appendChild(make('span', { /* Client */ 2267e56d7b71SSascha Leib 'class': 'icon_only client client cl_' + (data._client ? data._client.id : 'unknown'), 2268e56d7b71SSascha Leib 'title': "Client: " + clientName 2269e56d7b71SSascha Leib }, clientName)); 2270e56d7b71SSascha Leib } 2271e56d7b71SSascha Leib 2272e56d7b71SSascha Leib // identifier: 2273e56d7b71SSascha Leib if (data._type == BM_USERTYPE.KNOWN_BOT) { /* Bot only */ 2274e56d7b71SSascha Leib 2275e56d7b71SSascha Leib const botName = ( data._bot && data._bot.n ? data._bot.n : "Unknown"); 2276e56d7b71SSascha Leib span1.appendChild(make('span', { /* Bot */ 2277e56d7b71SSascha Leib 'class': 'has_icon bot bot_' + (data._bot ? data._bot.id : 'unknown'), 2278e56d7b71SSascha Leib 'title': "Bot: " + botName 2279e56d7b71SSascha Leib }, botName)); 2280e56d7b71SSascha Leib 2281e56d7b71SSascha Leib } else if (data._type == BM_USERTYPE.KNOWN_USER) { /* User only */ 2282e56d7b71SSascha Leib 2283e56d7b71SSascha Leib span1.appendChild(make('span', { /* User */ 2284e56d7b71SSascha Leib 'class': 'has_icon user_known', 2285e56d7b71SSascha Leib 'title': "User: " + data.usr 2286e56d7b71SSascha Leib }, data.usr)); 2287e56d7b71SSascha Leib 2288e56d7b71SSascha Leib } else { /* others */ 2289e56d7b71SSascha Leib 2290e56d7b71SSascha Leib 2291e56d7b71SSascha Leib /*span1.appendChild(make('span', { // IP-Address 2292e56d7b71SSascha Leib 'class': 'has_icon ipaddr ip' + ipType, 2293e56d7b71SSascha Leib 'title': "IP-Address: " + data.ip 2294e56d7b71SSascha Leib }, data.ip));*/ 2295e56d7b71SSascha Leib 2296e56d7b71SSascha Leib span1.appendChild(make('span', { /* Internal ID */ 2297e56d7b71SSascha Leib 'class': 'has_icon session typ_' + data.typ, 2298e56d7b71SSascha Leib 'title': "ID: " + data.id 2299e56d7b71SSascha Leib }, data.id)); 2300e56d7b71SSascha Leib } 2301e56d7b71SSascha Leib 2302e56d7b71SSascha Leib // country flag: 2303e56d7b71SSascha Leib if (data.geo && data.geo !== 'ZZ') { 2304e56d7b71SSascha Leib span1.appendChild(make('span', { 2305e56d7b71SSascha Leib 'class': 'icon_only country ctry_' + data.geo.toLowerCase(), 2306e56d7b71SSascha Leib 'data-ctry': data.geo, 2307e56d7b71SSascha Leib 'title': "Country: " + ( data._country || "Unknown") 2308e56d7b71SSascha Leib }, ( data._country || "Unknown") )); 2309e56d7b71SSascha Leib } 2310e56d7b71SSascha Leib 2311e56d7b71SSascha Leib // referer icons: 2312e56d7b71SSascha Leib if ((data._type == BM_USERTYPE.PROBABLY_HUMAN || data._type == BM_USERTYPE.LIKELY_BOT) && data.ref) { 2313e56d7b71SSascha Leib const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref); 2314e56d7b71SSascha Leib span1.appendChild(make('span', { 2315e56d7b71SSascha Leib 'class': 'icon_only referer ref_' + refInfo.id, 2316e56d7b71SSascha Leib 'title': "Referer: " + data.ref 2317e56d7b71SSascha Leib }, refInfo.n)); 2318e56d7b71SSascha Leib } 2319e56d7b71SSascha Leib 2320e56d7b71SSascha Leib summary.appendChild(span1); 2321e56d7b71SSascha Leib const span2 = make('span'); /* right-hand group */ 2322e56d7b71SSascha Leib 2323e56d7b71SSascha Leib span2.appendChild(make('span', { /* first-seen */ 2324e56d7b71SSascha Leib 'class': 'has_iconfirst-seen', 2325e56d7b71SSascha Leib 'title': "First seen: " + data._firstSeen.toLocaleString() + " UTC" 2326e56d7b71SSascha Leib }, BotMon.t._formatTime(data._firstSeen))); 2327e56d7b71SSascha Leib 2328e56d7b71SSascha Leib span2.appendChild(make('span', { /* page views */ 2329e56d7b71SSascha Leib 'class': 'has_icon pageviews', 2330e56d7b71SSascha Leib 'title': data._pageViews.length + " page view(s)" 2331e56d7b71SSascha Leib }, data._pageViews.length)); 2332e56d7b71SSascha Leib 2333e56d7b71SSascha Leib summary.appendChild(span2); 2334e56d7b71SSascha Leib 2335e56d7b71SSascha Leib // add details expandable section: 2336e56d7b71SSascha Leib details.appendChild(BotMon.live.gui.lists._makeVisitorDetails(data, type)); 2337e56d7b71SSascha Leib 2338e56d7b71SSascha Leib li.appendChild(details); 2339e56d7b71SSascha Leib return li; 2340e56d7b71SSascha Leib }, 2341e56d7b71SSascha Leib 2342e56d7b71SSascha Leib _makeVisitorDetails: function(data, type) { 2343e56d7b71SSascha Leib 2344e56d7b71SSascha Leib // shortcut for neater code: 2345e56d7b71SSascha Leib const make = BotMon.t._makeElement; 2346e56d7b71SSascha Leib 2347e56d7b71SSascha Leib let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' ); 2348e56d7b71SSascha Leib if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0'; 2349e56d7b71SSascha Leib const platformName = (data._platform ? data._platform.n : 'Unknown'); 2350e56d7b71SSascha Leib const clientName = (data._client ? data._client.n: 'Unknown'); 2351e56d7b71SSascha Leib 2352e56d7b71SSascha Leib const dl = make('dl', {'class': 'visitor_details'}); 2353e56d7b71SSascha Leib 2354e56d7b71SSascha Leib if (data._type == BM_USERTYPE.KNOWN_BOT) { 2355e56d7b71SSascha Leib 2356e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Bot name:")); /* bot info */ 2357e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'icon_only bot bot_' + (data._bot ? data._bot.id : 'unknown')}, 2358e56d7b71SSascha Leib (data._bot ? data._bot.n : 'Unknown'))); 2359e56d7b71SSascha Leib 2360e56d7b71SSascha Leib if (data._bot && data._bot.url) { 2361e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Bot info:")); /* bot info */ 2362e56d7b71SSascha Leib const botInfoDd = dl.appendChild(make('dd')); 2363e56d7b71SSascha Leib botInfoDd.appendChild(make('a', { 2364e56d7b71SSascha Leib 'href': data._bot.url, 2365e56d7b71SSascha Leib 'target': '_blank' 2366e56d7b71SSascha Leib }, data._bot.url)); /* bot info link*/ 2367e56d7b71SSascha Leib 2368e56d7b71SSascha Leib } 2369e56d7b71SSascha Leib 2370e56d7b71SSascha Leib } else { /* not for bots */ 2371e56d7b71SSascha Leib 2372e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Client:")); /* client */ 2373e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'has_icon client cl_' + (data._client ? data._client.id : 'unknown')}, 2374e56d7b71SSascha Leib clientName + ( data._client.v > 0 ? ' (' + data._client.v + ')' : '' ) )); 2375e56d7b71SSascha Leib 2376e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Platform:")); /* platform */ 2377e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')}, 2378e56d7b71SSascha Leib platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); 2379e56d7b71SSascha Leib 2380e56d7b71SSascha Leib /*dl.appendChild(make('dt', {}, "ID:")); 2381e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/ 2382e56d7b71SSascha Leib } 2383e56d7b71SSascha Leib 2384e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "IP-Address:")); 2385e56d7b71SSascha Leib const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType}); 2386e56d7b71SSascha Leib ipItem.appendChild(make('span', {'class': 'address'} , data.ip)); 2387e56d7b71SSascha Leib ipItem.appendChild(make('a', { 2388e56d7b71SSascha Leib 'class': 'icon_only extlink ipinfo', 2389e56d7b71SSascha Leib 'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`, 2390e56d7b71SSascha Leib 'target': 'ipinfo', 2391e56d7b71SSascha Leib 'title': "View this address on IPInfo.io" 2392e56d7b71SSascha Leib } , "DNS Info")); 2393e56d7b71SSascha Leib ipItem.appendChild(make('a', { 2394e56d7b71SSascha Leib 'class': 'icon_only extlink abuseipdb', 2395e56d7b71SSascha Leib 'href': `https://www.abuseipdb.com/check/${encodeURIComponent(data.ip)}`, 2396e56d7b71SSascha Leib 'target': 'abuseipdb', 2397e56d7b71SSascha Leib 'title': "Check this address on AbuseIPDB.com" 2398e56d7b71SSascha Leib } , "Check on AbuseIPDB")); 2399e56d7b71SSascha Leib dl.appendChild(ipItem); 2400e56d7b71SSascha Leib 2401e56d7b71SSascha Leib if (Math.abs(data._lastSeen - data._firstSeen) < 100) { 2402e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Seen:")); 2403e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString())); 2404e56d7b71SSascha Leib } else { 2405e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "First seen:")); 2406e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'firstSeen'}, data._firstSeen.toLocaleString())); 2407e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Last seen:")); 2408e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'lastSeen'}, data._lastSeen.toLocaleString())); 2409e56d7b71SSascha Leib } 2410e56d7b71SSascha Leib 2411e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "User-Agent:")); 2412e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'agent'}, data.agent)); 2413e56d7b71SSascha Leib 2414e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Languages:")); 2415e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'langs'}, ` [${data.accept}]`)); 2416e56d7b71SSascha Leib 2417e56d7b71SSascha Leib if (data.geo && data.geo !=='') { 2418e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Location:")); 2419e56d7b71SSascha Leib dl.appendChild(make('dd', { 2420e56d7b71SSascha Leib 'class': 'has_icon country ctry_' + data.geo.toLowerCase(), 2421e56d7b71SSascha Leib 'data-ctry': data.geo, 2422e56d7b71SSascha Leib 'title': "Country: " + data._country 2423e56d7b71SSascha Leib }, data._country + ' (' + data.geo + ')')); 2424e56d7b71SSascha Leib } 2425e56d7b71SSascha Leib 2426e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Session ID:")); 2427e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'has_icon session typ_' + data.typ}, data.id)); 2428e56d7b71SSascha Leib 2429e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Seen by:")); 2430e56d7b71SSascha Leib dl.appendChild(make('dd', undefined, data._seenBy.join(', ') )); 2431e56d7b71SSascha Leib 2432e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Visited pages:")); 2433e56d7b71SSascha Leib const pagesDd = make('dd', {'class': 'pages'}); 2434e56d7b71SSascha Leib const pageList = make('ul'); 2435e56d7b71SSascha Leib 2436e56d7b71SSascha Leib /* list all page views */ 2437e56d7b71SSascha Leib data._pageViews.sort( (a, b) => a._firstSeen - b._firstSeen ); 2438e56d7b71SSascha Leib data._pageViews.forEach( (page) => { 2439e56d7b71SSascha Leib pageList.appendChild(BotMon.live.gui.lists._makePageViewItem(page)); 2440e56d7b71SSascha Leib }); 2441e56d7b71SSascha Leib pagesDd.appendChild(pageList); 2442e56d7b71SSascha Leib dl.appendChild(pagesDd); 2443e56d7b71SSascha Leib 2444e56d7b71SSascha Leib /* bot evaluation rating */ 2445e56d7b71SSascha Leib if (data._type !== BM_USERTYPE.KNOWN_BOT && data._type !== BM_USERTYPE.KNOWN_USER) { 2446e56d7b71SSascha Leib dl.appendChild(make('dt', undefined, "Bot rating:")); 2447e56d7b71SSascha Leib dl.appendChild(make('dd', {'class': 'bot-rating'}, ( data._botVal ? data._botVal : '–' ) + ' (of ' + BotMon.live.data.rules._threshold + ')')); 2448e56d7b71SSascha Leib 2449e56d7b71SSascha Leib /* add bot evaluation details: */ 2450e56d7b71SSascha Leib if (data._eval) { 2451e56d7b71SSascha Leib dl.appendChild(make('dt', {}, "Bot evaluation:")); 2452e56d7b71SSascha Leib const evalDd = make('dd', {'class': 'eval'}); 2453e56d7b71SSascha Leib const testList = make('ul'); 2454e56d7b71SSascha Leib data._eval.forEach( test => { 2455e56d7b71SSascha Leib 2456e56d7b71SSascha Leib const tObj = BotMon.live.data.rules.getRuleInfo(test); 2457e56d7b71SSascha Leib let tDesc = tObj ? tObj.desc : test; 2458e56d7b71SSascha Leib 2459e56d7b71SSascha Leib // special case for Bot IP range test: 2460e56d7b71SSascha Leib if (tObj.func == 'fromKnownBotIP') { 2461e56d7b71SSascha Leib const rangeInfo = BotMon.live.data.ipRanges.match(data.ip); 2462e56d7b71SSascha Leib if (rangeInfo) { 2463e56d7b71SSascha Leib const owner = BotMon.live.data.ipRanges.getOwner(rangeInfo.g); 2464e56d7b71SSascha Leib tDesc += ' (range: “' + rangeInfo.cidr + '”, ' + owner + ')'; 2465e56d7b71SSascha Leib } 2466e56d7b71SSascha Leib } 2467e56d7b71SSascha Leib 2468e56d7b71SSascha Leib // create the entry field 2469e56d7b71SSascha Leib const tstLi = make('li'); 2470e56d7b71SSascha Leib tstLi.appendChild(make('span', { 2471e56d7b71SSascha Leib 'data-testid': test 2472e56d7b71SSascha Leib }, tDesc)); 2473e56d7b71SSascha Leib tstLi.appendChild(make('span', {}, ( tObj ? tObj.bot : '—') )); 2474e56d7b71SSascha Leib testList.appendChild(tstLi); 2475e56d7b71SSascha Leib }); 2476e56d7b71SSascha Leib 2477e56d7b71SSascha Leib // add total row 2478e56d7b71SSascha Leib const tst2Li = make('li', { 2479e56d7b71SSascha Leib 'class': 'total' 2480e56d7b71SSascha Leib }); 2481e56d7b71SSascha Leib /*tst2Li.appendChild(make('span', {}, "Total:")); 2482e56d7b71SSascha Leib tst2Li.appendChild(make('span', {}, data._botVal)); 2483e56d7b71SSascha Leib testList.appendChild(tst2Li);*/ 2484e56d7b71SSascha Leib 2485e56d7b71SSascha Leib evalDd.appendChild(testList); 2486e56d7b71SSascha Leib dl.appendChild(evalDd); 2487e56d7b71SSascha Leib } 2488e56d7b71SSascha Leib } 2489e56d7b71SSascha Leib // return the element to add to the UI: 2490e56d7b71SSascha Leib return dl; 2491e56d7b71SSascha Leib }, 2492e56d7b71SSascha Leib 2493e56d7b71SSascha Leib // make a page view item: 2494e56d7b71SSascha Leib _makePageViewItem: function(page) { 2495e56d7b71SSascha Leib //console.log("makePageViewItem:",page); 2496e56d7b71SSascha Leib 2497e56d7b71SSascha Leib // shortcut for neater code: 2498e56d7b71SSascha Leib const make = BotMon.t._makeElement; 2499e56d7b71SSascha Leib 2500e56d7b71SSascha Leib // the actual list item: 2501e56d7b71SSascha Leib const pgLi = make('li'); 2502e56d7b71SSascha Leib 2503e56d7b71SSascha Leib const row1 = make('div', {'class': 'row'}); 2504e56d7b71SSascha Leib 2505e56d7b71SSascha Leib row1.appendChild(make('a', { // page id is the left group 2506e56d7b71SSascha Leib 'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(page.pg), 2507e56d7b71SSascha Leib 'target': 'preview', 2508e56d7b71SSascha Leib 'hreflang': page.lang, 2509e56d7b71SSascha Leib 'title': "PageID: " + page.pg 2510e56d7b71SSascha Leib }, page.pg)); /* DW Page ID */ 2511e56d7b71SSascha Leib 2512e56d7b71SSascha Leib // get the time difference: 2513e56d7b71SSascha Leib row1.appendChild(make('span', { 2514e56d7b71SSascha Leib 'class': 'first-seen', 2515e56d7b71SSascha Leib 'title': "First visited: " + page._firstSeen.toLocaleString() + " UTC" 2516e56d7b71SSascha Leib }, BotMon.t._formatTime(page._firstSeen))); 2517e56d7b71SSascha Leib 2518e56d7b71SSascha Leib pgLi.appendChild(row1); 2519e56d7b71SSascha Leib 2520e56d7b71SSascha Leib /* LINE 2 */ 2521e56d7b71SSascha Leib 2522e56d7b71SSascha Leib const row2 = make('div', {'class': 'row'}); 2523e56d7b71SSascha Leib 2524e56d7b71SSascha Leib // page referrer: 2525e56d7b71SSascha Leib if (page._ref) { 2526e56d7b71SSascha Leib row2.appendChild(make('span', { 2527e56d7b71SSascha Leib 'class': 'referer', 2528e56d7b71SSascha Leib 'title': "Referrer: " + page._ref.href 2529e56d7b71SSascha Leib }, page._ref.hostname)); 2530e56d7b71SSascha Leib } else { 2531e56d7b71SSascha Leib row2.appendChild(make('span', { 2532e56d7b71SSascha Leib 'class': 'referer' 2533e56d7b71SSascha Leib }, "No referer")); 2534e56d7b71SSascha Leib } 2535e56d7b71SSascha Leib 2536e56d7b71SSascha Leib // visit duration: 2537e56d7b71SSascha Leib let visitTimeStr = "Bounce"; 2538e56d7b71SSascha Leib const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime(); 2539e56d7b71SSascha Leib if (visitDuration > 0) { 2540e56d7b71SSascha Leib visitTimeStr = Math.floor(visitDuration / 1000) + "s"; 2541e56d7b71SSascha Leib } 2542e56d7b71SSascha Leib const tDiff = BotMon.t._formatTimeDiff(page._firstSeen, page._lastSeen); 2543e56d7b71SSascha Leib if (tDiff) { 2544e56d7b71SSascha Leib row2.appendChild(make('span', {'class': 'visit-length', 'title': 'Last seen: ' + page._lastSeen.toLocaleString()}, tDiff)); 2545e56d7b71SSascha Leib } else { 2546e56d7b71SSascha Leib row2.appendChild(make('span', { 2547e56d7b71SSascha Leib 'class': 'bounce', 2548e56d7b71SSascha Leib 'title': "Visitor bounced"}, "Bounce")); 2549e56d7b71SSascha Leib } 2550e56d7b71SSascha Leib 2551e56d7b71SSascha Leib pgLi.appendChild(row2); 2552e56d7b71SSascha Leib 2553e56d7b71SSascha Leib return pgLi; 2554e56d7b71SSascha Leib } 2555e56d7b71SSascha Leib } 2556e56d7b71SSascha Leib } 2557e56d7b71SSascha Leib}; 2558e56d7b71SSascha Leib 2559e56d7b71SSascha Leib/* launch only if the BotMon admin panel is open: */ 2560e56d7b71SSascha Leibif (document.getElementById('botmon__admin')) { 2561e56d7b71SSascha Leib BotMon.init(); 2562e56d7b71SSascha Leib}