1"use strict"; 2/* DokuWiki BotMon Plugin Script file */ 3/* 14.10.2025 - 0.5.0 - pre-release */ 4/* Author: Sascha Leib <ad@hominem.info> */ 5 6// enumeration of user types: 7const BM_USERTYPE = Object.freeze({ 8 'UNKNOWN': 'unknown', 9 'KNOWN_USER': 'user', 10 'PROBABLY_HUMAN': 'human', 11 'LIKELY_BOT': 'likely_bot', 12 'KNOWN_BOT': 'known_bot' 13}); 14 15// enumeration of log types: 16const BM_LOGTYPE = Object.freeze({ 17 'SERVER': 'srv', 18 'CLIENT': 'log', 19 'TICKER': 'tck' 20}); 21 22// enumeration of IP versions: 23const BM_IPVERSION = Object.freeze({ 24 'IPv4': 4, 25 'IPv6': 6 26}); 27 28/* BotMon root object */ 29const BotMon = { 30 31 init: function() { 32 //console.info('BotMon.init()'); 33 34 // find the plugin basedir: 35 this._baseDir = document.currentScript.src.substring(0, document.currentScript.src.lastIndexOf('/')+1); 36 37 // read the page language from the DOM: 38 this._lang = document.getRootNode().documentElement.lang || this._lang; 39 40 // get the time offset: 41 this._timeDiff = BotMon.t._getTimeOffset(); 42 43 // get yesterday's date: 44 let d = new Date(); 45 d.setDate(d.getDate() - 1); 46 this._datestr = d.toISOString().slice(0, 10); 47 48 // init the sub-objects: 49 BotMon.t._callInit(this); 50 }, 51 52 _baseDir: null, 53 _lang: 'en', 54 _datestr: '', 55 _timeDiff: '', 56 57 /* internal tools */ 58 t: { 59 /* helper function to call inits of sub-objects */ 60 _callInit: function(obj) { 61 //console.info('BotMon.t._callInit(obj=',obj,')'); 62 63 /* call init / _init on each sub-object: */ 64 Object.keys(obj).forEach( (key,i) => { 65 const sub = obj[key]; 66 let init = null; 67 if (typeof sub === 'object' && sub.init) { 68 init = sub.init; 69 } 70 71 // bind to object 72 if (typeof init == 'function') { 73 const init2 = init.bind(sub); 74 init2(obj); 75 } 76 }); 77 }, 78 79 /* helper function to calculate the time difference to UTC: */ 80 _getTimeOffset: function() { 81 const now = new Date(); 82 let offset = now.getTimezoneOffset(); // in minutes 83 const sign = Math.sign(offset); // +1 or -1 84 offset = Math.abs(offset); // always positive 85 86 let hours = 0; 87 while (offset >= 60) { 88 hours += 1; 89 offset -= 60; 90 } 91 return ( hours > 0 ? sign * hours + ' h' : '') + (offset > 0 ? ` ${offset} min` : ''); 92 }, 93 94 /* helper function to create a new element with all attributes and text content */ 95 _makeElement: function(name, atlist = undefined, text = undefined) { 96 var r = null; 97 try { 98 r = document.createElement(name); 99 if (atlist) { 100 for (let attr in atlist) { 101 r.setAttribute(attr, atlist[attr]); 102 } 103 } 104 if (text) { 105 r.textContent = text.toString(); 106 } 107 } catch(e) { 108 console.error(e); 109 } 110 return r; 111 }, 112 113 /* helper to convert an ip address string to a normalised format: */ 114 _ip2Num: function(ip) { 115 if (!ip) { 116 return 'null'; 117 } else if (ip.indexOf(':') > 0) { /* IP6 */ 118 return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(':')); 119 } else { /* IP4 */ 120 return ip.split('.').map(d => ('000'+d).slice(-3) ).join('.'); 121 } 122 }, 123 124 /* helper function to format a Date object to show only the time. */ 125 /* returns String */ 126 _formatTime: function(date) { 127 128 if (date) { 129 return date.getHours() + ':' + ('0'+date.getMinutes()).slice(-2) + ':' + ('0'+date.getSeconds()).slice(-2); 130 } else { 131 return null; 132 } 133 134 }, 135 136 /* helper function to show a time difference in seconds or minutes */ 137 /* returns String */ 138 _formatTimeDiff: function(dateA, dateB) { 139 140 // if the second date is ealier, swap them: 141 if (dateA > dateB) dateB = [dateA, dateA = dateB][0]; 142 143 // get the difference in milliseconds: 144 let ms = dateB - dateA; 145 146 if (ms > 50) { /* ignore small time spans */ 147 const h = Math.floor((ms / (1000 * 60 * 60)) % 24); 148 const m = Math.floor((ms / (1000 * 60)) % 60); 149 const s = Math.floor((ms / 1000) % 60); 150 151 return ( h>0 ? h + 'h ': '') + ( m>0 ? m + 'm ': '') + ( s>0 ? s + 's': ''); 152 } 153 154 return null; 155 156 }, 157 158 // calcualte a reduced ration between two numbers 159 // adapted from https://stackoverflow.com/questions/3946373/math-in-js-how-do-i-get-a-ratio-from-a-percentage 160 _getRatio: function(a, b, tolerance) { 161 162 var bg = b; 163 var sm = a; 164 if (a > b) { 165 var bg = a; 166 var sm = b; 167 } 168 169 for (var i = 1; i < 1000000; i++) { 170 var d = sm / i; 171 var res = bg / d; 172 var howClose = Math.abs(res - res.toFixed(0)); 173 if (howClose < tolerance) { 174 if (a > b) { 175 return res.toFixed(0) + ':' + i; 176 } else { 177 return i + ':' + res.toFixed(0); 178 } 179 } 180 } 181 } 182 } 183}; 184 185/* everything specific to the "Latest" tab is self-contained in the "live" object: */ 186BotMon.live = { 187 init: function() { 188 console.info('BotMon.live.init()'); 189 190 // set the title: 191 const tDiff = '<abbr title="Coordinated Universal Time">UTC</abbr> ' + (BotMon._timeDiff != '' ? ` (offset: ${BotMon._timeDiff}` : '' ) + ')'; 192 BotMon.live.gui.status.setTitle(`Data for <time datetime="${BotMon._datestr}">${BotMon._datestr}</time> ${tDiff}`); 193 194 // init sub-objects: 195 BotMon.t._callInit(this); 196 }, 197 198 data: { 199 init: function() { 200 //console.info('BotMon.live.data.init()'); 201 202 // call sub-inits: 203 BotMon.t._callInit(this); 204 }, 205 206 // this will be called when the known json files are done loading: 207 _dispatch: function(file) { 208 //console.info('BotMon.live.data._dispatch(,',file,')'); 209 210 // shortcut to make code more readable: 211 const data = BotMon.live.data; 212 213 // set the flags: 214 switch(file) { 215 case 'rules': 216 data._dispatchRulesLoaded = true; 217 break; 218 case 'ipranges': 219 data._dispatchIPRangesLoaded = true; 220 break; 221 case 'bots': 222 data._dispatchBotsLoaded = true; 223 break; 224 case 'clients': 225 data._dispatchClientsLoaded = true; 226 break; 227 case 'platforms': 228 data._dispatchPlatformsLoaded = true; 229 break; 230 default: 231 // ignore 232 } 233 234 // are all the flags set? 235 if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded && data._dispatchIPRangesLoaded) { 236 // chain the log files loading: 237 BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded); 238 } 239 }, 240 // flags to track which data files have been loaded: 241 _dispatchBotsLoaded: false, 242 _dispatchClientsLoaded: false, 243 _dispatchPlatformsLoaded: false, 244 _dispatchIPRangesLoaded: false, 245 _dispatchRulesLoaded: false, 246 247 // event callback, after the server log has been loaded: 248 _onServerLogLoaded: function() { 249 //console.info('BotMon.live.data._onServerLogLoaded()'); 250 251 // chain the client log file to load: 252 BotMon.live.data.loadLogFile(BM_LOGTYPE.CLIENT, BotMon.live.data._onClientLogLoaded); 253 }, 254 255 // event callback, after the client log has been loaded: 256 _onClientLogLoaded: function() { 257 //console.info('BotMon.live.data._onClientLogLoaded()'); 258 259 // chain the ticks file to load: 260 BotMon.live.data.loadLogFile(BM_LOGTYPE.TICKER, BotMon.live.data._onTicksLogLoaded); 261 262 }, 263 264 // event callback, after the tiker log has been loaded: 265 _onTicksLogLoaded: function() { 266 //console.info('BotMon.live.data._onTicksLogLoaded()'); 267 268 // analyse the data: 269 BotMon.live.data.analytics.analyseAll(); 270 271 // sort the data: 272 // #TODO 273 274 // display the data: 275 BotMon.live.gui.overview.make(); 276 277 //console.log(BotMon.live.data.model._visitors); 278 279 }, 280 281 // the data model: 282 model: { 283 // visitors storage: 284 _visitors: [], 285 286 // find an already existing visitor record: 287 findVisitor: function(visitor, type) { 288 //console.info('BotMon.live.data.model.findVisitor()', type); 289 //console.log(visitor); 290 291 // shortcut to make code more readable: 292 const model = BotMon.live.data.model; 293 294 const timeout = 60 * 60 * 1000; // session timeout: One hour 295 296 if (visitor._type == BM_USERTYPE.KNOWN_BOT) { // known bots match by their bot ID: 297 298 for (let i=0; i<model._visitors.length; i++) { 299 const v = model._visitors[i]; 300 301 // bots match when their ID matches: 302 if (v._bot && v._bot.id == visitor._bot.id) { 303 return v; 304 } 305 } 306 } else { // other types match by their DW/PHPIDs: 307 308 // loop over all visitors already registered and check for ID matches: 309 for (let i=0; i<model._visitors.length; i++) { 310 const v = model._visitors[i]; 311 312 if ( v.id == visitor.id) { // match the DW/PHP IDs 313 return v; 314 } 315 } 316 317 // if not found, try to match IP address and user agent: 318 for (let i=0; i<model._visitors.length; i++) { 319 const v = model._visitors[i]; 320 if ( v.ip == visitor.ip && v.agent == visitor.agent) { 321 return v; 322 } 323 } 324 } 325 326 return null; // nothing found 327 }, 328 329 /* if there is already this visit registered, return the page view item */ 330 _getPageView: function(visit, view) { 331 332 // shortcut to make code more readable: 333 const model = BotMon.live.data.model; 334 335 for (let i=0; i<visit._pageViews.length; i++) { 336 const pv = visit._pageViews[i]; 337 if (pv.pg == view.pg) { 338 return pv; 339 } 340 } 341 return null; // not found 342 }, 343 344 // register a new visitor (or update if already exists) 345 registerVisit: function(nv, type) { 346 //console.info('registerVisit', nv, type); 347 348 // shortcut to make code more readable: 349 const model = BotMon.live.data.model; 350 351 // is it a known bot? 352 const bot = BotMon.live.data.bots.match(nv.agent); 353 354 // enrich new visitor with relevant data: 355 if (!nv._bot) nv._bot = bot ?? null; // bot info 356 nv._type = ( bot ? BM_USERTYPE.KNOWN_BOT : ( nv.usr && nv.usr !== '' ? BM_USERTYPE.KNOWN_USER : BM_USERTYPE.UNKNOWN ) ); // user type 357 if (bot && bot.geo) { 358 if (!nv.geo || nv.geo == '' || nv.geo == 'ZZ') nv.geo = bot.geo; 359 } else if (!nv.geo ||nv.geo == '') { 360 nv.geo = 'ZZ'; 361 } 362 363 // update first and last seen: 364 if (!nv._firstSeen) nv._firstSeen = nv.ts; // first-seen 365 nv._lastSeen = nv.ts; // last-seen 366 367 // country name: 368 try { 369 nv._country = ( nv.geo == 'local' ? "localhost" : "Unknown" ); 370 if (nv.geo && nv.geo !== '' && nv.geo !== 'ZZ' && nv.geo !== 'local') { 371 const countryName = new Intl.DisplayNames(['en', BotMon._lang], {type: 'region'}); 372 nv._country = countryName.of(nv.geo.substring(0,2)) ?? nv.geo; 373 } 374 } catch (err) { 375 console.error(err); 376 nv._country = 'Error'; 377 } 378 379 // check if it already exists: 380 let visitor = model.findVisitor(nv, type); 381 if (!visitor) { 382 visitor = nv; 383 visitor._seenBy = [type]; 384 visitor._pageViews = []; // array of page views 385 visitor._hasReferrer = false; // has at least one referrer 386 visitor._jsClient = false; // visitor has been seen logged by client js as well 387 visitor._client = BotMon.live.data.clients.match(nv.agent) ?? null; // client info 388 visitor._platform = BotMon.live.data.platforms.match(nv.agent); // platform info 389 model._visitors.push(visitor); 390 } else { // update existing 391 if (visitor._firstSeen > nv.ts) { 392 visitor._firstSeen = nv.ts; 393 } 394 } 395 396 // find browser 397 398 // is this visit already registered? 399 let prereg = model._getPageView(visitor, nv); 400 if (!prereg) { 401 // add new page view: 402 prereg = model._makePageView(nv, type); 403 visitor._pageViews.push(prereg); 404 } else { 405 // update last seen date 406 prereg._lastSeen = nv.ts; 407 // increase view count: 408 prereg._viewCount += 1; 409 prereg._tickCount += 1; 410 } 411 412 // update referrer state: 413 visitor._hasReferrer = visitor._hasReferrer || 414 (prereg.ref !== undefined && prereg.ref !== ''); 415 416 // update time stamp for last-seen: 417 if (visitor._lastSeen < nv.ts) { 418 visitor._lastSeen = nv.ts; 419 } 420 421 // if needed: 422 return visitor; 423 }, 424 425 // updating visit data from the client-side log: 426 updateVisit: function(dat) { 427 //console.info('updateVisit', dat); 428 429 // shortcut to make code more readable: 430 const model = BotMon.live.data.model; 431 432 const type = BM_LOGTYPE.CLIENT; 433 434 let visitor = BotMon.live.data.model.findVisitor(dat, type); 435 if (!visitor) { 436 visitor = model.registerVisit(dat, type); 437 } 438 if (visitor) { 439 440 if (visitor._lastSeen < dat.ts) { 441 visitor._lastSeen = dat.ts; 442 } 443 if (!visitor._seenBy.includes(type)) { 444 visitor._seenBy.push(type); 445 } 446 visitor._jsClient = true; // seen by client js 447 } 448 449 // find the page view: 450 let prereg = BotMon.live.data.model._getPageView(visitor, dat); 451 if (prereg) { 452 // update the page view: 453 prereg._lastSeen = dat.ts; 454 if (!prereg._seenBy.includes(type)) prereg._seenBy.push(type); 455 prereg._jsClient = true; // seen by client js 456 } else { 457 // add the page view to the visitor: 458 prereg = model._makePageView(dat, type); 459 visitor._pageViews.push(prereg); 460 } 461 prereg._tickCount += 1; 462 }, 463 464 // updating visit data from the ticker log: 465 updateTicks: function(dat) { 466 //console.info('updateTicks', dat); 467 468 // shortcut to make code more readable: 469 const model = BotMon.live.data.model; 470 471 const type = BM_LOGTYPE.TICKER; 472 473 // find the visit info: 474 let visitor = model.findVisitor(dat, type); 475 if (!visitor) { 476 console.info(`No visitor with ID “${dat.id}” found, registering as a new one.`); 477 visitor = model.registerVisit(dat, type); 478 } 479 if (visitor) { 480 // update visitor: 481 if (visitor._lastSeen < dat.ts) visitor._lastSeen = dat.ts; 482 if (!visitor._seenBy.includes(type)) visitor._seenBy.push(type); 483 484 // get the page view info: 485 let pv = model._getPageView(visitor, dat); 486 if (!pv) { 487 console.info(`No page view for visit ID “${dat.id}”, page “${dat.pg}”, registering a new one.`); 488 pv = model._makePageView(dat, type); 489 visitor._pageViews.push(pv); 490 } 491 492 // update the page view info: 493 if (!pv._seenBy.includes(type)) pv._seenBy.push(type); 494 if (pv._lastSeen.getTime() < dat.ts.getTime()) pv._lastSeen = dat.ts; 495 pv._tickCount += 1; 496 497 } 498 }, 499 500 // helper function to create a new "page view" item: 501 _makePageView: function(data, type) { 502 // console.info('_makePageView', data); 503 504 // try to parse the referrer: 505 let rUrl = null; 506 try { 507 rUrl = ( data.ref && data.ref !== '' ? new URL(data.ref) : null ); 508 } catch (e) { 509 console.warn(`Invalid referer: “${data.ref}”.`); 510 console.info(data); 511 } 512 513 return { 514 _by: type, 515 ip: data.ip, 516 pg: data.pg, 517 lang: data.lang || '??', 518 _ref: rUrl, 519 _firstSeen: data.ts, 520 _lastSeen: data.ts, 521 _seenBy: [type], 522 _jsClient: ( type !== BM_LOGTYPE.SERVER), 523 _viewCount: 1, 524 _tickCount: 0 525 }; 526 } 527 }, 528 529 // functions to analyse the data: 530 analytics: { 531 532 /** 533 * Initializes the analytics data storage object: 534 */ 535 init: function() { 536 //console.info('BotMon.live.data.analytics.init()'); 537 }, 538 539 // data storage: 540 data: { 541 totalVisits: 0, 542 totalPageViews: 0, 543 humanPageViews: 0, 544 bots: { 545 known: 0, 546 suspected: 0, 547 human: 0, 548 users: 0 549 } 550 }, 551 552 // sort the visits by type: 553 groups: { 554 knownBots: [], 555 suspectedBots: [], 556 humans: [], 557 users: [] 558 }, 559 560 // all analytics 561 analyseAll: function() { 562 //console.info('BotMon.live.data.analytics.analyseAll()'); 563 564 // shortcut to make code more readable: 565 const model = BotMon.live.data.model; 566 const me = BotMon.live.data.analytics; 567 568 BotMon.live.gui.status.showBusy("Analysing data …"); 569 570 // loop over all visitors: 571 model._visitors.forEach( (v) => { 572 573 // count visits and page views: 574 this.data.totalVisits += 1; 575 this.data.totalPageViews += v._pageViews.length; 576 577 // check for typical bot aspects: 578 let botScore = 0; 579 580 if (v._type == BM_USERTYPE.KNOWN_BOT) { // known bots 581 582 this.data.bots.known += v._pageViews.length; 583 this.groups.knownBots.push(v); 584 585 } else if (v._type == BM_USERTYPE.KNOWN_USER) { // known users */ 586 587 this.data.bots.users += v._pageViews.length; 588 this.groups.users.push(v); 589 590 } else { 591 592 // get evaluation: 593 const e = BotMon.live.data.rules.evaluate(v); 594 v._eval = e.rules; 595 v._botVal = e.val; 596 597 if (e.isBot) { // likely bots 598 v._type = BM_USERTYPE.LIKELY_BOT; 599 this.data.bots.suspected += v._pageViews.length; 600 this.groups.suspectedBots.push(v); 601 } else { // probably humans 602 v._type = BM_USERTYPE.PROBABLY_HUMAN; 603 this.data.bots.human += v._pageViews.length; 604 this.groups.humans.push(v); 605 } 606 } 607 608 // perform actions depending on the visitor type: 609 if (v._type == BM_USERTYPE.KNOWN_BOT ) { /* known bots only */ 610 611 } else if (v._type == BM_USERTYPE.LIKELY_BOT) { /* probable bots only */ 612 613 // add bot views to IP range information: 614 me.addToIpRanges(v); 615 616 } else { /* humans only */ 617 618 // add browser and platform statistics: 619 me.addBrowserPlatform(v); 620 621 // add 622 v._pageViews.forEach( pv => { 623 me.addToRefererList(pv._ref); 624 me.addToPagesList(pv.pg); 625 }); 626 } 627 628 // add to the country lists: 629 me.addToCountries(v.geo, v._country, v._type); 630 631 }); 632 633 BotMon.live.gui.status.hideBusy('Done.'); 634 }, 635 636 // get a list of known bots: 637 getTopBots: function(max) { 638 //console.info('BotMon.live.data.analytics.getTopBots('+max+')'); 639 640 //console.log(BotMon.live.data.analytics.groups.knownBots); 641 642 let botsList = BotMon.live.data.analytics.groups.knownBots.toSorted( (a, b) => { 643 return b._pageViews.length - a._pageViews.length; 644 }); 645 646 const other = { 647 'id': 'other', 648 'name': "Others", 649 'count': 0 650 }; 651 652 const rList = []; 653 const max2 = ( botsList.length > max ? max-1 : botsList.length ); 654 let total = 0; // adding up the items 655 for (let i=0; i<botsList.length; i++) { 656 const it = botsList[i]; 657 if (it && it._bot) { 658 if (i < max2) { 659 rList.push({ 660 id: it._bot.id, 661 name: (it._bot.n ? it._bot.n : it._bot.id), 662 count: it._pageViews.length 663 }); 664 } else { 665 other.count += it._pageViews.length; 666 }; 667 total += it._pageViews.length; 668 } 669 }; 670 671 // add the "other" item, if needed: 672 if (botsList.length > max2) { 673 rList.push(other); 674 }; 675 676 rList.forEach( it => { 677 it.pct = (it.count * 100 / total); 678 }); 679 680 return rList; 681 }, 682 683 // most visited pages list: 684 _pagesList: [], 685 686 /** 687 * Add a page view to the list of most visited pages. 688 * @param {string} pageId - The page ID to add to the list. 689 * @example 690 * BotMon.live.data.analytics.addToPagesList('1234567890'); 691 */ 692 addToPagesList: function(pageId) { 693 //console.log('BotMon.live.data.analytics.addToPagesList', pageId); 694 695 const me = BotMon.live.data.analytics; 696 697 // already exists? 698 let pgObj = null; 699 for (let i = 0; i < me._pagesList.length; i++) { 700 if (me._pagesList[i].id == pageId) { 701 pgObj = me._pagesList[i]; 702 break; 703 } 704 } 705 706 // if not exists, create it: 707 if (!pgObj) { 708 pgObj = { 709 id: pageId, 710 count: 1 711 }; 712 me._pagesList.push(pgObj); 713 } else { 714 pgObj.count += 1; 715 } 716 }, 717 718 getTopPages: function(max) { 719 //console.info('BotMon.live.data.analytics.getTopPages('+max+')'); 720 const me = BotMon.live.data.analytics; 721 return me._pagesList.toSorted( (a, b) => { 722 return b.count - a.count; 723 }).slice(0,max); 724 }, 725 726 // Referer List: 727 _refererList: [], 728 729 addToRefererList: function(ref) { 730 //console.log('BotMon.live.data.analytics.addToRefererList',ref); 731 732 const me = BotMon.live.data.analytics; 733 734 // ignore internal references: 735 if (ref && ref.host == window.location.host) { 736 return; 737 } 738 739 const refInfo = me.getRefererInfo(ref); 740 741 // already exists? 742 let refObj = null; 743 for (let i = 0; i < me._refererList.length; i++) { 744 if (me._refererList[i].id == refInfo.id) { 745 refObj = me._refererList[i]; 746 break; 747 } 748 } 749 750 // if not exists, create it: 751 if (!refObj) { 752 refObj = refInfo; 753 refObj.count = 1; 754 me._refererList.push(refObj); 755 } else { 756 refObj.count += 1; 757 } 758 }, 759 760 getRefererInfo: function(url) { 761 //console.log('BotMon.live.data.analytics.getRefererInfo',url); 762 try { 763 url = new URL(url); 764 } catch (e) { 765 return { 766 'id': 'null', 767 'n': 'Invalid Referer' 768 }; 769 } 770 771 // find the referer ID: 772 let refId = 'null'; 773 let refName = 'No Referer'; 774 if (url && url.host) { 775 const hArr = url.host.split('.'); 776 const tld = hArr[hArr.length-1]; 777 refId = ( tld == 'localhost' ? tld : hArr[hArr.length-2]); 778 refName = hArr[hArr.length-2] + '.' + tld; 779 } 780 781 return { 782 'id': refId, 783 'n': refName 784 }; 785 }, 786 787 /** 788 * Get a sorted list of the top referers. 789 * The list is sorted in descending order of count. 790 * If the array has more items than the given maximum, the rest of the items are added to an "other" item. 791 * Each item in the list has a "pct" property, which is the percentage of the total count. 792 * @param {number} max - The maximum number of items to return. 793 * @return {Array} The sorted list of top referers. 794 */ 795 getTopReferers: function(max) { 796 //console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')')); 797 798 const me = BotMon.live.data.analytics; 799 800 return me._makeTopList(me._refererList, max); 801 }, 802 803 /** 804 * Create a sorted list of top items from a given array. 805 * The list is sorted in descending order of count. 806 * If the array has more items than the given maximum, the rest of the items are added to an "other" item. 807 * Each item in the list has a "pct" property, which is the percentage of the total count. 808 * @param {Array} arr - The array to sort and truncate. 809 * @param {number} max - The maximum number of items to return. 810 * @return {Array} The sorted list of top items. 811 */ 812 _makeTopList: function(arr, max) { 813 //console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')')); 814 815 const me = BotMon.live.data.analytics; 816 817 // sort the list: 818 arr.sort( (a,b) => { 819 return b.count - a.count; 820 }); 821 822 const rList = []; // return array 823 const max2 = ( arr.length >= max ? max-1 : arr.length ); 824 const other = { 825 'id': 'other', 826 'name': "Others", 827 'typ': 'other', 828 'count': 0 829 }; 830 let total = 0; // adding up the items 831 for (let i=0; Math.min(max, arr.length) > i; i++) { 832 const it = arr[i]; 833 if (it) { 834 if (i < max2) { 835 const rIt = { 836 id: it.id, 837 name: (it.n ? it.n : it.id), 838 typ: it.typ || it.id, 839 count: it.count 840 }; 841 rList.push(rIt); 842 } else { 843 other.count += it.count; 844 } 845 total += it.count; 846 } 847 } 848 849 // add the "other" item, if needed: 850 if (arr.length > max2) { 851 rList.push(other); 852 }; 853 854 rList.forEach( it => { 855 it.pct = (it.count * 100 / total); 856 }); 857 858 return rList; 859 }, 860 861 /* countries of visits */ 862 _countries: { 863 'human': [], 864 'bot': [] 865 }, 866 /** 867 * Adds a country code to the statistics. 868 * 869 * @param {string} iso The ISO 3166-1 alpha-2 country code. 870 */ 871 addToCountries: function(iso, name, type) { 872 873 const me = BotMon.live.data.analytics; 874 875 // find the correct array: 876 let arr = null; 877 switch (type) { 878 879 case BM_USERTYPE.KNOWN_USER: 880 case BM_USERTYPE.PROBABLY_HUMAN: 881 arr = me._countries.human; 882 break; 883 case BM_USERTYPE.LIKELY_BOT: 884 case BM_USERTYPE.KNOWN_BOT: 885 arr = me._countries.bot; 886 break; 887 default: 888 console.warn(`Unknown user type ${type} in function addToCountries.`); 889 } 890 891 if (arr) { 892 let cRec = arr.find( it => it.id == iso); 893 if (!cRec) { 894 cRec = { 895 'id': iso, 896 'n': name, 897 'count': 1 898 }; 899 arr.push(cRec); 900 } else { 901 cRec.count += 1; 902 } 903 } 904 }, 905 906 /** 907 * Returns a list of countries with visit counts, sorted by visit count in descending order. 908 * 909 * @param {BM_USERTYPE} type array of types type of visitors to return. 910 * @param {number} max The maximum number of entries to return. 911 * @return {Array} A list of objects with properties 'iso' (ISO 3166-1 alpha-2 country code) and 'count' (visit count). 912 */ 913 getCountryList: function(type, max) { 914 915 const me = BotMon.live.data.analytics; 916 917 // find the correct array: 918 let arr = null; 919 switch (type) { 920 921 case 'human': 922 arr = me._countries.human; 923 break; 924 case 'bot': 925 arr = me._countries.bot; 926 break; 927 default: 928 console.warn(`Unknown user type ${type} in function getCountryList.`); 929 return; 930 } 931 932 return me._makeTopList(arr, max); 933 }, 934 935 /* browser and platform of human visitors */ 936 _browsers: [], 937 _platforms: [], 938 939 addBrowserPlatform: function(visitor) { 940 //console.info('addBrowserPlatform', visitor); 941 942 const me = BotMon.live.data.analytics; 943 944 // add to browsers list: 945 let browserRec = ( visitor._client ? visitor._client : {'id': 'unknown'}); 946 if (visitor._client) { 947 let bRec = me._browsers.find( it => it.id == browserRec.id); 948 if (!bRec) { 949 bRec = { 950 id: browserRec.id, 951 n: browserRec.n, 952 count: 1 953 }; 954 me._browsers.push(bRec); 955 } else { 956 bRec.count += 1; 957 } 958 } 959 960 // add to platforms list: 961 let platformRec = ( visitor._platform ? visitor._platform : {'id': 'unknown'}); 962 if (visitor._platform) { 963 let pRec = me._platforms.find( it => it.id == platformRec.id); 964 if (!pRec) { 965 pRec = { 966 id: platformRec.id, 967 n: platformRec.n, 968 count: 1 969 }; 970 me._platforms.push(pRec); 971 } else { 972 pRec.count += 1; 973 } 974 } 975 976 }, 977 978 getTopBrowsers: function(max) { 979 980 const me = BotMon.live.data.analytics; 981 982 return me._makeTopList(me._browsers, max); 983 }, 984 985 getTopPlatforms: function(max) { 986 987 const me = BotMon.live.data.analytics; 988 989 return me._makeTopList(me._platforms, max); 990 }, 991 992 /* bounces are counted, not calculates: */ 993 getBounceCount: function(type) { 994 995 const me = BotMon.live.data.analytics; 996 var bounces = 0; 997 const list = me.groups[type]; 998 999 list.forEach(it => { 1000 bounces += (it._pageViews.length <= 1 ? 1 : 0); 1001 }); 1002 1003 return bounces; 1004 }, 1005 1006 _ipRanges: [], 1007 1008 /* adds a visit to the ip ranges arrays */ 1009 addToIpRanges: function(v) { 1010 //console.info('addToIpRanges', v.ip); 1011 1012 const me = BotMon.live.data.analytics; 1013 const ipRanges = BotMon.live.data.ipRanges; 1014 1015 // Number of IP address segments to look at: 1016 const kIP4Segments = 1; 1017 const kIP6Segments = 3; 1018 1019 let ipGroup = ''; // group name 1020 let ipSeg = []; // IP segment array 1021 let rawIP = ''; // raw ip prefix 1022 let ipName = ''; // IP group display name 1023 1024 const ipType = v.ip.indexOf(':') > 0 ? BM_IPVERSION.IPv6 : BM_IPVERSION.IPv4; 1025 const ipAddr = BotMon.t._ip2Num(v.ip); 1026 1027 // is there already a known IP range assigned? 1028 if (v._ipRange) { 1029 1030 ipGroup = v._ipRange.g; // group name 1031 ipName = ipRanges.getOwner( v._ipRange.g ) || "Unknown"; 1032 1033 } else { // no known IP range, let's collect necessary information: 1034 1035 1036 // collect basic IP address info: 1037 if (ipType == BM_IPVERSION.IPv6) { 1038 ipSeg = ipAddr.split(':'); 1039 const prefix = v.ip.split(':').slice(0, kIP6Segments).join(':'); 1040 rawIP = ipSeg.slice(0, kIP6Segments).join(':'); 1041 ipGroup = 'ip6-' + rawIP.replaceAll(':', '-'); 1042 ipName = prefix + '::' + '/' + (16 * kIP6Segments); 1043 } else { 1044 ipSeg = ipAddr.split('.'); 1045 const prefix = v.ip.split('.').slice(0, kIP4Segments).join('.'); 1046 rawIP = ipSeg.slice(0, kIP4Segments).join('.') ; 1047 ipGroup = 'ip4-' + rawIP.replaceAll('.', '-'); 1048 ipName = prefix + '.x.x.x'.substring(0, 1+(4-kIP4Segments)*2) + '/' + (8 * kIP4Segments); 1049 } 1050 } 1051 1052 // check if record already exists: 1053 let ipRec = me._ipRanges.find( it => it.g == ipGroup); 1054 if (!ipRec) { 1055 1056 // ip info record initialised: 1057 ipRec = { 1058 g: ipGroup, 1059 n: ipName, 1060 count: 0 1061 } 1062 1063 // existing record? 1064 if (v._ipRange) { 1065 1066 ipRec.from = v._ipRange.from; 1067 ipRec.to = v._ipRange.to; 1068 ipRec.typ = 'net'; 1069 1070 } else { // no known IP range, let's collect necessary information: 1071 1072 // complete the ip info record: 1073 if (ipType == BM_IPVERSION.IPv6) { 1074 ipRec.from = rawIP + ':0000:0000:0000:0000:0000:0000:0000'.substring(0, (8-kIP6Segments)*5); 1075 ipRec.to = rawIP + ':FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF'.substring(0, (8-kIP6Segments)*5); 1076 ipRec.typ = '6'; 1077 } else { 1078 ipRec.from = rawIP + '.000.000.000.000'.substring(0, (4-kIP4Segments)*4); 1079 ipRec.to = rawIP + '.255.255.255.254'.substring(0, (4-kIP4Segments)*4); 1080 ipRec.typ = '4'; 1081 } 1082 } 1083 1084 me._ipRanges.push(ipRec); 1085 } 1086 1087 // add to counter: 1088 ipRec.count += v._pageViews.length; 1089 1090 }, 1091 1092 getTopBotISPs: function(max) { 1093 1094 const me = BotMon.live.data.analytics; 1095 1096 return me._makeTopList(me._ipRanges, max); 1097 }, 1098 }, 1099 1100 // information on "known bots": 1101 bots: { 1102 // loads the list of known bots from a JSON file: 1103 init: async function() { 1104 //console.info('BotMon.live.data.bots.init()'); 1105 1106 // Load the list of known bots: 1107 BotMon.live.gui.status.showBusy("Loading known bots …"); 1108 const url = BotMon._baseDir + 'config/known-bots.json'; 1109 try { 1110 const response = await fetch(url); 1111 if (!response.ok) { 1112 throw new Error(`${response.status} ${response.statusText}`); 1113 } 1114 1115 this._list = await response.json(); 1116 this._ready = true; 1117 1118 } catch (error) { 1119 BotMon.live.gui.status.setError("Error while loading the known bots file:", error.message); 1120 } finally { 1121 BotMon.live.gui.status.hideBusy("Status: Done."); 1122 BotMon.live.data._dispatch('bots') 1123 } 1124 }, 1125 1126 // returns bot info if the clientId matches a known bot, null otherwise: 1127 match: function(agent) { 1128 //console.info('BotMon.live.data.bots.match(',agent,')'); 1129 1130 const BotList = BotMon.live.data.bots._list; 1131 1132 // default is: not found! 1133 let botInfo = null; 1134 1135 if (!agent) return null; 1136 1137 // check for known bots: 1138 BotList.find(bot => { 1139 let r = false; 1140 for (let j=0; j<bot.rx.length; j++) { 1141 const rxr = agent.match(new RegExp(bot.rx[j])); 1142 if (rxr) { 1143 botInfo = { 1144 n : bot.n, 1145 id: bot.id, 1146 geo: (bot.geo ? bot.geo : null), 1147 url: bot.url, 1148 v: (rxr.length > 1 ? rxr[1] : -1) 1149 }; 1150 r = true; 1151 break; 1152 }; 1153 }; 1154 return r; 1155 }); 1156 1157 // check for unknown bots: 1158 if (!botInfo) { 1159 const botmatch = agent.match(/([\s\d\w\-]*bot|[\s\d\w\-]*crawler|[\s\d\w\-]*spider)[\/\s\w\-;\),\\.$]/i); 1160 if(botmatch) { 1161 botInfo = {'id': ( botmatch[1] || "other_" ), 'n': "Other" + ( botmatch[1] ? " (" + botmatch[1] + ")" : "" ) , "bot": botmatch[1] }; 1162 } 1163 } 1164 1165 //console.log("botInfo:", botInfo); 1166 return botInfo; 1167 }, 1168 1169 1170 // indicates if the list is loaded and ready to use: 1171 _ready: false, 1172 1173 // the actual bot list is stored here: 1174 _list: [] 1175 }, 1176 1177 // information on known clients (browsers): 1178 clients: { 1179 // loads the list of known clients from a JSON file: 1180 init: async function() { 1181 //console.info('BotMon.live.data.clients.init()'); 1182 1183 // Load the list of known bots: 1184 BotMon.live.gui.status.showBusy("Loading known clients"); 1185 const url = BotMon._baseDir + 'config/known-clients.json'; 1186 try { 1187 const response = await fetch(url); 1188 if (!response.ok) { 1189 throw new Error(`${response.status} ${response.statusText}`); 1190 } 1191 1192 BotMon.live.data.clients._list = await response.json(); 1193 BotMon.live.data.clients._ready = true; 1194 1195 } catch (error) { 1196 BotMon.live.gui.status.setError("Error while loading the known clients file: " + error.message); 1197 } finally { 1198 BotMon.live.gui.status.hideBusy("Status: Done."); 1199 BotMon.live.data._dispatch('clients') 1200 } 1201 }, 1202 1203 // returns bot info if the user-agent matches a known bot, null otherwise: 1204 match: function(agent) { 1205 //console.info('BotMon.live.data.clients.match(',agent,')'); 1206 1207 let match = {"n": "Unknown", "v": -1, "id": 'null'}; 1208 1209 if (agent) { 1210 BotMon.live.data.clients._list.find(client => { 1211 let r = false; 1212 for (let j=0; j<client.rx.length; j++) { 1213 const rxr = agent.match(new RegExp(client.rx[j])); 1214 if (rxr) { 1215 match.n = client.n; 1216 match.v = (rxr.length > 1 ? rxr[1] : -1); 1217 match.id = client.id || null; 1218 r = true; 1219 break; 1220 } 1221 } 1222 return r; 1223 }); 1224 } 1225 1226 //console.log(match) 1227 return match; 1228 }, 1229 1230 // return the browser name for a browser ID: 1231 getName: function(id) { 1232 const it = BotMon.live.data.clients._list.find(client => client.id == id); 1233 return ( it && it.n ? it.n : "Unknown"); //it.n; 1234 }, 1235 1236 // indicates if the list is loaded and ready to use: 1237 _ready: false, 1238 1239 // the actual bot list is stored here: 1240 _list: [] 1241 1242 }, 1243 1244 // information on known platforms (operating systems): 1245 platforms: { 1246 // loads the list of known platforms from a JSON file: 1247 init: async function() { 1248 //console.info('BotMon.live.data.platforms.init()'); 1249 1250 // Load the list of known bots: 1251 BotMon.live.gui.status.showBusy("Loading known platforms"); 1252 const url = BotMon._baseDir + 'config/known-platforms.json'; 1253 try { 1254 const response = await fetch(url); 1255 if (!response.ok) { 1256 throw new Error(`${response.status} ${response.statusText}`); 1257 } 1258 1259 BotMon.live.data.platforms._list = await response.json(); 1260 BotMon.live.data.platforms._ready = true; 1261 1262 } catch (error) { 1263 BotMon.live.gui.status.setError("Error while loading the known platforms file: " + error.message); 1264 } finally { 1265 BotMon.live.gui.status.hideBusy("Status: Done."); 1266 BotMon.live.data._dispatch('platforms') 1267 } 1268 }, 1269 1270 // returns bot info if the browser id matches a known platform: 1271 match: function(cid) { 1272 //console.info('BotMon.live.data.platforms.match(',cid,')'); 1273 1274 let match = {"n": "Unknown", "id": 'null'}; 1275 1276 if (cid) { 1277 BotMon.live.data.platforms._list.find(platform => { 1278 let r = false; 1279 for (let j=0; j<platform.rx.length; j++) { 1280 const rxr = cid.match(new RegExp(platform.rx[j])); 1281 if (rxr) { 1282 match.n = platform.n; 1283 match.v = (rxr.length > 1 ? rxr[1] : -1); 1284 match.id = platform.id || null; 1285 r = true; 1286 break; 1287 } 1288 } 1289 return r; 1290 }); 1291 } 1292 1293 return match; 1294 }, 1295 1296 // return the platform name for a given ID: 1297 getName: function(id) { 1298 const it = BotMon.live.data.platforms._list.find( pf => pf.id == id); 1299 return ( it ? it.n : 'Unknown' ); 1300 }, 1301 1302 1303 // indicates if the list is loaded and ready to use: 1304 _ready: false, 1305 1306 // the actual bot list is stored here: 1307 _list: [] 1308 1309 }, 1310 1311 // storage and functions for the known bot IP-Ranges: 1312 ipRanges: { 1313 1314 init: function() { 1315 //console.log('BotMon.live.data.ipRanges.init()'); 1316 // #TODO: Load from separate IP-Ranges file 1317 // load the rules file: 1318 const me = BotMon.live.data; 1319 1320 try { 1321 BotMon.live.data._loadSettingsFile(['user-ipranges', 'known-ipranges'], 1322 (json) => { 1323 1324 // groups can be just saved in the data structure: 1325 if (json.groups && json.groups.constructor.name == 'Array') { 1326 me.ipRanges._groups = json.groups; 1327 } 1328 1329 // groups can be just saved in the data structure: 1330 if (json.ranges && json.ranges.constructor.name == 'Array') { 1331 json.ranges.forEach(range => { 1332 me.ipRanges.add(range); 1333 }) 1334 } 1335 1336 // finished loading 1337 BotMon.live.gui.status.hideBusy("Status: Done."); 1338 BotMon.live.data._dispatch('ipranges') 1339 }); 1340 } catch (error) { 1341 BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); 1342 } 1343 }, 1344 1345 // the actual bot list is stored here: 1346 _list: [], 1347 _groups: [], 1348 1349 add: function(data) { 1350 //console.log('BotMon.live.data.ipRanges.add(',data,')'); 1351 1352 const me = BotMon.live.data.ipRanges; 1353 1354 // convert IP address to easier comparable form: 1355 const ip2Num = BotMon.t._ip2Num; 1356 1357 let item = { 1358 'cidr': data.from.replaceAll(/::+/g, '::') + '/' + ( data.m ? data.m : '??' ), 1359 'from': ip2Num(data.from), 1360 'to': ip2Num(data.to), 1361 'm': data.m, 1362 'g': data.g 1363 }; 1364 me._list.push(item); 1365 1366 }, 1367 1368 getOwner: function(gid) { 1369 1370 const me = BotMon.live.data.ipRanges; 1371 1372 for (let i=0; i < me._groups.length; i++) { 1373 const it = me._groups[i]; 1374 if (it.id == gid) { 1375 return it.name; 1376 } 1377 } 1378 return null; 1379 }, 1380 1381 match: function(ip) { 1382 //console.log('BotMon.live.data.ipRanges.match(',ip,')'); 1383 1384 const me = BotMon.live.data.ipRanges; 1385 1386 // convert IP address to easier comparable form: 1387 const ipNum = BotMon.t._ip2Num(ip); 1388 1389 for (let i=0; i < me._list.length; i++) { 1390 const ipRange = me._list[i]; 1391 1392 if (ipNum >= ipRange.from && ipNum <= ipRange.to) { 1393 return ipRange; 1394 } 1395 1396 }; 1397 return null; 1398 } 1399 }, 1400 1401 // storage for the rules and related functions 1402 rules: { 1403 1404 /** 1405 * Initializes the rules data. 1406 * 1407 * Loads the default config file and the user config file (if present). 1408 * The default config file is used if the user config file does not have a certain setting. 1409 * The user config file can override settings from the default config file. 1410 * 1411 * The rules are loaded from the `rules` property of the config files. 1412 * The IP ranges are loaded from the `ipRanges` property of the config files. 1413 * 1414 * If an error occurs while loading the config file, it is displayed in the status bar. 1415 * After the config file is loaded, the status bar is hidden. 1416 */ 1417 init: async function() { 1418 //console.info('BotMon.live.data.rules.init()'); 1419 1420 // Load the list of known bots: 1421 BotMon.live.gui.status.showBusy("Loading list of rules …"); 1422 1423 // load the rules file: 1424 const me = BotMon.live.data; 1425 1426 try { 1427 BotMon.live.data._loadSettingsFile(['user-config', 'default-config'], 1428 (json) => { 1429 1430 // override the threshold? 1431 if (json.threshold) me._threshold = json.threshold; 1432 1433 // set the rules list: 1434 if (json.rules && json.rules.constructor.name == 'Array') { 1435 me.rules._rulesList = json.rules; 1436 } 1437 1438 BotMon.live.gui.status.hideBusy("Status: Done."); 1439 BotMon.live.data._dispatch('rules') 1440 } 1441 ); 1442 } catch (error) { 1443 BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); 1444 } 1445 }, 1446 1447 _rulesList: [], // list of rules to find out if a visitor is a bot 1448 _threshold: 100, // above this, it is considered a bot. 1449 1450 // returns a descriptive text for a rule id 1451 getRuleInfo: function(ruleId) { 1452 // console.info('getRuleInfo', ruleId); 1453 1454 // shortcut for neater code: 1455 const me = BotMon.live.data.rules; 1456 1457 for (let i=0; i<me._rulesList.length; i++) { 1458 const rule = me._rulesList[i]; 1459 if (rule.id == ruleId) { 1460 return rule; 1461 } 1462 } 1463 return null; 1464 1465 }, 1466 1467 // evaluate a visitor for lkikelihood of being a bot 1468 evaluate: function(visitor) { 1469 1470 // shortcut for neater code: 1471 const me = BotMon.live.data.rules; 1472 1473 let r = { // evaluation result 1474 'val': 0, 1475 'rules': [], 1476 'isBot': false 1477 }; 1478 1479 for (let i=0; i<me._rulesList.length; i++) { 1480 const rule = me._rulesList[i]; 1481 const params = ( rule.params ? rule.params : [] ); 1482 1483 if (rule.func) { // rule is calling a function 1484 if (me.func[rule.func]) { 1485 if(me.func[rule.func](visitor, ...params)) { 1486 r.val += rule.bot; 1487 r.rules.push(rule.id) 1488 } 1489 } else { 1490 //console.warn("Unknown rule function: “${rule.func}”. Ignoring rule.") 1491 } 1492 } 1493 } 1494 1495 // is a bot? 1496 r.isBot = (r.val >= me._threshold); 1497 1498 return r; 1499 }, 1500 1501 // list of functions that can be called by the rules list to evaluate a visitor: 1502 func: { 1503 1504 // check if client is on the list passed as parameter: 1505 matchesClient: function(visitor, ...clients) { 1506 1507 const clientId = ( visitor._client ? visitor._client.id : ''); 1508 return clients.includes(clientId); 1509 }, 1510 1511 // check if OS/Platform is one of the obsolete ones: 1512 matchesPlatform: function(visitor, ...platforms) { 1513 1514 const pId = ( visitor._platform ? visitor._platform.id : ''); 1515 1516 if (visitor._platform.id == null) console.log(visitor._platform); 1517 1518 return platforms.includes(pId); 1519 }, 1520 1521 // are there at lest num pages loaded? 1522 smallPageCount: function(visitor, num) { 1523 return (visitor._pageViews.length <= Number(num)); 1524 }, 1525 1526 // There was no entry in a specific log file for this visitor: 1527 // note that this will also trigger the "noJavaScript" rule: 1528 noRecord: function(visitor, type) { 1529 return !visitor._seenBy.includes(type); 1530 }, 1531 1532 // there are no referrers in any of the page visits: 1533 noReferrer: function(visitor) { 1534 1535 let r = false; // return value 1536 for (let i = 0; i < visitor._pageViews.length; i++) { 1537 if (!visitor._pageViews[i]._ref) { 1538 r = true; 1539 break; 1540 } 1541 } 1542 return r; 1543 }, 1544 1545 // test for specific client identifiers: 1546 matchesUserAgent: function(visitor, ...list) { 1547 1548 for (let i=0; i<list.length; i++) { 1549 if (visitor.agent == list[i]) { 1550 return true 1551 } 1552 }; 1553 return false; 1554 }, 1555 1556 // unusual combinations of Platform and Client: 1557 combinationTest: function(visitor, ...combinations) { 1558 1559 for (let i=0; i<combinations.length; i++) { 1560 1561 if (visitor._platform.id == combinations[i][0] 1562 && visitor._client.id == combinations[i][1]) { 1563 return true 1564 } 1565 }; 1566 1567 return false; 1568 }, 1569 1570 // is the IP address from a known bot network? 1571 fromKnownBotIP: function(visitor) { 1572 //console.info('fromKnownBotIP()', visitor.ip); 1573 1574 const ipInfo = BotMon.live.data.ipRanges.match(visitor.ip); 1575 1576 if (ipInfo) { 1577 visitor._ipInKnownBotRange = true; 1578 visitor._ipRange = ipInfo; 1579 } 1580 1581 return (ipInfo !== null); 1582 }, 1583 1584 // is the page language mentioned in the client's accepted languages? 1585 // the parameter holds an array of exceptions, i.e. page languages that should be ignored. 1586 matchLang: function(visitor, ...exceptions) { 1587 1588 if (visitor.lang && visitor.accept && exceptions.indexOf(visitor.lang) < 0) { 1589 return (visitor.accept.split(',').indexOf(visitor.lang) < 0); 1590 } 1591 return false; 1592 }, 1593 1594 // the "Accept language" header contains certain entries: 1595 clientAccepts: function(visitor, ...languages) { 1596 //console.info('clientAccepts', visitor.accept, languages); 1597 1598 if (visitor.accept && languages) {; 1599 return ( visitor.accept.split(',').filter(lang => languages.includes(lang)).length > 0 ); 1600 } 1601 return false; 1602 }, 1603 1604 // Is there an accept-language field defined at all? 1605 noAcceptLang: function(visitor) { 1606 1607 if (!visitor.accept || visitor.accept.length <= 0) { // no accept-languages header 1608 return true; 1609 } 1610 // TODO: parametrize this! 1611 return false; 1612 }, 1613 // At least x page views were recorded, but they come within less than y seconds 1614 loadSpeed: function(visitor, minItems, maxTime) { 1615 1616 if (visitor._pageViews.length >= minItems) { 1617 //console.log('loadSpeed', visitor._pageViews.length, minItems, maxTime); 1618 1619 const pvArr = visitor._pageViews.map(pv => pv._lastSeen).sort(); 1620 1621 let totalTime = 0; 1622 for (let i=1; i < pvArr.length; i++) { 1623 totalTime += (pvArr[i] - pvArr[i-1]); 1624 } 1625 1626 //console.log(' ', totalTime , Math.round(totalTime / (pvArr.length * 1000)), (( totalTime / pvArr.length ) <= maxTime * 1000), visitor.ip); 1627 1628 return (( totalTime / pvArr.length ) <= maxTime * 1000); 1629 } 1630 }, 1631 1632 // Country code matches one of those in the list: 1633 matchesCountry: function(visitor, ...countries) { 1634 1635 // ingore if geoloc is not set or unknown: 1636 if (visitor.geo) { 1637 return (countries.indexOf(visitor.geo) >= 0); 1638 } 1639 return false; 1640 }, 1641 1642 // Country does not match one of the given codes. 1643 notFromCountry: function(visitor, ...countries) { 1644 1645 // ingore if geoloc is not set or unknown: 1646 if (visitor.geo && visitor.geo !== 'ZZ') { 1647 return (countries.indexOf(visitor.geo) < 0); 1648 } 1649 return false; 1650 } 1651 } 1652 }, 1653 1654 /** 1655 * Loads a settings file from the specified list of filenames. 1656 * If the file is successfully loaded, it will call the callback function 1657 * with the loaded JSON data. 1658 * If no file can be loaded, it will display an error message. 1659 * 1660 * @param {string[]} fns - list of filenames to load 1661 * @param {function} callback - function to call with the loaded JSON data 1662 */ 1663 _loadSettingsFile: async function(fns, callback) { 1664 //console.info('BotMon.live.data._loadSettingsFile()', fns); 1665 1666 const kJsonExt = '.json'; 1667 let loaded = false; // if successfully loaded file 1668 1669 for (let i=0; i<fns.length; i++) { 1670 const filename = fns[i] +kJsonExt; 1671 try { 1672 const response = await fetch(DOKU_BASE + 'lib/plugins/botmon/config/' + filename); 1673 if (!response.ok) { 1674 continue; 1675 } else { 1676 loaded = true; 1677 } 1678 const json = await response.json(); 1679 if (callback && typeof callback === 'function') { 1680 callback(json); 1681 } 1682 break; 1683 } catch (e) { 1684 BotMon.live.gui.status.setError("Error while loading the config file: " + filename); 1685 } 1686 } 1687 1688 if (!loaded) { 1689 BotMon.live.gui.status.setError("Could not load a config file."); 1690 } 1691 }, 1692 1693 /** 1694 * Loads a log file (server, page load, or ticker) and parses it. 1695 * @param {String} type - the type of the log file to load (srv, log, or tck) 1696 * @param {Function} [onLoaded] - an optional callback function to call after loading is finished. 1697 */ 1698 loadLogFile: async function(type, onLoaded = undefined) { 1699 //console.info('BotMon.live.data.loadLogFile(',type,')'); 1700 1701 let typeName = ''; 1702 let columns = []; 1703 1704 switch (type) { 1705 case "srv": 1706 typeName = "Server"; 1707 columns = ['ts','ip','pg','id','typ','usr','agent','ref','lang','accept','geo']; 1708 break; 1709 case "log": 1710 typeName = "Page load"; 1711 columns = ['ts','ip','pg','id','usr','lt','ref','agent']; 1712 break; 1713 case "tck": 1714 typeName = "Ticker"; 1715 columns = ['ts','ip','pg','id','agent']; 1716 break; 1717 default: 1718 console.warn(`Unknown log type ${type}.`); 1719 return; 1720 } 1721 1722 // Show the busy indicator and set the visible status: 1723 BotMon.live.gui.status.showBusy(`Loading ${typeName} log file …`); 1724 1725 // compose the URL from which to load: 1726 const url = BotMon._baseDir + `logs/${BotMon._datestr}.${type}.txt`; 1727 //console.log("Loading:",url); 1728 1729 // fetch the data: 1730 try { 1731 const response = await fetch(url); 1732 if (!response.ok) { 1733 1734 throw new Error(`${response.status} ${response.statusText}`); 1735 1736 } else { 1737 1738 // parse the data: 1739 const logtxt = await response.text(); 1740 if (logtxt.length <= 0) { 1741 throw new Error(`Empty log file ${url}.`); 1742 } 1743 1744 logtxt.split('\n').forEach((line) => { 1745 if (line.trim() === '') return; // skip empty lines 1746 const cols = line.split('\t'); 1747 1748 // assign the columns to an object: 1749 const data = {}; 1750 cols.forEach( (colVal,i) => { 1751 const colName = columns[i] || `col${i}`; 1752 const colValue = (colName == 'ts' ? new Date(colVal) : colVal.trim()); 1753 data[colName] = colValue; 1754 }); 1755 1756 // register the visit in the model: 1757 switch(type) { 1758 case BM_LOGTYPE.SERVER: 1759 BotMon.live.data.model.registerVisit(data, type); 1760 break; 1761 case BM_LOGTYPE.CLIENT: 1762 data.typ = 'js'; 1763 BotMon.live.data.model.updateVisit(data); 1764 break; 1765 case BM_LOGTYPE.TICKER: 1766 data.typ = 'js'; 1767 BotMon.live.data.model.updateTicks(data); 1768 break; 1769 default: 1770 console.warn(`Unknown log type ${type}.`); 1771 return; 1772 } 1773 }); 1774 } 1775 1776 } catch (error) { 1777 BotMon.live.gui.status.setError(`Error while loading the ${typeName} log file: ${error.message} – data may be incomplete.`); 1778 } finally { 1779 BotMon.live.gui.status.hideBusy("Status: Done."); 1780 if (onLoaded) { 1781 onLoaded(); // callback after loading is finished. 1782 } 1783 } 1784 } 1785 }, 1786 1787 gui: { 1788 init: function() { 1789 //console.log('BotMon.live.gui.init()'); 1790 1791 // init sub-objects: 1792 BotMon.t._callInit(this); 1793 }, 1794 1795 tabs: { 1796 init: function() { 1797 //console.log('BotMon.live.gui.tabs.init()'); 1798 1799 /* find and add all existing tabs */ 1800 document.querySelectorAll('#botmon__admin *[role=tablist]') 1801 .forEach((tablist) => { 1802 tablist.querySelectorAll('*[role=tab]') 1803 .forEach( t => t.addEventListener('click', this._onTabClick) ) 1804 }); 1805 }, 1806 1807 /* callback for tab click */ 1808 _onTabClick: function(e) { 1809 //console.log('BotMon.live.gui.tabs._onTabClick()'); 1810 1811 /* reusable constants: */ 1812 const kAriaSelected = 'aria-selected'; 1813 const kAriaControls = 'aria-controls'; 1814 const kTrue = 'true'; 1815 const kFalse = 'false'; 1816 const kHidden = 'hidden'; 1817 1818 /* cancel default action */ 1819 e.preventDefault(); 1820 1821 /* if the active tab is clicked, do nothing: */ 1822 let selState = this.getAttribute(kAriaSelected); 1823 if ( selState && selState == kTrue ) { 1824 return; 1825 } 1826 1827 /* find the active tab element: */ 1828 var aItem = null; 1829 let tablist = this.parentNode; 1830 while (tablist.getAttribute('role') !== 'tablist') { 1831 tablist = tablist.parentNode; 1832 } 1833 1834 if (tablist.getAttribute('role') == 'tablist') { 1835 let lis = tablist.querySelectorAll('*[role=tab]'); 1836 lis.forEach( (it) => { 1837 let selected = it.getAttribute(kAriaSelected); 1838 if ( selected && selected == kTrue ) { 1839 aItem = it; 1840 } 1841 }); 1842 } 1843 1844 /* swap the active states: */ 1845 this.setAttribute(kAriaSelected, kTrue); 1846 if (aItem) { 1847 aItem.setAttribute(kAriaSelected, kFalse); 1848 let aId = aItem.getAttribute(kAriaControls); 1849 let aObj = document.getElementById(aId); 1850 if (aObj) aObj.hidden = true; 1851 } 1852 1853 /* show the new panel: */ 1854 let nId = this.getAttribute(kAriaControls); 1855 let nObj = document.getElementById(nId); 1856 if (nObj) nObj.hidden = false; 1857 } 1858 }, 1859 1860 /* The Overview / web metrics section of the live tab */ 1861 overview: { 1862 /** 1863 * Populates the overview part of the today tab with the analytics data. 1864 * 1865 * @method make 1866 * @memberof BotMon.live.gui.overview 1867 */ 1868 make: function() { 1869 1870 const data = BotMon.live.data.analytics.data; 1871 1872 const maxItemsPerList = 5; // how many list items to show? 1873 1874 const kNoData = '–'; // shown when data is missing 1875 1876 // shortcut for neater code: 1877 const makeElement = BotMon.t._makeElement; 1878 1879 const botsVsHumans = document.getElementById('botmon__today__botsvshumans'); 1880 if (botsVsHumans) { 1881 botsVsHumans.appendChild(makeElement('dt', {}, "Page views")); 1882 1883 for (let i = 0; i <= 5; i++) { 1884 const dd = makeElement('dd'); 1885 let title = ''; 1886 let value = ''; 1887 switch(i) { 1888 case 0: 1889 title = "Known bots:"; 1890 value = data.bots.known || kNoData; 1891 break; 1892 case 1: 1893 title = "Suspected bots:"; 1894 value = data.bots.suspected || kNoData; 1895 break; 1896 case 2: 1897 title = "Probably humans:"; 1898 value = data.bots.human || kNoData; 1899 break; 1900 case 3: 1901 title = "Registered users:"; 1902 value = data.bots.users || kNoData; 1903 break; 1904 case 4: 1905 title = "Total:"; 1906 value = data.totalPageViews || kNoData; 1907 break; 1908 case 5: 1909 title = "Bots-humans ratio:"; 1910 value = BotMon.t._getRatio(data.bots.suspected + data.bots.known, data.bots.users + data.bots.human, 100); 1911 break; 1912 default: 1913 console.warn(`Unknown list type ${i}.`); 1914 } 1915 dd.appendChild(makeElement('span', {}, title)); 1916 dd.appendChild(makeElement('strong', {}, value)); 1917 botsVsHumans.appendChild(dd); 1918 } 1919 } 1920 1921 // update known bots list: 1922 const botElement = document.getElementById('botmon__botslist'); /* Known bots */ 1923 if (botElement) { 1924 botElement.appendChild(makeElement('dt', {}, `Top known bots`)); 1925 1926 let botList = BotMon.live.data.analytics.getTopBots(maxItemsPerList); 1927 botList.forEach( (botInfo) => { 1928 const bli = makeElement('dd'); 1929 bli.appendChild(makeElement('span', {'class': 'has_icon bot bot_' + botInfo.id }, botInfo.name)); 1930 bli.appendChild(makeElement('span', {'class': 'count' }, botInfo.count)); 1931 botElement.append(bli) 1932 }); 1933 } 1934 1935 // update the suspected bot IP ranges list: 1936 const botIps = document.getElementById('botmon__botips'); 1937 if (botIps) { 1938 botIps.appendChild(makeElement('dt', {}, "Top bot Networks")); 1939 1940 const ispList = BotMon.live.data.analytics.getTopBotISPs(5); 1941 //console.log(ispList); 1942 ispList.forEach( (netInfo) => { 1943 const li = makeElement('dd'); 1944 li.appendChild(makeElement('span', {'class': 'has_icon ipaddr ip' + netInfo.typ }, netInfo.name)); 1945 li.appendChild(makeElement('span', {'class': 'count' }, netInfo.count)); 1946 botIps.append(li) 1947 }); 1948 } 1949 1950 // update the top bot countries list: 1951 const botCountries = document.getElementById('botmon__botcountries'); 1952 if (botCountries) { 1953 botCountries.appendChild(makeElement('dt', {}, `Top bot Countries`)); 1954 const countryList = BotMon.live.data.analytics.getCountryList('bot', 5); 1955 countryList.forEach( (cInfo) => { 1956 const cLi = makeElement('dd'); 1957 cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name)); 1958 cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count)); 1959 botCountries.appendChild(cLi); 1960 }); 1961 } 1962 1963 // update the webmetrics overview: 1964 const wmoverview = document.getElementById('botmon__today__wm_overview'); 1965 if (wmoverview) { 1966 1967 const humanVisits = BotMon.live.data.analytics.groups.users.length + BotMon.live.data.analytics.groups.humans.length; 1968 const bounceRate = Math.round(100 * (BotMon.live.data.analytics.getBounceCount('users') + BotMon.live.data.analytics.getBounceCount('humans')) / humanVisits); 1969 1970 wmoverview.appendChild(makeElement('dt', {}, "Humans’ metrics")); 1971 for (let i = 0; i <= 4; i++) { 1972 const dd = makeElement('dd'); 1973 let title = ''; 1974 let value = ''; 1975 switch(i) { 1976 case 0: 1977 title = "Registered users’ page views:"; 1978 value = data.bots.users || kNoData; 1979 break; 1980 case 1: 1981 title = "“Probably humans” page views:"; 1982 value = data.bots.human || kNoData; 1983 break; 1984 case 2: 1985 title = "Total human page views:"; 1986 value = (data.bots.users + data.bots.human) || kNoData; 1987 break; 1988 case 3: 1989 title = "Total human visits:"; 1990 value = humanVisits || kNoData; 1991 break; 1992 case 4: 1993 title = "Humans’ bounce rate:"; 1994 value = bounceRate + '%'; 1995 break; 1996 default: 1997 console.warn(`Unknown list type ${i}.`); 1998 } 1999 dd.appendChild(makeElement('span', {}, title)); 2000 dd.appendChild(makeElement('strong', {}, value)); 2001 wmoverview.appendChild(dd); 2002 } 2003 } 2004 2005 // update the webmetrics clients list: 2006 const wmclients = document.getElementById('botmon__today__wm_clients'); 2007 if (wmclients) { 2008 2009 wmclients.appendChild(makeElement('dt', {}, "Browsers")); 2010 2011 const clientList = BotMon.live.data.analytics.getTopBrowsers(maxItemsPerList); 2012 if (clientList) { 2013 clientList.forEach( (cInfo) => { 2014 const cDd = makeElement('dd'); 2015 cDd.appendChild(makeElement('span', {'class': 'has_icon client cl_' + cInfo.id }, ( cInfo.name ? cInfo.name : cInfo.id))); 2016 cDd.appendChild(makeElement('span', { 2017 'class': 'count', 2018 'title': cInfo.count + " page views" 2019 }, cInfo.pct.toFixed(1) + '%')); 2020 wmclients.appendChild(cDd); 2021 }); 2022 } 2023 } 2024 2025 // update the webmetrics platforms list: 2026 const wmplatforms = document.getElementById('botmon__today__wm_platforms'); 2027 if (wmplatforms) { 2028 2029 wmplatforms.appendChild(makeElement('dt', {}, "Platforms")); 2030 2031 const pfList = BotMon.live.data.analytics.getTopPlatforms(maxItemsPerList); 2032 if (pfList) { 2033 pfList.forEach( (pInfo) => { 2034 const pDd = makeElement('dd'); 2035 pDd.appendChild(makeElement('span', {'class': 'has_icon platform pf_' + pInfo.id }, ( pInfo.name ? pInfo.name : pInfo.id))); 2036 pDd.appendChild(makeElement('span', { 2037 'class': 'count', 2038 'title': pInfo.count + " page views" 2039 }, pInfo.pct.toFixed(1) + '%')); 2040 wmplatforms.appendChild(pDd); 2041 }); 2042 } 2043 } 2044 2045 // update the top bot countries list: 2046 const usrCountries = document.getElementById('botmon__today__wm_countries'); 2047 if (usrCountries) { 2048 usrCountries.appendChild(makeElement('dt', {}, `Top visitor Countries:`)); 2049 const usrCtryList = BotMon.live.data.analytics.getCountryList('human', 5); 2050 usrCtryList.forEach( (cInfo) => { 2051 const cLi = makeElement('dd'); 2052 cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name)); 2053 cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count)); 2054 usrCountries.appendChild(cLi); 2055 }); 2056 } 2057 2058 // update the top pages; 2059 const wmpages = document.getElementById('botmon__today__wm_pages'); 2060 if (wmpages) { 2061 2062 wmpages.appendChild(makeElement('dt', {}, "Top pages")); 2063 2064 const pgList = BotMon.live.data.analytics.getTopPages(maxItemsPerList); 2065 if (pgList) { 2066 pgList.forEach( (pgInfo) => { 2067 const pgDd = makeElement('dd'); 2068 pgDd.appendChild(makeElement('a', { 2069 'class': 'page_icon', 2070 'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(pgInfo.id), 2071 'target': 'preview', 2072 'title': "PageID: " + pgInfo.id 2073 }, pgInfo.id)); 2074 pgDd.appendChild(makeElement('span', { 2075 'class': 'count', 2076 'title': pgInfo.count + " page views" 2077 }, pgInfo.count)); 2078 wmpages.appendChild(pgDd); 2079 }); 2080 } 2081 } 2082 2083 // update the top referrers; 2084 const wmreferers = document.getElementById('botmon__today__wm_referers'); 2085 if (wmreferers) { 2086 2087 wmreferers.appendChild(makeElement('dt', {}, "Referers")); 2088 2089 const refList = BotMon.live.data.analytics.getTopReferers(maxItemsPerList); 2090 if (refList) { 2091 refList.forEach( (rInfo) => { 2092 const rDd = makeElement('dd'); 2093 rDd.appendChild(makeElement('span', {'class': 'has_icon referer ref_' + rInfo.id }, rInfo.name)); 2094 rDd.appendChild(makeElement('span', { 2095 'class': 'count', 2096 'title': rInfo.count + " references" 2097 }, rInfo.pct.toFixed(1) + '%')); 2098 wmreferers.appendChild(rDd); 2099 }); 2100 } 2101 } 2102 } 2103 }, 2104 2105 status: { 2106 setText: function(txt) { 2107 const el = document.getElementById('botmon__today__status'); 2108 if (el && BotMon.live.gui.status._errorCount <= 0) { 2109 el.innerText = txt; 2110 } 2111 }, 2112 2113 setTitle: function(html) { 2114 const el = document.getElementById('botmon__today__title'); 2115 if (el) { 2116 el.innerHTML = html; 2117 } 2118 }, 2119 2120 setError: function(txt) { 2121 console.error(txt); 2122 BotMon.live.gui.status._errorCount += 1; 2123 const el = document.getElementById('botmon__today__status'); 2124 if (el) { 2125 el.innerText = "Data may be incomplete."; 2126 el.classList.add('error'); 2127 } 2128 }, 2129 _errorCount: 0, 2130 2131 showBusy: function(txt = null) { 2132 BotMon.live.gui.status._busyCount += 1; 2133 const el = document.getElementById('botmon__today__busy'); 2134 if (el) { 2135 el.style.display = 'inline-block'; 2136 } 2137 if (txt) BotMon.live.gui.status.setText(txt); 2138 }, 2139 _busyCount: 0, 2140 2141 hideBusy: function(txt = null) { 2142 const el = document.getElementById('botmon__today__busy'); 2143 BotMon.live.gui.status._busyCount -= 1; 2144 if (BotMon.live.gui.status._busyCount <= 0) { 2145 if (el) el.style.display = 'none'; 2146 if (txt) BotMon.live.gui.status.setText(txt); 2147 } 2148 } 2149 }, 2150 2151 lists: { 2152 init: function() { 2153 2154 // function shortcut: 2155 const makeElement = BotMon.t._makeElement; 2156 2157 const parent = document.getElementById('botmon__today__visitorlists'); 2158 if (parent) { 2159 2160 for (let i=0; i < 4; i++) { 2161 2162 // change the id and title by number: 2163 let listTitle = ''; 2164 let listId = ''; 2165 let infolink = null; 2166 switch (i) { 2167 case 0: 2168 listTitle = "Registered users"; 2169 listId = 'users'; 2170 break; 2171 case 1: 2172 listTitle = "Probably humans"; 2173 listId = 'humans'; 2174 break; 2175 case 2: 2176 listTitle = "Suspected bots"; 2177 listId = 'suspectedBots'; 2178 infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/suspected_bots'; 2179 break; 2180 case 3: 2181 listTitle = "Known bots"; 2182 listId = 'knownBots'; 2183 infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/known_bots'; 2184 break; 2185 default: 2186 console.warn('Unknown list number.'); 2187 } 2188 2189 const details = makeElement('details', { 2190 'data-group': listId, 2191 'data-loaded': false 2192 }); 2193 const title = details.appendChild(makeElement('summary')); 2194 title.appendChild(makeElement('span', {'class': 'title'}, listTitle)); 2195 if (infolink) { 2196 title.appendChild(makeElement('a', { 2197 'class': 'ext_info', 2198 'target': '_blank', 2199 'href': infolink, 2200 'title': "More information" 2201 }, "Info")); 2202 } 2203 details.addEventListener("toggle", this._onDetailsToggle); 2204 2205 parent.appendChild(details); 2206 2207 } 2208 } 2209 }, 2210 2211 _onDetailsToggle: function(e) { 2212 //console.info('BotMon.live.gui.lists._onDetailsToggle()'); 2213 2214 const target = e.target; 2215 2216 if (target.getAttribute('data-loaded') == 'false') { // only if not loaded yet 2217 target.setAttribute('data-loaded', 'loading'); 2218 2219 const fillType = target.getAttribute('data-group'); 2220 const fillList = BotMon.live.data.analytics.groups[fillType]; 2221 if (fillList && fillList.length > 0) { 2222 2223 const ul = BotMon.t._makeElement('ul'); 2224 2225 fillList.forEach( (it) => { 2226 ul.appendChild(BotMon.live.gui.lists._makeVisitorItem(it, fillType)); 2227 }); 2228 2229 target.appendChild(ul); 2230 target.setAttribute('data-loaded', 'true'); 2231 } else { 2232 target.setAttribute('data-loaded', 'false'); 2233 } 2234 2235 } 2236 }, 2237 2238 _makeVisitorItem: function(data, type) { 2239 2240 // shortcut for neater code: 2241 const make = BotMon.t._makeElement; 2242 2243 let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' ); 2244 if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0'; 2245 2246 const platformName = (data._platform ? data._platform.n : 'Unknown'); 2247 const clientName = (data._client ? data._client.n: 'Unknown'); 2248 2249 const sumClass = ( !data._seenBy || data._seenBy.indexOf(BM_LOGTYPE.SERVER) < 0 ? 'noServer' : 'hasServer'); 2250 2251 const li = make('li'); // root list item 2252 const details = make('details'); 2253 const summary = make('summary', { 2254 'class': sumClass 2255 }); 2256 details.appendChild(summary); 2257 2258 const span1 = make('span'); /* left-hand group */ 2259 2260 if (data._type !== BM_USERTYPE.KNOWN_BOT) { /* No platform/client for bots */ 2261 span1.appendChild(make('span', { /* Platform */ 2262 'class': 'icon_only platform pf_' + (data._platform ? data._platform.id : 'unknown'), 2263 'title': "Platform: " + platformName 2264 }, platformName)); 2265 2266 span1.appendChild(make('span', { /* Client */ 2267 'class': 'icon_only client client cl_' + (data._client ? data._client.id : 'unknown'), 2268 'title': "Client: " + clientName 2269 }, clientName)); 2270 } 2271 2272 // identifier: 2273 if (data._type == BM_USERTYPE.KNOWN_BOT) { /* Bot only */ 2274 2275 const botName = ( data._bot && data._bot.n ? data._bot.n : "Unknown"); 2276 span1.appendChild(make('span', { /* Bot */ 2277 'class': 'has_icon bot bot_' + (data._bot ? data._bot.id : 'unknown'), 2278 'title': "Bot: " + botName 2279 }, botName)); 2280 2281 } else if (data._type == BM_USERTYPE.KNOWN_USER) { /* User only */ 2282 2283 span1.appendChild(make('span', { /* User */ 2284 'class': 'has_icon user_known', 2285 'title': "User: " + data.usr 2286 }, data.usr)); 2287 2288 } else { /* others */ 2289 2290 2291 /*span1.appendChild(make('span', { // IP-Address 2292 'class': 'has_icon ipaddr ip' + ipType, 2293 'title': "IP-Address: " + data.ip 2294 }, data.ip));*/ 2295 2296 span1.appendChild(make('span', { /* Internal ID */ 2297 'class': 'has_icon session typ_' + data.typ, 2298 'title': "ID: " + data.id 2299 }, data.id)); 2300 } 2301 2302 // seen by icon: 2303 span1.appendChild(make('span', { 2304 'class': 'icon_only seenby sb_' + data._seenBy.join(''), 2305 'title': "Seen by: " + data._seenBy.join('+') 2306 }, data._seenBy.join(', '))); 2307 2308 // country flag: 2309 if (data.geo && data.geo !== 'ZZ') { 2310 span1.appendChild(make('span', { 2311 'class': 'icon_only country ctry_' + data.geo.toLowerCase(), 2312 'data-ctry': data.geo, 2313 'title': "Country: " + ( data._country || "Unknown") 2314 }, ( data._country || "Unknown") )); 2315 } 2316 2317 // referer icons: 2318 if ((data._type == BM_USERTYPE.PROBABLY_HUMAN || data._type == BM_USERTYPE.LIKELY_BOT) && data.ref) { 2319 const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref); 2320 span1.appendChild(make('span', { 2321 'class': 'icon_only referer ref_' + refInfo.id, 2322 'title': "Referer: " + data.ref 2323 }, refInfo.n)); 2324 } 2325 2326 summary.appendChild(span1); 2327 const span2 = make('span'); /* right-hand group */ 2328 2329 span2.appendChild(make('span', { /* first-seen */ 2330 'class': 'has_iconfirst-seen', 2331 'title': "First seen: " + data._firstSeen.toLocaleString() + " UTC" 2332 }, BotMon.t._formatTime(data._firstSeen))); 2333 2334 span2.appendChild(make('span', { /* page views */ 2335 'class': 'has_icon pageviews', 2336 'title': data._pageViews.length + " page view(s)" 2337 }, data._pageViews.length)); 2338 2339 summary.appendChild(span2); 2340 2341 // add details expandable section: 2342 details.appendChild(BotMon.live.gui.lists._makeVisitorDetails(data, type)); 2343 2344 li.appendChild(details); 2345 return li; 2346 }, 2347 2348 _makeVisitorDetails: function(data, type) { 2349 2350 // shortcut for neater code: 2351 const make = BotMon.t._makeElement; 2352 2353 let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' ); 2354 if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0'; 2355 const platformName = (data._platform ? data._platform.n : 'Unknown'); 2356 const clientName = (data._client ? data._client.n: 'Unknown'); 2357 2358 const dl = make('dl', {'class': 'visitor_details'}); 2359 2360 if (data._type == BM_USERTYPE.KNOWN_BOT) { 2361 2362 dl.appendChild(make('dt', {}, "Bot name:")); /* bot info */ 2363 dl.appendChild(make('dd', {'class': 'icon_only bot bot_' + (data._bot ? data._bot.id : 'unknown')}, 2364 (data._bot ? data._bot.n : 'Unknown'))); 2365 2366 if (data._bot && data._bot.url) { 2367 dl.appendChild(make('dt', {}, "Bot info:")); /* bot info */ 2368 const botInfoDd = dl.appendChild(make('dd')); 2369 botInfoDd.appendChild(make('a', { 2370 'href': data._bot.url, 2371 'target': '_blank' 2372 }, data._bot.url)); /* bot info link*/ 2373 2374 } 2375 2376 } else { /* not for bots */ 2377 2378 dl.appendChild(make('dt', {}, "Client:")); /* client */ 2379 dl.appendChild(make('dd', {'class': 'has_icon client cl_' + (data._client ? data._client.id : 'unknown')}, 2380 clientName + ( data._client.v > 0 ? ' (' + data._client.v + ')' : '' ) )); 2381 2382 dl.appendChild(make('dt', {}, "Platform:")); /* platform */ 2383 dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')}, 2384 platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); 2385 2386 /*dl.appendChild(make('dt', {}, "ID:")); 2387 dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/ 2388 } 2389 2390 dl.appendChild(make('dt', {}, "IP-Address:")); 2391 const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType}); 2392 ipItem.appendChild(make('span', {'class': 'address'} , data.ip)); 2393 ipItem.appendChild(make('a', { 2394 'class': 'icon_only extlink ipinfo', 2395 'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`, 2396 'target': 'ipinfo', 2397 'title': "View this address on IPInfo.io" 2398 } , "DNS Info")); 2399 ipItem.appendChild(make('a', { 2400 'class': 'icon_only extlink abuseipdb', 2401 'href': `https://www.abuseipdb.com/check/${encodeURIComponent(data.ip)}`, 2402 'target': 'abuseipdb', 2403 'title': "Check this address on AbuseIPDB.com" 2404 } , "Check on AbuseIPDB")); 2405 dl.appendChild(ipItem); 2406 2407 if (Math.abs(data._lastSeen - data._firstSeen) < 100) { 2408 dl.appendChild(make('dt', {}, "Seen:")); 2409 dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString())); 2410 } else { 2411 dl.appendChild(make('dt', {}, "First seen:")); 2412 dl.appendChild(make('dd', {'class': 'firstSeen'}, data._firstSeen.toLocaleString())); 2413 dl.appendChild(make('dt', {}, "Last seen:")); 2414 dl.appendChild(make('dd', {'class': 'lastSeen'}, data._lastSeen.toLocaleString())); 2415 } 2416 2417 dl.appendChild(make('dt', {}, "User-Agent:")); 2418 dl.appendChild(make('dd', {'class': 'agent'}, data.agent)); 2419 2420 if (data.ref && data.ref !== '') { 2421 dl.appendChild(make('dt', {}, "Referrer:")); 2422 2423 const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref); 2424 const refDd = dl.appendChild(make('dd', { 2425 'class': 'has_icon referer ref_' + refInfo.id 2426 })); 2427 refDd.appendChild(make('a', { 2428 'href': data.ref, 2429 'target': 'refTarget' 2430 }, data.ref)); 2431 } 2432 2433 dl.appendChild(make('dt', {}, "Languages:")); 2434 dl.appendChild(make('dd', {'class': 'langs'}, ` [${data.accept}]`)); 2435 2436 if (data.geo && data.geo !=='') { 2437 dl.appendChild(make('dt', {}, "Location:")); 2438 dl.appendChild(make('dd', { 2439 'class': 'has_icon country ctry_' + data.geo.toLowerCase(), 2440 'data-ctry': data.geo, 2441 'title': "Country: " + data._country 2442 }, data._country + ' (' + data.geo + ')')); 2443 } 2444 2445 dl.appendChild(make('dt', {}, "Session ID:")); 2446 dl.appendChild(make('dd', {'class': 'has_icon session typ_' + data.typ}, data.id)); 2447 2448 dl.appendChild(make('dt', {}, "Seen by:")); 2449 dl.appendChild(make('dd', {'class': 'has_icon seenby sb_' + data._seenBy.join('')}, data._seenBy.join(', ') )); 2450 2451 dl.appendChild(make('dt', {}, "Visited pages:")); 2452 const pagesDd = make('dd', {'class': 'pages'}); 2453 const pageList = make('ul'); 2454 2455 /* list all page views */ 2456 data._pageViews.sort( (a, b) => a._firstSeen - b._firstSeen ); 2457 data._pageViews.forEach( (page) => { 2458 pageList.appendChild(BotMon.live.gui.lists._makePageViewItem(page)); 2459 }); 2460 pagesDd.appendChild(pageList); 2461 dl.appendChild(pagesDd); 2462 2463 /* bot evaluation rating */ 2464 if (data._type !== BM_USERTYPE.KNOWN_BOT && data._type !== BM_USERTYPE.KNOWN_USER) { 2465 dl.appendChild(make('dt', undefined, "Bot rating:")); 2466 dl.appendChild(make('dd', {'class': 'bot-rating'}, ( data._botVal ? data._botVal : '–' ) + ' (of ' + BotMon.live.data.rules._threshold + ')')); 2467 2468 /* add bot evaluation details: */ 2469 if (data._eval) { 2470 dl.appendChild(make('dt', {}, "Bot evaluation:")); 2471 const evalDd = make('dd', {'class': 'eval'}); 2472 const testList = make('ul'); 2473 data._eval.forEach( test => { 2474 2475 const tObj = BotMon.live.data.rules.getRuleInfo(test); 2476 let tDesc = tObj ? tObj.desc : test; 2477 2478 // special case for Bot IP range test: 2479 if (tObj.func == 'fromKnownBotIP') { 2480 const rangeInfo = BotMon.live.data.ipRanges.match(data.ip); 2481 if (rangeInfo) { 2482 const owner = BotMon.live.data.ipRanges.getOwner(rangeInfo.g); 2483 tDesc += ' (range: “' + rangeInfo.cidr + '”, ' + owner + ')'; 2484 } 2485 } 2486 2487 // create the entry field 2488 const tstLi = make('li'); 2489 tstLi.appendChild(make('span', { 2490 'data-testid': test 2491 }, tDesc)); 2492 tstLi.appendChild(make('span', {}, ( tObj ? tObj.bot : '—') )); 2493 testList.appendChild(tstLi); 2494 }); 2495 2496 // add total row 2497 const tst2Li = make('li', { 2498 'class': 'total' 2499 }); 2500 /*tst2Li.appendChild(make('span', {}, "Total:")); 2501 tst2Li.appendChild(make('span', {}, data._botVal)); 2502 testList.appendChild(tst2Li);*/ 2503 2504 evalDd.appendChild(testList); 2505 dl.appendChild(evalDd); 2506 } 2507 } 2508 // return the element to add to the UI: 2509 return dl; 2510 }, 2511 2512 // make a page view item: 2513 _makePageViewItem: function(page) { 2514 //console.log("makePageViewItem:",page); 2515 2516 // shortcut for neater code: 2517 const make = BotMon.t._makeElement; 2518 2519 // the actual list item: 2520 const pgLi = make('li'); 2521 2522 const row1 = make('div', {'class': 'row'}); 2523 2524 row1.appendChild(make('a', { // page id is the left group 2525 'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(page.pg), 2526 'target': 'preview', 2527 'hreflang': page.lang, 2528 'title': "PageID: " + page.pg 2529 }, page.pg)); /* DW Page ID */ 2530 2531 // get the time difference: 2532 row1.appendChild(make('span', { 2533 'class': 'first-seen', 2534 'title': "First visited: " + page._firstSeen.toLocaleString() + " UTC" 2535 }, BotMon.t._formatTime(page._firstSeen))); 2536 2537 pgLi.appendChild(row1); 2538 2539 /* LINE 2 */ 2540 2541 const row2 = make('div', {'class': 'row'}); 2542 2543 // page referrer: 2544 if (page._ref) { 2545 row2.appendChild(make('span', { 2546 'class': 'referer', 2547 'title': "Referrer: " + page._ref.href 2548 }, page._ref.hostname)); 2549 } else { 2550 row2.appendChild(make('span', { 2551 'class': 'referer' 2552 }, "No referer")); 2553 } 2554 2555 // visit duration: 2556 let visitTimeStr = "Bounce"; 2557 const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime(); 2558 if (visitDuration > 0) { 2559 visitTimeStr = Math.floor(visitDuration / 1000) + "s"; 2560 } 2561 const tDiff = BotMon.t._formatTimeDiff(page._firstSeen, page._lastSeen); 2562 if (tDiff) { 2563 row2.appendChild(make('span', {'class': 'visit-length', 'title': 'Last seen: ' + page._lastSeen.toLocaleString()}, tDiff)); 2564 } else { 2565 row2.appendChild(make('span', { 2566 'class': 'bounce', 2567 'title': "Visitor bounced"}, "Bounce")); 2568 } 2569 2570 pgLi.appendChild(row2); 2571 2572 return pgLi; 2573 } 2574 } 2575 } 2576}; 2577 2578/* launch only if the BotMon admin panel is open: */ 2579if (document.getElementById('botmon__admin')) { 2580 BotMon.init(); 2581}