1"use strict"; 2/* DokuWiki BotMon Plugin Script file */ 3/* 14.10.2025 - 0.5.0 - pre-release */ 4/* Author: Sascha Leib <ad@hominem.info> */ 5 6// enumeration of user types: 7const BM_USERTYPE = Object.freeze({ 8 'UNKNOWN': 'unknown', 9 'KNOWN_USER': 'user', 10 'PROBABLY_HUMAN': 'human', 11 'LIKELY_BOT': 'likely_bot', 12 'KNOWN_BOT': 'known_bot' 13}); 14 15// enumeration of log types: 16const BM_LOGTYPE = Object.freeze({ 17 'SERVER': 'srv', 18 'CLIENT': 'log', 19 'TICKER': 'tck' 20}); 21 22// enumeration of IP versions: 23const BM_IPVERSION = Object.freeze({ 24 'IPv4': 4, 25 'IPv6': 6 26}); 27 28/* BotMon root object */ 29const BotMon = { 30 31 init: function() { 32 //console.info('BotMon.init()'); 33 34 // find the plugin basedir: 35 this._baseDir = document.currentScript.src.substring(0, document.currentScript.src.lastIndexOf('/')+1); 36 37 // read the page language from the DOM: 38 this._lang = document.getRootNode().documentElement.lang || this._lang; 39 40 // get the time offset: 41 this._timeDiff = BotMon.t._getTimeOffset(); 42 43 // get yesterday's date: 44 let d = new Date(); 45 if (BMSettings.showday == 'yesterday') d.setDate(d.getDate() - 1); 46 this._datestr = d.toISOString().slice(0, 10); 47 48 // init the sub-objects: 49 BotMon.t._callInit(this); 50 }, 51 52 _baseDir: null, 53 _lang: 'en', 54 _datestr: '', 55 _timeDiff: '', 56 57 /* internal tools */ 58 t: { 59 /* helper function to call inits of sub-objects */ 60 _callInit: function(obj) { 61 //console.info('BotMon.t._callInit(obj=',obj,')'); 62 63 /* call init / _init on each sub-object: */ 64 Object.keys(obj).forEach( (key,i) => { 65 const sub = obj[key]; 66 let init = null; 67 if (typeof sub === 'object' && sub.init) { 68 init = sub.init; 69 } 70 71 // bind to object 72 if (typeof init == 'function') { 73 const init2 = init.bind(sub); 74 init2(obj); 75 } 76 }); 77 }, 78 79 /* helper function to calculate the time difference to UTC: */ 80 _getTimeOffset: function() { 81 const now = new Date(); 82 let offset = now.getTimezoneOffset(); // in minutes 83 const sign = Math.sign(offset); // +1 or -1 84 offset = Math.abs(offset); // always positive 85 86 let hours = 0; 87 while (offset >= 60) { 88 hours += 1; 89 offset -= 60; 90 } 91 return ( hours > 0 ? sign * hours + ' h' : '') + (offset > 0 ? ` ${offset} min` : ''); 92 }, 93 94 /* helper function to create a new element with all attributes and text content */ 95 _makeElement: function(name, atlist = undefined, text = undefined) { 96 var r = null; 97 try { 98 r = document.createElement(name); 99 if (atlist) { 100 for (let attr in atlist) { 101 r.setAttribute(attr, atlist[attr]); 102 } 103 } 104 if (text) { 105 r.textContent = text.toString(); 106 } 107 } catch(e) { 108 console.error(e); 109 } 110 return r; 111 }, 112 113 /* helper to convert an ip address string to a normalised format: */ 114 _ip2Num: function(ip) { 115 if (!ip) { 116 return 'null'; 117 } else if (ip.indexOf(':') > 0) { /* IP6 */ 118 return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(':')); 119 } else { /* IP4 */ 120 return ip.split('.').map(d => ('000'+d).slice(-3) ).join('.'); 121 } 122 }, 123 124 /* helper function to format a Date object to show only the time. */ 125 /* returns String */ 126 _formatTime: function(date) { 127 128 if (date) { 129 return date.getHours() + ':' + ('0'+date.getMinutes()).slice(-2) + ':' + ('0'+date.getSeconds()).slice(-2); 130 } else { 131 return null; 132 } 133 134 }, 135 136 /* helper function to show a time difference in seconds or minutes */ 137 /* returns String */ 138 _formatTimeDiff: function(dateA, dateB) { 139 140 // if the second date is ealier, swap them: 141 if (dateA > dateB) dateB = [dateA, dateA = dateB][0]; 142 143 // get the difference in milliseconds: 144 let ms = dateB - dateA; 145 146 if (ms > 50) { /* ignore small time spans */ 147 const h = Math.floor((ms / (1000 * 60 * 60)) % 24); 148 const m = Math.floor((ms / (1000 * 60)) % 60); 149 const s = Math.floor((ms / 1000) % 60); 150 151 return ( h>0 ? h + 'h ': '') + ( m>0 ? m + 'm ': '') + ( s>0 ? s + 's': ''); 152 } 153 154 return null; 155 156 }, 157 158 // calcualte a reduced ration between two numbers 159 // adapted from https://stackoverflow.com/questions/3946373/math-in-js-how-do-i-get-a-ratio-from-a-percentage 160 _getRatio: function(a, b, tolerance) { 161 162 var bg = b; 163 var sm = a; 164 if (a == 0 || b == 0) return '—'; 165 if (a > b) { 166 var bg = a; 167 var sm = b; 168 } 169 170 for (var i = 1; i < 1000000; i++) { 171 var d = sm / i; 172 var res = bg / d; 173 var howClose = Math.abs(res - res.toFixed(0)); 174 if (howClose < tolerance) { 175 if (a > b) { 176 return res.toFixed(0) + ':' + i; 177 } else { 178 return i + ':' + res.toFixed(0); 179 } 180 } 181 } 182 } 183 } 184}; 185 186/* everything specific to the 'Latest' tab is self-contained in the 'live' object: */ 187BotMon.live = { 188 init: function() { 189 //console.info('BotMon.live.init()'); 190 191 // set the title: 192 const tDiff = '<abbr title="Coordinated Universal Time">UTC</abbr> ' + (BotMon._timeDiff != '' ? ` (offset: ${BotMon._timeDiff}` : '' ) + ')'; 193 BotMon.live.gui.status.setTitle(`Data for <time datetime="${BotMon._datestr}">${BotMon._datestr}</time> ${tDiff}`); 194 195 // init sub-objects: 196 BotMon.t._callInit(this); 197 }, 198 199 data: { 200 init: function() { 201 //console.info('BotMon.live.data.init()'); 202 203 // call sub-inits: 204 BotMon.t._callInit(this); 205 }, 206 207 // this will be called when the known json files are done loading: 208 _dispatch: function(file) { 209 //console.info('BotMon.live.data._dispatch(,',file,')'); 210 211 // shortcut to make code more readable: 212 const data = BotMon.live.data; 213 214 // set the flags: 215 switch(file) { 216 case 'rules': 217 data._dispatchRulesLoaded = true; 218 break; 219 case 'ipranges': 220 data._dispatchIPRangesLoaded = true; 221 break; 222 case 'bots': 223 data._dispatchBotsLoaded = true; 224 break; 225 case 'clients': 226 data._dispatchClientsLoaded = true; 227 break; 228 case 'platforms': 229 data._dispatchPlatformsLoaded = true; 230 break; 231 default: 232 // ignore 233 } 234 235 // are all the flags set? 236 if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded && data._dispatchIPRangesLoaded) { 237 // chain the log files loading: 238 BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded); 239 } 240 }, 241 // flags to track which data files have been loaded: 242 _dispatchBotsLoaded: false, 243 _dispatchClientsLoaded: false, 244 _dispatchPlatformsLoaded: false, 245 _dispatchIPRangesLoaded: false, 246 _dispatchRulesLoaded: false, 247 248 // event callback, after the server log has been loaded: 249 _onServerLogLoaded: function() { 250 //console.info('BotMon.live.data._onServerLogLoaded()'); 251 252 // chain the client log file to load: 253 BotMon.live.data.loadLogFile(BM_LOGTYPE.CLIENT, BotMon.live.data._onClientLogLoaded); 254 }, 255 256 // event callback, after the client log has been loaded: 257 _onClientLogLoaded: function() { 258 //console.info('BotMon.live.data._onClientLogLoaded()'); 259 260 // chain the ticks file to load: 261 BotMon.live.data.loadLogFile(BM_LOGTYPE.TICKER, BotMon.live.data._onTicksLogLoaded); 262 263 }, 264 265 // event callback, after the tiker log has been loaded: 266 _onTicksLogLoaded: function() { 267 //console.info('BotMon.live.data._onTicksLogLoaded()'); 268 269 // analyse the data: 270 BotMon.live.data.analytics.analyseAll(); 271 272 // sort the data: 273 // #TODO 274 275 // display the data: 276 BotMon.live.gui.overview.make(); 277 278 //console.log(BotMon.live.data.model._visitors); 279 280 }, 281 282 // the data model: 283 model: { 284 // visitors storage: 285 _visitors: [], 286 287 // find an already existing visitor record: 288 findVisitor: function(visitor, type) { 289 //console.info('BotMon.live.data.model.findVisitor()', type); 290 //console.log(visitor); 291 292 // shortcut to make code more readable: 293 const model = BotMon.live.data.model; 294 295 // combine Bot networks to one visitor? 296 const combineNets = (BMSettings.hasOwnProperty('combineNets') ? BMSettings['combineNets'] : true);; 297 298 if (visitor._type == BM_USERTYPE.KNOWN_BOT) { // known bots match by their bot ID: 299 300 for (let i=0; i<model._visitors.length; i++) { 301 const v = model._visitors[i]; 302 303 // bots match when their ID matches: 304 if (v._bot && v._bot.id == visitor._bot.id) { 305 return v; 306 } 307 } 308 309 } else if (combineNets && visitor.hasOwnProperty('_ipRange')) { // combine with other visits from the same range 310 311 let nonRangeVisitor = null; 312 313 for (let i=0; i<model._visitors.length; i++) { 314 const v = model._visitors[i]; 315 316 if ( v.hasOwnProperty('_ipRange') && v._ipRange.g == visitor._ipRange.g ) { // match the IPRange Group IDs 317 return v; 318 } else if ( v.id.trim() !== '' && v.id == visitor.id) { // match the DW/PHP IDs 319 nonRangeVisitor = v; 320 } 321 } 322 323 // if no ip range was found, return the non-range visitor instead 324 if (nonRangeVisitor) return nonRangeVisitor; 325 326 } else { // other types match by their DW/PHPIDs: 327 328 // loop over all visitors already registered and check for ID matches: 329 for (let i=0; i<model._visitors.length; i++) { 330 const v = model._visitors[i]; 331 332 if ( v.id.trim() !== '' && v.id == visitor.id) { // match the DW/PHP IDs 333 return v; 334 } 335 } 336 337 // if not found, try to match IP address and user agent: 338 for (let i=0; i<model._visitors.length; i++) { 339 const v = model._visitors[i]; 340 if ( v.ip == visitor.ip && v.agent == visitor.agent) { 341 return v; 342 } 343 } 344 } 345 346 return null; // nothing found 347 }, 348 349 /* if there is already this visit registered, return the page view item */ 350 _getPageView: function(visit, view) { 351 352 // shortcut to make code more readable: 353 const model = BotMon.live.data.model; 354 355 for (let i=0; i<visit._pageViews.length; i++) { 356 const pv = visit._pageViews[i]; 357 if (pv.pg == view.pg) { 358 return pv; 359 } 360 } 361 return null; // not found 362 }, 363 364 // register a new visitor (or update if already exists) 365 registerVisit: function(nv, type) { 366 //console.info('registerVisit', nv, type); 367 368 369 // shortcut to make code more readable: 370 const model = BotMon.live.data.model; 371 372 // is it a known bot? 373 const bot = BotMon.live.data.bots.match(nv.agent); 374 375 // enrich new visitor with relevant data: 376 if (!nv._bot) nv._bot = bot ?? null; // bot info 377 nv._type = ( bot ? BM_USERTYPE.KNOWN_BOT : ( nv.usr && nv.usr !== '' ? BM_USERTYPE.KNOWN_USER : BM_USERTYPE.UNKNOWN ) ); // user type 378 if (bot && bot.geo) { 379 if (!nv.geo || nv.geo == '' || nv.geo == 'ZZ') nv.geo = bot.geo; 380 } else if (!nv.geo ||nv.geo == '') { 381 nv.geo = 'ZZ'; 382 } 383 384 // update first and last seen: 385 if (!nv._firstSeen) nv._firstSeen = nv.ts; // first-seen 386 nv._lastSeen = nv.ts; // last-seen 387 388 // known bot IP range? 389 if (nv._type == BM_USERTYPE.UNKNOWN) { // only for unknown visitors 390 const ipInfo = BotMon.live.data.ipRanges.match(nv.ip); 391 if (ipInfo) nv._ipRange = ipInfo; 392 } 393 394 // country name: 395 try { 396 nv._country = ( nv.geo == 'local' ? "localhost" : "Unknown" ); 397 if (nv.geo && nv.geo !== '' && nv.geo !== 'ZZ' && nv.geo !== 'local') { 398 const countryName = new Intl.DisplayNames(['en', BotMon._lang], {type: 'region'}); 399 nv._country = countryName.of(nv.geo.substring(0,2)) ?? nv.geo; 400 } 401 } catch (err) { 402 console.error(err); 403 nv._country = 'Error'; 404 } 405 406 // check if it already exists: 407 let visitor = model.findVisitor(nv, type); 408 if (!visitor) { 409 visitor = {...nv, ...{ 410 _seenBy: [type], 411 _viewCount: 0, // number of page views 412 _loadCount: 0, // number of page loads (not necessarily views!) 413 _pageViews: [], // array of page views 414 _hasReferrer: false, // has at least one referrer 415 _jsClient: false, // visitor has been seen logged by client js as well 416 _client: BotMon.live.data.clients.match(nv.agent) ?? null, // client info 417 _platform: BotMon.live.data.platforms.match(nv.agent), // platform info 418 _captcha: {'X': 0, 'Y': 0, 'N': 0, 'W':0, 'H': 0, 419 _str: function() { return (this.X > 0 ? 'X' : '') + (this.Y > 0 ? 'Y' : '') + (this.N > 0 ? 'N' : '') + (this.W > 0 ? 'W' : '') + (this.H > 0 ? 'H' : ''); } 420 } // captcha counter 421 }}; 422 model._visitors.push(visitor); 423 }; 424 425 // update first and last seen: 426 if (visitor._firstSeen > nv.ts) { 427 visitor._firstSeen = nv.ts; 428 } 429 if (visitor._lastSeen < nv.ts) { 430 visitor._lastSeen = nv.ts; 431 } 432 433 // update total loads and views (not the same!): 434 visitor._loadCount += 1; 435 visitor._viewCount += (nv.captcha == 'Y' ? 0 : 1); 436 437 // ...because also a captcha is a "load", but not a "view". 438 // let's count the captcha statuses as well: 439 if (nv.captcha) visitor._captcha[nv.captcha] += 1; 440 441 // is this visit already registered? 442 let prereg = model._getPageView(visitor, nv); 443 if (!prereg) { 444 // add new page view: 445 prereg = model._makePageView(nv, type); 446 visitor._pageViews.push(prereg); 447 } 448 449 // update last seen date 450 prereg._lastSeen = nv.ts; 451 452 // increase view count: 453 prereg._loadCount += (visitor.captcha == 'Y' ? 0 : 1); 454 //prereg._tickCount += 1; 455 456 // update referrer state: 457 visitor._hasReferrer = visitor._hasReferrer || 458 (prereg.ref !== undefined && prereg.ref !== ''); 459 460 // update time stamp for last-seen: 461 if (visitor._lastSeen < nv.ts) { 462 visitor._lastSeen = nv.ts; 463 } 464 465 // if needed: 466 return visitor; 467 }, 468 469 // updating visit data from the client-side log: 470 updateVisit: function(dat) { 471 //console.info('updateVisit', dat); 472 473 // shortcut to make code more readable: 474 const model = BotMon.live.data.model; 475 476 const type = BM_LOGTYPE.CLIENT; 477 478 let visitor = BotMon.live.data.model.findVisitor(dat, type); 479 if (!visitor) { 480 visitor = model.registerVisit(dat, type); 481 } 482 if (visitor) { 483 484 if (visitor._lastSeen < dat.ts) { 485 visitor._lastSeen = dat.ts; 486 } 487 if (!visitor._seenBy.includes(type)) { 488 visitor._seenBy.push(type); 489 } 490 visitor._jsClient = true; // seen by client js 491 } 492 493 // find the page view: 494 let prereg = BotMon.live.data.model._getPageView(visitor, dat); 495 if (prereg) { 496 // update the page view: 497 prereg._lastSeen = dat.ts; 498 if (!prereg._seenBy.includes(type)) prereg._seenBy.push(type); 499 prereg._jsClient = true; // seen by client js 500 } else { 501 // add the page view to the visitor: 502 prereg = model._makePageView(dat, type); 503 visitor._pageViews.push(prereg); 504 } 505 prereg._tickCount += 1; 506 }, 507 508 // updating visit data from the ticker log: 509 updateTicks: function(dat) { 510 //console.info('updateTicks', dat); 511 512 // shortcut to make code more readable: 513 const model = BotMon.live.data.model; 514 515 const type = BM_LOGTYPE.TICKER; 516 517 // find the visit info: 518 let visitor = model.findVisitor(dat, type); 519 if (!visitor) { 520 console.info(`No visitor with ID “${dat.id}” found, registering as a new one.`); 521 visitor = model.registerVisit(dat, type); 522 } 523 if (visitor) { 524 // update visitor: 525 if (visitor._lastSeen < dat.ts) visitor._lastSeen = dat.ts; 526 if (!visitor._seenBy.includes(type)) visitor._seenBy.push(type); 527 528 // get the page view info: 529 let pv = model._getPageView(visitor, dat); 530 if (!pv) { 531 console.info(`No page view for visit ID “${dat.id}”, page “${dat.pg}”, registering a new one.`); 532 pv = model._makePageView(dat, type); 533 visitor._pageViews.push(pv); 534 } 535 536 // update the page view info: 537 if (!pv._seenBy.includes(type)) pv._seenBy.push(type); 538 if (pv._lastSeen.getTime() < dat.ts.getTime()) pv._lastSeen = dat.ts; 539 pv._tickCount += 1; 540 541 } 542 }, 543 544 // helper function to create a new "page view" item: 545 _makePageView: function(data, type) { 546 // console.info('_makePageView', data); 547 548 // try to parse the referrer: 549 let rUrl = null; 550 try { 551 rUrl = ( data.ref && data.ref !== '' ? new URL(data.ref) : null ); 552 } catch (e) { 553 console.warn(`Invalid referer: “${data.ref}”.`); 554 console.info(data); 555 } 556 557 return { 558 _by: type, 559 ip: data.ip, 560 pg: data.pg, 561 lang: data.lang || '??', 562 _ref: rUrl, 563 _firstSeen: data.ts, 564 _lastSeen: data.ts, 565 _seenBy: [type], 566 _jsClient: ( type !== BM_LOGTYPE.SERVER), 567 _viewCount: 0, 568 _loadCount: 0, 569 _tickCount: 0 570 }; 571 }, 572 573 // helper function to make a human-readable title from the Captcha statuses: 574 _makeCaptchaTitle: function(cObj) { 575 const cStr = cObj._str(); 576 switch (cStr) { 577 case 'Y': 578 case 'NY': return "Blocked."; 579 case 'YN': return "Solved"; 580 case 'W': return "Whitelisted"; 581 case 'H': return "HEAD request, no captcha"; 582 default: return "Undefined: " + cStr; 583 } 584 } 585 }, 586 587 // functions to analyse the data: 588 analytics: { 589 590 /** 591 * Initializes the analytics data storage object: 592 */ 593 init: function() { 594 //console.info('BotMon.live.data.analytics.init()'); 595 }, 596 597 // data storage: 598 data: { 599 visits: { 600 bots: 0, 601 suspected: 0, 602 humans: 0, 603 users: 0, 604 total: 0 605 }, 606 views: { 607 bots: 0, 608 suspected: 0, 609 humans: 0, 610 users: 0, 611 total: 0 612 }, 613 loads: { 614 bots: 0, 615 suspected: 0, 616 humans: 0, 617 users: 0, 618 total: 0 619 }, 620 captcha: { 621 bots_blocked: 0, 622 bots_passed: 0, 623 bots_whitelisted: 0, 624 humans_blocked: 0, 625 humans_passed: 0, 626 sus_blocked: 0, 627 sus_passed: 0, 628 sus_whitelisted: 0 629 } 630 }, 631 632 // sort the visits by type: 633 groups: { 634 knownBots: [], 635 suspectedBots: [], 636 humans: [], 637 users: [] 638 }, 639 640 // all analytics 641 analyseAll: function() { 642 //console.info('BotMon.live.data.analytics.analyseAll()'); 643 644 // shortcut to make code more readable: 645 const model = BotMon.live.data.model; 646 const data = BotMon.live.data.analytics.data; 647 const me = BotMon.live.data.analytics; 648 649 BotMon.live.gui.status.showBusy("Analysing data …"); 650 651 // loop over all visitors: 652 model._visitors.forEach( (v) => { 653 654 const captchaStr = v._captcha._str(); 655 656 // count total visits and page views: 657 data.visits.total += 1; 658 data.loads.total += v._loadCount; 659 data.views.total += v._viewCount; 660 661 // check for typical bot aspects: 662 let botScore = 0; 663 664 if (v._type == BM_USERTYPE.KNOWN_BOT) { // known bots 665 666 data.visits.bots += 1; 667 data.views.bots += v._viewCount; 668 this.groups.knownBots.push(v); 669 670 // captcha counter 671 if (captchaStr == 'Y') { 672 data.captcha.bots_blocked += 1; 673 } else if (captchaStr == 'YN') { 674 data.captcha.bots_passed += 1; 675 } else if (captchaStr == 'W') { 676 data.captcha.bots_whitelisted += 1; 677 } 678 679 } else if (v._type == BM_USERTYPE.KNOWN_USER) { // known users */ 680 681 data.visits.users += 1; 682 data.views.users += v._viewCount; 683 this.groups.users.push(v); 684 685 } else { 686 687 // get evaluation: 688 const e = BotMon.live.data.rules.evaluate(v); 689 v._eval = e.rules; 690 v._botVal = e.val; 691 692 if (e.isBot) { // likely bots 693 694 v._type = BM_USERTYPE.LIKELY_BOT; 695 data.visits.suspected += 1; 696 data.views.suspected += v._viewCount; 697 this.groups.suspectedBots.push(v); 698 699 // captcha counter 700 if (captchaStr == 'Y') { 701 data.captcha.sus_blocked += 1; 702 } else if (captchaStr == 'YN') { 703 data.captcha.sus_passed += 1; 704 } else if (captchaStr == 'W') { 705 data.captcha.sus_whitelisted += 1; 706 } 707 708 } else { // probably humans 709 710 v._type = BM_USERTYPE.PROBABLY_HUMAN; 711 data.visits.humans += 1; 712 data.views.humans += v._viewCount; 713 714 this.groups.humans.push(v); 715 716 // captcha counter 717 if (captchaStr == 'Y') { 718 data.captcha.humans_blocked += 1; 719 } else if (captchaStr == 'YN') { 720 data.captcha.humans_passed += 1; 721 } 722 } 723 } 724 725 // perform actions depending on the visitor type: 726 if (v._type == BM_USERTYPE.KNOWN_BOT ) { /* known bots only */ 727 728 // no specific actions here. 729 730 } else if (v._type == BM_USERTYPE.LIKELY_BOT) { /* probable bots only */ 731 732 // add bot views to IP range information: 733 me.addToIpRanges(v); 734 735 } else { /* registered users and probable humans */ 736 737 // add browser and platform statistics: 738 me.addBrowserPlatform(v); 739 740 // add to referrer and pages lists: 741 v._pageViews.forEach( pv => { 742 me.addToRefererList(pv._ref); 743 me.addToPagesList(pv.pg); 744 }); 745 } 746 747 // add to the country lists: 748 me.addToCountries(v.geo, v._country, v._type); 749 750 }); 751 752 BotMon.live.gui.status.hideBusy('Done.'); 753 }, 754 755 // get a list of known bots: 756 getTopBots: function(max) { 757 //console.info('BotMon.live.data.analytics.getTopBots('+max+')'); 758 759 //console.log(BotMon.live.data.analytics.groups.knownBots); 760 761 let botsList = BotMon.live.data.analytics.groups.knownBots.toSorted( (a, b) => { 762 return b._viewCount - a._viewCount; 763 }); 764 765 const other = { 766 'id': 'other', 767 'name': "Others", 768 'count': 0 769 }; 770 771 const rList = []; 772 const max2 = ( botsList.length > max ? max-1 : botsList.length ); 773 let total = 0; // adding up the items 774 for (let i=0; i<botsList.length; i++) { 775 const it = botsList[i]; 776 if (it && it._bot) { 777 if (i < max2) { 778 rList.push({ 779 id: it._bot.id, 780 name: (it._bot.n ? it._bot.n : it._bot.id), 781 count: it._viewCount 782 }); 783 } else { 784 other.count += it._pageViews.length; 785 }; 786 total += it._viewCount; 787 } 788 }; 789 790 // add the "other" item, if needed: 791 if (botsList.length > max2) { 792 rList.push(other); 793 }; 794 795 rList.forEach( it => { 796 it.pct = (it.count * 100 / total); 797 }); 798 799 return rList; 800 }, 801 802 // most visited pages list: 803 _pagesList: [], 804 805 /** 806 * Add a page view to the list of most visited pages. 807 * @param {string} pageId - The page ID to add to the list. 808 * @example 809 * BotMon.live.data.analytics.addToPagesList('1234567890'); 810 */ 811 addToPagesList: function(pageId) { 812 //console.log('BotMon.live.data.analytics.addToPagesList', pageId); 813 814 const me = BotMon.live.data.analytics; 815 816 // already exists? 817 let pgObj = null; 818 for (let i = 0; i < me._pagesList.length; i++) { 819 if (me._pagesList[i].id == pageId) { 820 pgObj = me._pagesList[i]; 821 break; 822 } 823 } 824 825 // if not exists, create it: 826 if (!pgObj) { 827 pgObj = { 828 id: pageId, 829 count: 1 830 }; 831 me._pagesList.push(pgObj); 832 } else { 833 pgObj.count += 1; 834 } 835 }, 836 837 getTopPages: function(max) { 838 //console.info('BotMon.live.data.analytics.getTopPages('+max+')'); 839 const me = BotMon.live.data.analytics; 840 return me._pagesList.toSorted( (a, b) => { 841 return b.count - a.count; 842 }).slice(0,max); 843 }, 844 845 // Referer List: 846 _refererList: [], 847 848 addToRefererList: function(ref) { 849 //console.log('BotMon.live.data.analytics.addToRefererList',ref); 850 851 const me = BotMon.live.data.analytics; 852 853 // ignore internal references: 854 if (ref && ref.host == window.location.host) { 855 return; 856 } 857 858 const refInfo = me.getRefererInfo(ref); 859 860 // already exists? 861 let refObj = null; 862 for (let i = 0; i < me._refererList.length; i++) { 863 if (me._refererList[i].id == refInfo.id) { 864 refObj = me._refererList[i]; 865 break; 866 } 867 } 868 869 // if not exists, create it: 870 if (!refObj) { 871 refObj = refInfo; 872 refObj.count = 1; 873 me._refererList.push(refObj); 874 } else { 875 refObj.count += 1; 876 } 877 }, 878 879 getRefererInfo: function(url) { 880 //console.log('BotMon.live.data.analytics.getRefererInfo',url); 881 try { 882 url = new URL(url); 883 } catch (e) { 884 return { 885 'id': 'null', 886 'n': 'Invalid Referer' 887 }; 888 } 889 890 // find the referer ID: 891 let refId = 'null'; 892 let refName = 'No Referer'; 893 if (url && url.host) { 894 const hArr = url.host.split('.'); 895 const tld = hArr[hArr.length-1]; 896 refId = ( tld == 'localhost' ? tld : hArr[hArr.length-2]); 897 refName = hArr[hArr.length-2] + '.' + tld; 898 } 899 900 return { 901 'id': refId, 902 'n': refName 903 }; 904 }, 905 906 /** 907 * Get a sorted list of the top referers. 908 * The list is sorted in descending order of count. 909 * If the array has more items than the given maximum, the rest of the items are added to an "other" item. 910 * Each item in the list has a "pct" property, which is the percentage of the total count. 911 * @param {number} max - The maximum number of items to return. 912 * @return {Array} The sorted list of top referers. 913 */ 914 getTopReferers: function(max) { 915 //console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')')); 916 917 const me = BotMon.live.data.analytics; 918 919 return me._makeTopList(me._refererList, max); 920 }, 921 922 /** 923 * Create a sorted list of top items from a given array. 924 * The list is sorted in descending order of count. 925 * If the array has more items than the given maximum, the rest of the items are added to an "other" item. 926 * Each item in the list has a "pct" property, which is the percentage of the total count. 927 * @param {Array} arr - The array to sort and truncate. 928 * @param {number} max - The maximum number of items to return. 929 * @return {Array} The sorted list of top items. 930 */ 931 _makeTopList: function(arr, max) { 932 //console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')')); 933 934 const me = BotMon.live.data.analytics; 935 936 // sort the list: 937 arr.sort( (a,b) => { 938 return b.count - a.count; 939 }); 940 941 const rList = []; // return array 942 const max2 = ( arr.length >= max ? max-1 : arr.length ); 943 const other = { 944 'id': 'other', 945 'name': "Others", 946 'typ': 'other', 947 'count': 0 948 }; 949 let total = 0; // adding up the items 950 for (let i=0; Math.min(max, arr.length) > i; i++) { 951 const it = arr[i]; 952 if (it) { 953 if (i < max2) { 954 const rIt = { 955 id: it.id, 956 name: (it.n ? it.n : it.id), 957 typ: it.typ || it.id, 958 count: it.count 959 }; 960 rList.push(rIt); 961 } else { 962 other.count += it.count; 963 } 964 total += it.count; 965 } 966 } 967 968 // add the "other" item, if needed: 969 if (arr.length > max2) { 970 rList.push(other); 971 }; 972 973 rList.forEach( it => { 974 it.pct = (it.count * 100 / total); 975 }); 976 977 return rList; 978 }, 979 980 /* countries of visits */ 981 _countries: { 982 'human': [], 983 'bot': [] 984 }, 985 /** 986 * Adds a country code to the statistics. 987 * 988 * @param {string} iso The ISO 3166-1 alpha-2 country code. 989 */ 990 addToCountries: function(iso, name, type) { 991 992 const me = BotMon.live.data.analytics; 993 994 // find the correct array: 995 let arr = null; 996 switch (type) { 997 998 case BM_USERTYPE.KNOWN_USER: 999 case BM_USERTYPE.PROBABLY_HUMAN: 1000 arr = me._countries.human; 1001 break; 1002 case BM_USERTYPE.LIKELY_BOT: 1003 case BM_USERTYPE.KNOWN_BOT: 1004 arr = me._countries.bot; 1005 break; 1006 default: 1007 console.warn(`Unknown user type ${type} in function addToCountries.`); 1008 } 1009 1010 if (arr) { 1011 let cRec = arr.find( it => it.id == iso); 1012 if (!cRec) { 1013 cRec = { 1014 'id': iso, 1015 'n': name, 1016 'count': 1 1017 }; 1018 arr.push(cRec); 1019 } else { 1020 cRec.count += 1; 1021 } 1022 } 1023 }, 1024 1025 /** 1026 * Returns a list of countries with visit counts, sorted by visit count in descending order. 1027 * 1028 * @param {BM_USERTYPE} type array of types type of visitors to return. 1029 * @param {number} max The maximum number of entries to return. 1030 * @return {Array} A list of objects with properties 'iso' (ISO 3166-1 alpha-2 country code) and 'count' (visit count). 1031 */ 1032 getCountryList: function(type, max) { 1033 1034 const me = BotMon.live.data.analytics; 1035 1036 // find the correct array: 1037 let arr = null; 1038 switch (type) { 1039 1040 case 'human': 1041 arr = me._countries.human; 1042 break; 1043 case 'bot': 1044 arr = me._countries.bot; 1045 break; 1046 default: 1047 console.warn(`Unknown user type ${type} in function getCountryList.`); 1048 return; 1049 } 1050 1051 return me._makeTopList(arr, max); 1052 }, 1053 1054 /* browser and platform of human visitors */ 1055 _browsers: [], 1056 _platforms: [], 1057 1058 addBrowserPlatform: function(visitor) { 1059 //console.info('addBrowserPlatform', visitor); 1060 1061 const me = BotMon.live.data.analytics; 1062 1063 // add to browsers list: 1064 let browserRec = ( visitor._client ? visitor._client : {'id': 'unknown'}); 1065 if (visitor._client) { 1066 let bRec = me._browsers.find( it => it.id == browserRec.id); 1067 if (!bRec) { 1068 bRec = { 1069 id: browserRec.id, 1070 n: browserRec.n, 1071 count: 1 1072 }; 1073 me._browsers.push(bRec); 1074 } else { 1075 bRec.count += 1; 1076 } 1077 } 1078 1079 // add to platforms list: 1080 let platformRec = ( visitor._platform ? visitor._platform : {'id': 'unknown'}); 1081 if (visitor._platform) { 1082 let pRec = me._platforms.find( it => it.id == platformRec.id); 1083 if (!pRec) { 1084 pRec = { 1085 id: platformRec.id, 1086 n: platformRec.n, 1087 count: 1 1088 }; 1089 me._platforms.push(pRec); 1090 } else { 1091 pRec.count += 1; 1092 } 1093 } 1094 1095 }, 1096 1097 getTopBrowsers: function(max) { 1098 1099 const me = BotMon.live.data.analytics; 1100 1101 return me._makeTopList(me._browsers, max); 1102 }, 1103 1104 getTopPlatforms: function(max) { 1105 1106 const me = BotMon.live.data.analytics; 1107 1108 return me._makeTopList(me._platforms, max); 1109 }, 1110 1111 /* bounces are counted, not calculates: */ 1112 getBounceCount: function(type) { 1113 1114 const me = BotMon.live.data.analytics; 1115 var bounces = 0; 1116 const list = me.groups[type]; 1117 1118 list.forEach(it => { 1119 bounces += (it._viewCount <= 1 ? 1 : 0); 1120 }); 1121 1122 return bounces; 1123 }, 1124 1125 _ipRanges: [], 1126 1127 /* adds a visit to the ip ranges arrays */ 1128 addToIpRanges: function(v) { 1129 //console.info('addToIpRanges', v.ip); 1130 1131 const me = BotMon.live.data.analytics; 1132 const ipRanges = BotMon.live.data.ipRanges; 1133 1134 // Number of IP address segments to look at: 1135 const kIP4Segments = 1; 1136 const kIP6Segments = 3; 1137 1138 let ipGroup = ''; // group name 1139 let ipSeg = []; // IP segment array 1140 let rawIP = ''; // raw ip prefix 1141 let ipName = ''; // IP group display name 1142 1143 const ipType = v.ip.indexOf(':') > 0 ? BM_IPVERSION.IPv6 : BM_IPVERSION.IPv4; 1144 const ipAddr = BotMon.t._ip2Num(v.ip); 1145 1146 // is there already a known IP range assigned? 1147 if (v._ipRange) { 1148 1149 ipGroup = v._ipRange.g; // group name 1150 ipName = ipRanges.getOwner( v._ipRange.g ) || "Unknown"; 1151 1152 } else { // no known IP range, let's collect necessary information: 1153 1154 // collect basic IP address info: 1155 if (ipType == BM_IPVERSION.IPv6) { 1156 ipSeg = ipAddr.split(':'); 1157 const prefix = v.ip.split(':').slice(0, kIP6Segments).join(':'); 1158 rawIP = ipSeg.slice(0, kIP6Segments).join(':'); 1159 ipGroup = 'ip6-' + rawIP.replaceAll(':', '-'); 1160 ipName = prefix + '::'; // + '/' + (16 * kIP6Segments); 1161 } else { 1162 ipSeg = ipAddr.split('.'); 1163 const prefix = v.ip.split('.').slice(0, kIP4Segments).join('.'); 1164 rawIP = ipSeg.slice(0, kIP4Segments).join('.') ; 1165 ipGroup = 'ip4-' + rawIP.replaceAll('.', '-'); 1166 ipName = prefix + '.x.x.x'.substring(0, 1+(4-kIP4Segments)*2); // + '/' + (8 * kIP4Segments); 1167 } 1168 } 1169 1170 // check if record already exists: 1171 let ipRec = me._ipRanges.find( it => it.g == ipGroup); 1172 if (!ipRec) { 1173 1174 // ip info record initialised: 1175 ipRec = { 1176 g: ipGroup, 1177 n: ipName, 1178 count: 0 1179 } 1180 1181 // existing record? 1182 if (v._ipRange) { 1183 1184 ipRec.from = v._ipRange.from; 1185 ipRec.to = v._ipRange.to; 1186 ipRec.typ = 'net'; 1187 1188 } else { // no known IP range, let's collect necessary information: 1189 1190 // complete the ip info record: 1191 if (ipType == BM_IPVERSION.IPv6) { 1192 ipRec.from = rawIP + ':0000:0000:0000:0000:0000:0000:0000'.substring(0, (8-kIP6Segments)*5); 1193 ipRec.to = rawIP + ':FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF'.substring(0, (8-kIP6Segments)*5); 1194 ipRec.typ = '6'; 1195 } else { 1196 ipRec.from = rawIP + '.000.000.000.000'.substring(0, (4-kIP4Segments)*4); 1197 ipRec.to = rawIP + '.255.255.255.254'.substring(0, (4-kIP4Segments)*4); 1198 ipRec.typ = '4'; 1199 } 1200 } 1201 1202 me._ipRanges.push(ipRec); 1203 } 1204 1205 // add to counter: 1206 ipRec.count += v._viewCount; 1207 1208 }, 1209 1210 getTopBotISPs: function(max) { 1211 1212 const me = BotMon.live.data.analytics; 1213 1214 return me._makeTopList(me._ipRanges, max); 1215 }, 1216 }, 1217 1218 // information on "known bots": 1219 bots: { 1220 // loads the list of known bots from a JSON file: 1221 init: async function() { 1222 //console.info('BotMon.live.data.bots.init()'); 1223 1224 // Load the list of known bots: 1225 BotMon.live.gui.status.showBusy("Loading known bots …"); 1226 const url = BotMon._baseDir + 'config/known-bots.json'; 1227 try { 1228 const response = await fetch(url); 1229 if (!response.ok) { 1230 throw new Error(`${response.status} ${response.statusText}`); 1231 } 1232 1233 this._list = await response.json(); 1234 this._ready = true; 1235 1236 } catch (error) { 1237 BotMon.live.gui.status.setError("Error while loading the known bots file:", error.message); 1238 } finally { 1239 BotMon.live.gui.status.hideBusy("Status: Done."); 1240 BotMon.live.data._dispatch('bots') 1241 } 1242 }, 1243 1244 // returns bot info if the clientId matches a known bot, null otherwise: 1245 match: function(agent) { 1246 //console.info('BotMon.live.data.bots.match(',agent,')'); 1247 1248 const BotList = BotMon.live.data.bots._list; 1249 1250 // default is: not found! 1251 let botInfo = null; 1252 1253 if (!agent) return null; 1254 1255 // check for known bots: 1256 BotList.find(bot => { 1257 let r = false; 1258 for (let j=0; j<bot.rx.length; j++) { 1259 const rxr = agent.match(new RegExp(bot.rx[j])); 1260 if (rxr) { 1261 botInfo = { 1262 n : bot.n, 1263 id: bot.id, 1264 geo: (bot.geo ? bot.geo : null), 1265 url: bot.url, 1266 v: (rxr.length > 1 ? rxr[1] : -1) 1267 }; 1268 r = true; 1269 break; 1270 }; 1271 }; 1272 return r; 1273 }); 1274 1275 // check for unknown bots: 1276 if (!botInfo) { 1277 const botmatch = agent.match(/([\s\d\w\-]*bot|[\s\d\w\-]*crawler|[\s\d\w\-]*spider)[\/\s\w\-;\),\\.$]/i); 1278 if(botmatch) { 1279 botInfo = {'id': ( botmatch[1] || "other_" ), 'n': "Other" + ( botmatch[1] ? " (" + botmatch[1] + ")" : "" ) , "bot": botmatch[1] }; 1280 } 1281 } 1282 1283 //console.log("botInfo:", botInfo); 1284 return botInfo; 1285 }, 1286 1287 1288 // indicates if the list is loaded and ready to use: 1289 _ready: false, 1290 1291 // the actual bot list is stored here: 1292 _list: [] 1293 }, 1294 1295 // information on known clients (browsers): 1296 clients: { 1297 // loads the list of known clients from a JSON file: 1298 init: async function() { 1299 //console.info('BotMon.live.data.clients.init()'); 1300 1301 // Load the list of known bots: 1302 BotMon.live.gui.status.showBusy("Loading known clients"); 1303 const url = BotMon._baseDir + 'config/known-clients.json'; 1304 try { 1305 const response = await fetch(url); 1306 if (!response.ok) { 1307 throw new Error(`${response.status} ${response.statusText}`); 1308 } 1309 1310 BotMon.live.data.clients._list = await response.json(); 1311 BotMon.live.data.clients._ready = true; 1312 1313 } catch (error) { 1314 BotMon.live.gui.status.setError("Error while loading the known clients file: " + error.message); 1315 } finally { 1316 BotMon.live.gui.status.hideBusy("Status: Done."); 1317 BotMon.live.data._dispatch('clients') 1318 } 1319 }, 1320 1321 // returns bot info if the user-agent matches a known bot, null otherwise: 1322 match: function(agent) { 1323 //console.info('BotMon.live.data.clients.match(',agent,')'); 1324 1325 let match = {"n": "Unknown", "v": -1, "id": 'null'}; 1326 1327 if (agent) { 1328 BotMon.live.data.clients._list.find(client => { 1329 let r = false; 1330 for (let j=0; j<client.rx.length; j++) { 1331 const rxr = agent.match(new RegExp(client.rx[j])); 1332 if (rxr) { 1333 match.n = client.n; 1334 match.v = (rxr.length > 1 ? rxr[1] : -1); 1335 match.id = client.id || null; 1336 r = true; 1337 break; 1338 } 1339 } 1340 return r; 1341 }); 1342 } 1343 1344 //console.log(match) 1345 return match; 1346 }, 1347 1348 // return the browser name for a browser ID: 1349 getName: function(id) { 1350 const it = BotMon.live.data.clients._list.find(client => client.id == id); 1351 return ( it && it.n ? it.n : "Unknown"); //it.n; 1352 }, 1353 1354 // indicates if the list is loaded and ready to use: 1355 _ready: false, 1356 1357 // the actual bot list is stored here: 1358 _list: [] 1359 1360 }, 1361 1362 // information on known platforms (operating systems): 1363 platforms: { 1364 // loads the list of known platforms from a JSON file: 1365 init: async function() { 1366 //console.info('BotMon.live.data.platforms.init()'); 1367 1368 // Load the list of known bots: 1369 BotMon.live.gui.status.showBusy("Loading known platforms"); 1370 const url = BotMon._baseDir + 'config/known-platforms.json'; 1371 try { 1372 const response = await fetch(url); 1373 if (!response.ok) { 1374 throw new Error(`${response.status} ${response.statusText}`); 1375 } 1376 1377 BotMon.live.data.platforms._list = await response.json(); 1378 BotMon.live.data.platforms._ready = true; 1379 1380 } catch (error) { 1381 BotMon.live.gui.status.setError("Error while loading the known platforms file: " + error.message); 1382 } finally { 1383 BotMon.live.gui.status.hideBusy("Status: Done."); 1384 BotMon.live.data._dispatch('platforms') 1385 } 1386 }, 1387 1388 // returns bot info if the browser id matches a known platform: 1389 match: function(cid) { 1390 //console.info('BotMon.live.data.platforms.match(',cid,')'); 1391 1392 let match = {"n": "Unknown", "id": 'null'}; 1393 1394 if (cid) { 1395 BotMon.live.data.platforms._list.find(platform => { 1396 let r = false; 1397 for (let j=0; j<platform.rx.length; j++) { 1398 const rxr = cid.match(new RegExp(platform.rx[j])); 1399 if (rxr) { 1400 match.n = platform.n; 1401 match.v = (rxr.length > 1 ? rxr[1] : -1); 1402 match.id = platform.id || null; 1403 r = true; 1404 break; 1405 } 1406 } 1407 return r; 1408 }); 1409 } 1410 1411 return match; 1412 }, 1413 1414 // return the platform name for a given ID: 1415 getName: function(id) { 1416 const it = BotMon.live.data.platforms._list.find( pf => pf.id == id); 1417 return ( it ? it.n : 'Unknown' ); 1418 }, 1419 1420 1421 // indicates if the list is loaded and ready to use: 1422 _ready: false, 1423 1424 // the actual bot list is stored here: 1425 _list: [] 1426 1427 }, 1428 1429 // storage and functions for the known bot IP-Ranges: 1430 ipRanges: { 1431 1432 init: function() { 1433 //console.log('BotMon.live.data.ipRanges.init()'); 1434 // #TODO: Load from separate IP-Ranges file 1435 // load the rules file: 1436 const me = BotMon.live.data; 1437 1438 try { 1439 BotMon.live.data._loadSettingsFile(['user-ipranges', 'known-ipranges'], 1440 (json) => { 1441 1442 // groups can be just saved in the data structure: 1443 if (json.groups && json.groups.constructor.name == 'Array') { 1444 me.ipRanges._groups = json.groups; 1445 } 1446 1447 // groups can be just saved in the data structure: 1448 if (json.ranges && json.ranges.constructor.name == 'Array') { 1449 json.ranges.forEach(range => { 1450 me.ipRanges.add(range); 1451 }) 1452 } 1453 1454 // finished loading 1455 BotMon.live.gui.status.hideBusy("Status: Done."); 1456 BotMon.live.data._dispatch('ipranges') 1457 }); 1458 } catch (error) { 1459 BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); 1460 } 1461 }, 1462 1463 // the actual bot list is stored here: 1464 _list: [], 1465 _groups: [], 1466 1467 add: function(data) { 1468 //console.log('BotMon.live.data.ipRanges.add(',data,')'); 1469 1470 const me = BotMon.live.data.ipRanges; 1471 1472 // convert IP address to easier comparable form: 1473 const ip2Num = BotMon.t._ip2Num; 1474 1475 let item = { 1476 'cidr': data.from.replaceAll(/::+/g, '::') + '/' + ( data.m ? data.m : '??' ), 1477 'from': ip2Num(data.from), 1478 'to': ip2Num(data.to), 1479 'm': data.m, 1480 'g': data.g 1481 }; 1482 me._list.push(item); 1483 1484 }, 1485 1486 getOwner: function(gid) { 1487 1488 const me = BotMon.live.data.ipRanges; 1489 1490 for (let i=0; i < me._groups.length; i++) { 1491 const it = me._groups[i]; 1492 if (it.id == gid) { 1493 return it.name; 1494 } 1495 } 1496 return null; 1497 }, 1498 1499 match: function(ip) { 1500 //console.log('BotMon.live.data.ipRanges.match(',ip,')'); 1501 1502 const me = BotMon.live.data.ipRanges; 1503 1504 // convert IP address to easier comparable form: 1505 const ipNum = BotMon.t._ip2Num(ip); 1506 1507 for (let i=0; i < me._list.length; i++) { 1508 const ipRange = me._list[i]; 1509 1510 if (ipNum >= ipRange.from && ipNum <= ipRange.to) { 1511 return ipRange; 1512 } 1513 1514 }; 1515 return null; 1516 } 1517 }, 1518 1519 // storage for the rules and related functions 1520 rules: { 1521 1522 /** 1523 * Initializes the rules data. 1524 * 1525 * Loads the default config file and the user config file (if present). 1526 * The default config file is used if the user config file does not have a certain setting. 1527 * The user config file can override settings from the default config file. 1528 * 1529 * The rules are loaded from the `rules` property of the config files. 1530 * The IP ranges are loaded from the `ipRanges` property of the config files. 1531 * 1532 * If an error occurs while loading the config file, it is displayed in the status bar. 1533 * After the config file is loaded, the status bar is hidden. 1534 */ 1535 init: async function() { 1536 //console.info('BotMon.live.data.rules.init()'); 1537 1538 // Load the list of known bots: 1539 BotMon.live.gui.status.showBusy("Loading list of rules …"); 1540 1541 // load the rules file: 1542 const me = BotMon.live.data; 1543 1544 try { 1545 BotMon.live.data._loadSettingsFile(['user-config', 'default-config'], 1546 (json) => { 1547 1548 // override the threshold? 1549 if (json.threshold) me._threshold = json.threshold; 1550 1551 // set the rules list: 1552 if (json.rules && json.rules.constructor.name == 'Array') { 1553 me.rules._rulesList = json.rules; 1554 } 1555 1556 BotMon.live.gui.status.hideBusy("Status: Done."); 1557 BotMon.live.data._dispatch('rules') 1558 } 1559 ); 1560 } catch (error) { 1561 BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); 1562 } 1563 }, 1564 1565 _rulesList: [], // list of rules to find out if a visitor is a bot 1566 _threshold: 100, // above this, it is considered a bot. 1567 1568 // returns a descriptive text for a rule id 1569 getRuleInfo: function(ruleId) { 1570 // console.info('getRuleInfo', ruleId); 1571 1572 // shortcut for neater code: 1573 const me = BotMon.live.data.rules; 1574 1575 for (let i=0; i<me._rulesList.length; i++) { 1576 const rule = me._rulesList[i]; 1577 if (rule.id == ruleId) { 1578 return rule; 1579 } 1580 } 1581 return null; 1582 1583 }, 1584 1585 // evaluate a visitor for lkikelihood of being a bot 1586 evaluate: function(visitor) { 1587 1588 // shortcut for neater code: 1589 const me = BotMon.live.data.rules; 1590 1591 let r = { // evaluation result 1592 'val': 0, 1593 'rules': [], 1594 'isBot': false 1595 }; 1596 1597 for (let i=0; i<me._rulesList.length; i++) { 1598 const rule = me._rulesList[i]; 1599 const params = ( rule.params ? rule.params : [] ); 1600 1601 if (rule.func) { // rule is calling a function 1602 if (me.func[rule.func]) { 1603 if(me.func[rule.func](visitor, ...params)) { 1604 r.val += rule.bot; 1605 r.rules.push(rule.id) 1606 } 1607 } else { 1608 //console.warn("Unknown rule function: “${rule.func}”. Ignoring rule.") 1609 } 1610 } 1611 } 1612 1613 // is a bot? 1614 r.isBot = (r.val >= me._threshold); 1615 1616 return r; 1617 }, 1618 1619 // list of functions that can be called by the rules list to evaluate a visitor: 1620 func: { 1621 1622 // check if client is on the list passed as parameter: 1623 matchesClient: function(visitor, ...clients) { 1624 1625 const clientId = ( visitor._client ? visitor._client.id : ''); 1626 return clients.includes(clientId); 1627 }, 1628 1629 // check if OS/Platform is one of the obsolete ones: 1630 matchesPlatform: function(visitor, ...platforms) { 1631 1632 const pId = ( visitor._platform ? visitor._platform.id : ''); 1633 1634 if (visitor._platform.id == null) console.log(visitor._platform); 1635 1636 return platforms.includes(pId); 1637 }, 1638 1639 // are there at lest num pages loaded? 1640 smallPageCount: function(visitor, num) { 1641 return (visitor._viewCount <= Number(num)); 1642 }, 1643 1644 // There was no entry in a specific log file for this visitor: 1645 // note that this will also trigger the "noJavaScript" rule: 1646 noRecord: function(visitor, type) { 1647 if (!visitor._seenBy.includes('srv')) return false; // only if 'srv' is also specified! 1648 return !visitor._seenBy.includes(type); 1649 }, 1650 1651 // there are no referrers in any of the page visits: 1652 noReferrer: function(visitor) { 1653 1654 let r = false; // return value 1655 for (let i = 0; i < visitor._pageViews.length; i++) { 1656 if (!visitor._pageViews[i]._ref) { 1657 r = true; 1658 break; 1659 } 1660 } 1661 return r; 1662 }, 1663 1664 // test for specific client identifiers: 1665 matchesUserAgent: function(visitor, ...list) { 1666 1667 for (let i=0; i<list.length; i++) { 1668 if (visitor.agent == list[i]) { 1669 return true 1670 } 1671 }; 1672 return false; 1673 }, 1674 1675 // unusual combinations of Platform and Client: 1676 combinationTest: function(visitor, ...combinations) { 1677 1678 for (let i=0; i<combinations.length; i++) { 1679 1680 if (visitor._platform.id == combinations[i][0] 1681 && visitor._client.id == combinations[i][1]) { 1682 return true 1683 } 1684 }; 1685 1686 return false; 1687 }, 1688 1689 // is the IP address from a known bot network? 1690 fromKnownBotIP: function(visitor) { 1691 //console.info('fromKnownBotIP()', visitor.ip); 1692 1693 return visitor.hasOwnProperty('_ipRange'); 1694 }, 1695 1696 // is the page language mentioned in the client's accepted languages? 1697 // the parameter holds an array of exceptions, i.e. page languages that should be ignored. 1698 matchLang: function(visitor, ...exceptions) { 1699 1700 if (visitor.lang && visitor.accept && exceptions.indexOf(visitor.lang) < 0) { 1701 return (visitor.accept.split(',').indexOf(visitor.lang) < 0); 1702 } 1703 return false; 1704 }, 1705 1706 // the "Accept language" header contains certain entries: 1707 clientAccepts: function(visitor, ...languages) { 1708 //console.info('clientAccepts', visitor.accept, languages); 1709 1710 if (visitor.accept && languages) {; 1711 return ( visitor.accept.split(',').filter(lang => languages.includes(lang)).length > 0 ); 1712 } 1713 return false; 1714 }, 1715 1716 // Is there an accept-language field defined at all? 1717 noAcceptLang: function(visitor) { 1718 1719 if (!visitor.accept || visitor.accept.length <= 0) { // no accept-languages header 1720 return true; 1721 } 1722 // TODO: parametrize this! 1723 return false; 1724 }, 1725 // At least x page views were recorded, but they come within less than y seconds 1726 loadSpeed: function(visitor, minItems, maxTime) { 1727 1728 if (visitor._viewCount >= minItems) { 1729 //console.log('loadSpeed', visitor._pageViews.length, minItems, maxTime); 1730 1731 const pvArr = visitor._pageViews.map(pv => pv._lastSeen).sort(); 1732 1733 let totalTime = 0; 1734 for (let i=1; i < pvArr.length; i++) { 1735 totalTime += (pvArr[i] - pvArr[i-1]); 1736 } 1737 1738 //console.log(' ', totalTime , Math.round(totalTime / (pvArr.length * 1000)), (( totalTime / pvArr.length ) <= maxTime * 1000), visitor.ip); 1739 1740 return (( totalTime / pvArr.length ) <= maxTime * 1000); 1741 } 1742 }, 1743 1744 // Country code matches one of those in the list: 1745 matchesCountry: function(visitor, ...countries) { 1746 1747 // ingore if geoloc is not set or unknown: 1748 if (visitor.geo) { 1749 return (countries.indexOf(visitor.geo) >= 0); 1750 } 1751 return false; 1752 }, 1753 1754 // Country does not match one of the given codes. 1755 notFromCountry: function(visitor, ...countries) { 1756 1757 // ingore if geoloc is not set or unknown: 1758 if (visitor.geo && visitor.geo !== 'ZZ') { 1759 return (countries.indexOf(visitor.geo) < 0); 1760 } 1761 return false; 1762 }, 1763 1764 // Check if visitor never went beyond a captcha 1765 blockedByCaptcha: function(visitor) { 1766 return (visitor._captcha.Y > 0 && visitor._captcha.N === 0); 1767 }, 1768 1769 // Check if visitor came from a whitelisted IP-address 1770 whitelistedByCaptcha: function(visitor) { 1771 return (visitor._captcha.W > 0); 1772 } 1773 } 1774 }, 1775 1776 /** 1777 * Loads a settings file from the specified list of filenames. 1778 * If the file is successfully loaded, it will call the callback function 1779 * with the loaded JSON data. 1780 * If no file can be loaded, it will display an error message. 1781 * 1782 * @param {string[]} fns - list of filenames to load 1783 * @param {function} callback - function to call with the loaded JSON data 1784 */ 1785 _loadSettingsFile: async function(fns, callback) { 1786 //console.info('BotMon.live.data._loadSettingsFile()', fns); 1787 1788 const kJsonExt = '.json'; 1789 let loaded = false; // if successfully loaded file 1790 1791 for (let i=0; i<fns.length; i++) { 1792 const filename = fns[i] +kJsonExt; 1793 try { 1794 const response = await fetch(DOKU_BASE + 'lib/plugins/botmon/config/' + filename); 1795 if (!response.ok) { 1796 continue; 1797 } else { 1798 loaded = true; 1799 } 1800 const json = await response.json(); 1801 if (callback && typeof callback === 'function') { 1802 callback(json); 1803 } 1804 break; 1805 } catch (e) { 1806 BotMon.live.gui.status.setError("Error while loading the config file: " + filename); 1807 } 1808 } 1809 1810 if (!loaded) { 1811 BotMon.live.gui.status.setError("Could not load a config file."); 1812 } 1813 }, 1814 1815 /** 1816 * Loads a log file (server, page load, or ticker) and parses it. 1817 * @param {String} type - the type of the log file to load (srv, log, or tck) 1818 * @param {Function} [onLoaded] - an optional callback function to call after loading is finished. 1819 */ 1820 loadLogFile: async function(type, onLoaded = undefined) { 1821 //console.info('BotMon.live.data.loadLogFile(',type,')'); 1822 1823 let typeName = ''; 1824 let columns = []; 1825 1826 switch (type) { 1827 case "srv": 1828 typeName = "Server"; 1829 columns = ['ts','ip','pg','id','typ','usr','agent','ref','lang','accept','geo','captcha']; 1830 break; 1831 case "log": 1832 typeName = "Page load"; 1833 columns = ['ts','ip','pg','id','usr','lt','ref','agent']; 1834 break; 1835 case "tck": 1836 typeName = "Ticker"; 1837 columns = ['ts','ip','pg','id','agent']; 1838 break; 1839 default: 1840 console.warn(`Unknown log type ${type}.`); 1841 return; 1842 } 1843 1844 // Show the busy indicator and set the visible status: 1845 BotMon.live.gui.status.showBusy(`Loading ${typeName} log file …`); 1846 1847 // compose the URL from which to load: 1848 const url = BotMon._baseDir + `logs/${BotMon._datestr}.${type}.txt`; 1849 //console.log("Loading:",url); 1850 1851 // fetch the data: 1852 try { 1853 const response = await fetch(url); 1854 if (!response.ok) { 1855 1856 throw new Error(`${response.status} ${response.statusText}`); 1857 1858 } else { 1859 1860 // parse the data: 1861 const logtxt = await response.text(); 1862 if (logtxt.length <= 0) { 1863 throw new Error(`Empty log file ${url}.`); 1864 } 1865 1866 logtxt.split('\n').forEach((line) => { 1867 if (line.trim() === '') return; // skip empty lines 1868 const cols = line.split('\t'); 1869 1870 // assign the columns to an object: 1871 const data = {}; 1872 cols.forEach( (colVal,i) => { 1873 const colName = columns[i] || `col${i}`; 1874 const colValue = (colName == 'ts' ? new Date(colVal) : colVal.trim()); 1875 data[colName] = colValue; 1876 }); 1877 1878 // register the visit in the model: 1879 switch(type) { 1880 case BM_LOGTYPE.SERVER: 1881 BotMon.live.data.model.registerVisit(data, type); 1882 break; 1883 case BM_LOGTYPE.CLIENT: 1884 data.typ = 'js'; 1885 BotMon.live.data.model.updateVisit(data); 1886 break; 1887 case BM_LOGTYPE.TICKER: 1888 data.typ = 'js'; 1889 BotMon.live.data.model.updateTicks(data); 1890 break; 1891 default: 1892 console.warn(`Unknown log type ${type}.`); 1893 return; 1894 } 1895 }); 1896 } 1897 1898 } catch (error) { 1899 BotMon.live.gui.status.setError(`Error while loading the ${typeName} log file: ${error.message} – data may be incomplete.`); 1900 } finally { 1901 BotMon.live.gui.status.hideBusy("Status: Done."); 1902 if (onLoaded) { 1903 onLoaded(); // callback after loading is finished. 1904 } 1905 } 1906 } 1907 }, 1908 1909 gui: { 1910 init: function() { 1911 //console.log('BotMon.live.gui.init()'); 1912 1913 // init sub-objects: 1914 BotMon.t._callInit(this); 1915 }, 1916 1917 tabs: { 1918 init: function() { 1919 //console.log('BotMon.live.gui.tabs.init()'); 1920 1921 /* find and add all existing tabs */ 1922 document.querySelectorAll('#botmon__admin *[role=tablist]') 1923 .forEach((tablist) => { 1924 tablist.querySelectorAll('*[role=tab]') 1925 .forEach( t => t.addEventListener('click', this._onTabClick) ) 1926 }); 1927 }, 1928 1929 /* callback for tab click */ 1930 _onTabClick: function(e) { 1931 //console.log('BotMon.live.gui.tabs._onTabClick()'); 1932 1933 /* reusable constants: */ 1934 const kAriaSelected = 'aria-selected'; 1935 const kAriaControls = 'aria-controls'; 1936 const kTrue = 'true'; 1937 const kFalse = 'false'; 1938 const kHidden = 'hidden'; 1939 1940 /* cancel default action */ 1941 e.preventDefault(); 1942 1943 /* if the active tab is clicked, do nothing: */ 1944 let selState = this.getAttribute(kAriaSelected); 1945 if ( selState && selState == kTrue ) { 1946 return; 1947 } 1948 1949 /* find the active tab element: */ 1950 var aItem = null; 1951 let tablist = this.parentNode; 1952 while (tablist.getAttribute('role') !== 'tablist') { 1953 tablist = tablist.parentNode; 1954 } 1955 1956 if (tablist.getAttribute('role') == 'tablist') { 1957 let lis = tablist.querySelectorAll('*[role=tab]'); 1958 lis.forEach( (it) => { 1959 let selected = it.getAttribute(kAriaSelected); 1960 if ( selected && selected == kTrue ) { 1961 aItem = it; 1962 } 1963 }); 1964 } 1965 1966 /* swap the active states: */ 1967 this.setAttribute(kAriaSelected, kTrue); 1968 if (aItem) { 1969 aItem.setAttribute(kAriaSelected, kFalse); 1970 let aId = aItem.getAttribute(kAriaControls); 1971 let aObj = document.getElementById(aId); 1972 if (aObj) aObj.hidden = true; 1973 } 1974 1975 /* show the new panel: */ 1976 let nId = this.getAttribute(kAriaControls); 1977 let nObj = document.getElementById(nId); 1978 if (nObj) nObj.hidden = false; 1979 } 1980 }, 1981 1982 /* The Overview / web metrics section of the live tab */ 1983 overview: { 1984 /** 1985 * Populates the overview part of the today tab with the analytics data. 1986 * 1987 * @method make 1988 * @memberof BotMon.live.gui.overview 1989 */ 1990 make: function() { 1991 1992 const maxItemsPerList = 5; // how many list items to show? 1993 const useCaptcha = BMSettings.useCaptcha || false; 1994 1995 const kNoData = '–'; // shown when data is missing 1996 const kSeparator = ' / '; 1997 1998 // shortcuts for neater code: 1999 const makeElement = BotMon.t._makeElement; 2000 const data = BotMon.live.data.analytics.data; 2001 2002 const botsVsHumans = document.getElementById('botmon__today__botsvshumans'); 2003 if (botsVsHumans) { 2004 botsVsHumans.appendChild(makeElement('dt', {}, "Bot statistics")); 2005 2006 for (let i = 0; i <= ( useCaptcha ? 5 : 3 ); i++) { 2007 const dd = makeElement('dd'); 2008 let title = ''; 2009 let value = ''; 2010 switch(i) { 2011 case 0: 2012 title = "Total (loads / views / visits):"; 2013 value = (data.loads.total || kNoData) + kSeparator + (data.views.total || kNoData) + kSeparator + (data.visits.total || kNoData); 2014 break; 2015 case 1: 2016 title = "Known bots (views / visits):"; 2017 value = (data.views.bots || kNoData) + kSeparator + (data.visits.bots || kNoData); 2018 break; 2019 case 2: 2020 title = "Suspected bots (views / visits):"; 2021 value = (data.visits.suspected || kNoData) + kSeparator + (data.views.suspected || kNoData) 2022 break; 2023 case 3: 2024 title = "Bots-humans ratio (views / visits):"; 2025 value = BotMon.t._getRatio(data.views.suspected + data.views.bots, data.views.users + data.views.humans, 100) + kSeparator + BotMon.t._getRatio(data.visits.suspected + data.visits.bots, data.visits.users + data.visits.humans, 100); 2026 break; 2027 case 4: 2028 title = "Known bots blocked / passed / whitelisted:"; 2029 value = data.captcha.bots_blocked + kSeparator + data.captcha.bots_passed + kSeparator + data.captcha.bots_whitelisted; 2030 break; 2031 case 5: 2032 title = "Suspected bots blocked / passed / whitelisted:"; 2033 value = data.captcha.sus_blocked + kSeparator + data.captcha.sus_passed + kSeparator + data.captcha.sus_whitelisted; 2034 break; 2035 default: 2036 console.warn(`Unknown list type ${i}.`); 2037 } 2038 dd.appendChild(makeElement('span', {}, title)); 2039 dd.appendChild(makeElement('strong', {}, value)); 2040 botsVsHumans.appendChild(dd); 2041 } 2042 } 2043 2044 // update known bots list: 2045 const botElement = document.getElementById('botmon__botslist'); /* Known bots */ 2046 if (botElement) { 2047 botElement.appendChild(makeElement('dt', {}, `Top known bots`)); 2048 2049 let botList = BotMon.live.data.analytics.getTopBots(maxItemsPerList); 2050 botList.forEach( (botInfo) => { 2051 const bli = makeElement('dd'); 2052 bli.appendChild(makeElement('span', {'class': 'has_icon bot bot_' + botInfo.id }, botInfo.name)); 2053 bli.appendChild(makeElement('span', {'class': 'count' }, botInfo.count)); 2054 botElement.append(bli) 2055 }); 2056 } 2057 2058 // update the suspected bot IP ranges list: 2059 const botIps = document.getElementById('botmon__botips'); 2060 if (botIps) { 2061 botIps.appendChild(makeElement('dt', {}, "Top bot Networks")); 2062 2063 const ispList = BotMon.live.data.analytics.getTopBotISPs(5); 2064 //console.log(ispList); 2065 ispList.forEach( (netInfo) => { 2066 const li = makeElement('dd'); 2067 li.appendChild(makeElement('span', {'class': 'has_icon ipaddr ip' + netInfo.typ }, netInfo.name)); 2068 li.appendChild(makeElement('span', {'class': 'count' }, netInfo.count)); 2069 botIps.append(li) 2070 }); 2071 } 2072 2073 // update the top bot countries list: 2074 const botCountries = document.getElementById('botmon__botcountries'); 2075 if (botCountries) { 2076 botCountries.appendChild(makeElement('dt', {}, `Top bot Countries`)); 2077 const countryList = BotMon.live.data.analytics.getCountryList('bot', 5); 2078 countryList.forEach( (cInfo) => { 2079 const cLi = makeElement('dd'); 2080 cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name)); 2081 cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count)); 2082 botCountries.appendChild(cLi); 2083 }); 2084 } 2085 2086 // update the webmetrics overview: 2087 const wmoverview = document.getElementById('botmon__today__wm_overview'); 2088 if (wmoverview) { 2089 2090 const humanVisits = data.views.total; 2091 const bounceRate = Math.round(100 * (BotMon.live.data.analytics.getBounceCount('users') + BotMon.live.data.analytics.getBounceCount('humans')) / humanVisits); 2092 2093 wmoverview.appendChild(makeElement('dt', {}, "Humans’ metrics")); 2094 for (let i = 0; i <= 4; i++) { 2095 const dd = makeElement('dd'); 2096 let title = ''; 2097 let value = ''; 2098 switch(i) { 2099 case 0: 2100 title = "Registered users (views / visits):"; 2101 value = (data.views.users || kNoData) + kSeparator + (data.visits.users || kNoData); 2102 break; 2103 case 1: 2104 title = "Probably humans (views / visits):"; 2105 value = (data.views.humans || kNoData) + kSeparator + (data.visits.humans || kNoData); 2106 break; 2107 case 2: 2108 title = "Total human page views:"; 2109 value = (data.views.users + data.views.humans) || kNoData; 2110 break; 2111 case 3: 2112 title = "Total human visits:"; 2113 value = data.views.total || kNoData; 2114 break; 2115 case 4: 2116 title = "Humans’ bounce rate:"; 2117 value = bounceRate + '%'; 2118 break; 2119 default: 2120 console.warn(`Unknown list type ${i}.`); 2121 } 2122 dd.appendChild(makeElement('span', {}, title)); 2123 dd.appendChild(makeElement('strong', {}, value)); 2124 wmoverview.appendChild(dd); 2125 } 2126 } 2127 2128 // update the webmetrics clients list: 2129 const wmclients = document.getElementById('botmon__today__wm_clients'); 2130 if (wmclients) { 2131 2132 wmclients.appendChild(makeElement('dt', {}, "Browsers")); 2133 2134 const clientList = BotMon.live.data.analytics.getTopBrowsers(maxItemsPerList); 2135 if (clientList) { 2136 clientList.forEach( (cInfo) => { 2137 const cDd = makeElement('dd'); 2138 cDd.appendChild(makeElement('span', {'class': 'has_icon client cl_' + cInfo.id }, ( cInfo.name ? cInfo.name : cInfo.id))); 2139 cDd.appendChild(makeElement('span', { 2140 'class': 'count', 2141 'title': cInfo.count + " page views" 2142 }, cInfo.pct.toFixed(1) + '%')); 2143 wmclients.appendChild(cDd); 2144 }); 2145 } 2146 } 2147 2148 // update the webmetrics platforms list: 2149 const wmplatforms = document.getElementById('botmon__today__wm_platforms'); 2150 if (wmplatforms) { 2151 2152 wmplatforms.appendChild(makeElement('dt', {}, "Platforms")); 2153 2154 const pfList = BotMon.live.data.analytics.getTopPlatforms(maxItemsPerList); 2155 if (pfList) { 2156 pfList.forEach( (pInfo) => { 2157 const pDd = makeElement('dd'); 2158 pDd.appendChild(makeElement('span', {'class': 'has_icon platform pf_' + pInfo.id }, ( pInfo.name ? pInfo.name : pInfo.id))); 2159 pDd.appendChild(makeElement('span', { 2160 'class': 'count', 2161 'title': pInfo.count + " page views" 2162 }, pInfo.pct.toFixed(1) + '%')); 2163 wmplatforms.appendChild(pDd); 2164 }); 2165 } 2166 } 2167 2168 // update the top bot countries list: 2169 const usrCountries = document.getElementById('botmon__today__wm_countries'); 2170 if (usrCountries) { 2171 usrCountries.appendChild(makeElement('dt', {}, `Top visitor Countries:`)); 2172 const usrCtryList = BotMon.live.data.analytics.getCountryList('human', 5); 2173 usrCtryList.forEach( (cInfo) => { 2174 const cLi = makeElement('dd'); 2175 cLi.appendChild(makeElement('span', {'class': 'has_icon country ctry_' + cInfo.id.toLowerCase() }, cInfo.name)); 2176 cLi.appendChild(makeElement('span', {'class': 'count' }, cInfo.count)); 2177 usrCountries.appendChild(cLi); 2178 }); 2179 } 2180 2181 // update the top pages; 2182 const wmpages = document.getElementById('botmon__today__wm_pages'); 2183 if (wmpages) { 2184 2185 wmpages.appendChild(makeElement('dt', {}, "Top pages")); 2186 2187 const pgList = BotMon.live.data.analytics.getTopPages(maxItemsPerList); 2188 if (pgList) { 2189 pgList.forEach( (pgInfo) => { 2190 const pgDd = makeElement('dd'); 2191 pgDd.appendChild(makeElement('a', { 2192 'class': 'page_icon', 2193 'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(pgInfo.id), 2194 'target': 'preview', 2195 'title': "PageID: " + pgInfo.id 2196 }, pgInfo.id)); 2197 pgDd.appendChild(makeElement('span', { 2198 'class': 'count', 2199 'title': pgInfo.count + " page views" 2200 }, pgInfo.count)); 2201 wmpages.appendChild(pgDd); 2202 }); 2203 } 2204 } 2205 2206 // update the top referrers; 2207 const wmreferers = document.getElementById('botmon__today__wm_referers'); 2208 if (wmreferers) { 2209 2210 wmreferers.appendChild(makeElement('dt', {}, "Referers")); 2211 2212 const refList = BotMon.live.data.analytics.getTopReferers(maxItemsPerList); 2213 if (refList) { 2214 refList.forEach( (rInfo) => { 2215 const rDd = makeElement('dd'); 2216 rDd.appendChild(makeElement('span', {'class': 'has_icon referer ref_' + rInfo.id }, rInfo.name)); 2217 rDd.appendChild(makeElement('span', { 2218 'class': 'count', 2219 'title': rInfo.count + " references" 2220 }, rInfo.pct.toFixed(1) + '%')); 2221 wmreferers.appendChild(rDd); 2222 }); 2223 } 2224 } 2225 } 2226 }, 2227 2228 status: { 2229 setText: function(txt) { 2230 const el = document.getElementById('botmon__today__status'); 2231 if (el && BotMon.live.gui.status._errorCount <= 0) { 2232 el.innerText = txt; 2233 } 2234 }, 2235 2236 setTitle: function(html) { 2237 const el = document.getElementById('botmon__today__title'); 2238 if (el) { 2239 el.innerHTML = html; 2240 } 2241 }, 2242 2243 setError: function(txt) { 2244 console.error(txt); 2245 BotMon.live.gui.status._errorCount += 1; 2246 const el = document.getElementById('botmon__today__status'); 2247 if (el) { 2248 el.innerText = "Data may be incomplete."; 2249 el.classList.add('error'); 2250 } 2251 }, 2252 _errorCount: 0, 2253 2254 showBusy: function(txt = null) { 2255 BotMon.live.gui.status._busyCount += 1; 2256 const el = document.getElementById('botmon__today__busy'); 2257 if (el) { 2258 el.style.display = 'inline-block'; 2259 } 2260 if (txt) BotMon.live.gui.status.setText(txt); 2261 }, 2262 _busyCount: 0, 2263 2264 hideBusy: function(txt = null) { 2265 const el = document.getElementById('botmon__today__busy'); 2266 BotMon.live.gui.status._busyCount -= 1; 2267 if (BotMon.live.gui.status._busyCount <= 0) { 2268 if (el) el.style.display = 'none'; 2269 if (txt) BotMon.live.gui.status.setText(txt); 2270 } 2271 } 2272 }, 2273 2274 lists: { 2275 init: function() { 2276 2277 // function shortcut: 2278 const makeElement = BotMon.t._makeElement; 2279 2280 const parent = document.getElementById('botmon__today__visitorlists'); 2281 if (parent) { 2282 2283 for (let i=0; i < 4; i++) { 2284 2285 // change the id and title by number: 2286 let listTitle = ''; 2287 let listId = ''; 2288 let infolink = null; 2289 switch (i) { 2290 case 0: 2291 listTitle = "Registered users"; 2292 listId = 'users'; 2293 break; 2294 case 1: 2295 listTitle = "Probably humans"; 2296 listId = 'humans'; 2297 break; 2298 case 2: 2299 listTitle = "Suspected bots"; 2300 listId = 'suspectedBots'; 2301 infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/suspected_bots'; 2302 break; 2303 case 3: 2304 listTitle = "Known bots"; 2305 listId = 'knownBots'; 2306 infolink = 'https://leib.be/sascha/projects/dokuwiki/botmon/info/known_bots'; 2307 break; 2308 default: 2309 console.warn('Unknown list number.'); 2310 } 2311 2312 const details = makeElement('details', { 2313 'data-group': listId, 2314 'data-loaded': false 2315 }); 2316 const title = details.appendChild(makeElement('summary')); 2317 title.appendChild(makeElement('span', {'class': 'title'}, listTitle)); 2318 if (infolink) { 2319 title.appendChild(makeElement('a', { 2320 'class': 'ext_info', 2321 'target': '_blank', 2322 'href': infolink, 2323 'title': "More information" 2324 }, "Info")); 2325 } 2326 details.addEventListener("toggle", this._onDetailsToggle); 2327 2328 parent.appendChild(details); 2329 2330 } 2331 } 2332 }, 2333 2334 _onDetailsToggle: function(e) { 2335 //console.info('BotMon.live.gui.lists._onDetailsToggle()'); 2336 2337 const target = e.target; 2338 2339 if (target.getAttribute('data-loaded') == 'false') { // only if not loaded yet 2340 target.setAttribute('data-loaded', 'loading'); 2341 2342 const fillType = target.getAttribute('data-group'); 2343 const fillList = BotMon.live.data.analytics.groups[fillType]; 2344 if (fillList && fillList.length > 0) { 2345 2346 const ul = BotMon.t._makeElement('ul'); 2347 2348 fillList.forEach( (it) => { 2349 ul.appendChild(BotMon.live.gui.lists._makeVisitorItem(it, fillType)); 2350 }); 2351 2352 target.appendChild(ul); 2353 target.setAttribute('data-loaded', 'true'); 2354 } else { 2355 target.setAttribute('data-loaded', 'false'); 2356 } 2357 2358 } 2359 }, 2360 2361 _makeVisitorItem: function(data, type) { 2362 2363 // shortcuts for neater code: 2364 const make = BotMon.t._makeElement; 2365 const model = BotMon.live.data.model; 2366 2367 let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' ); 2368 if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0'; 2369 2370 const platformName = (data._platform ? data._platform.n : 'Unknown'); 2371 const clientName = (data._client ? data._client.n: 'Unknown'); 2372 2373 const sumClass = ( !data._seenBy || data._seenBy.indexOf(BM_LOGTYPE.SERVER) < 0 ? 'noServer' : 'hasServer'); 2374 2375 // combine with other networks? 2376 const combineNets = (BMSettings.hasOwnProperty('combineNets') ? BMSettings['combineNets'] : true) 2377 && data.hasOwnProperty('_ipRange'); 2378 2379 const li = make('li'); // root list item 2380 const details = make('details'); 2381 const summary = make('summary', { 2382 'class': sumClass 2383 }); 2384 details.appendChild(summary); 2385 2386 const span1 = make('span'); /* left-hand group */ 2387 2388 /*if (data._type !== BM_USERTYPE.KNOWN_BOT) { // No platform/client for bots // disabled because no longer relevant 2389 span1.appendChild(make('span', { // Platform 2390 'class': 'icon_only platform pf_' + (data._platform ? data._platform.id : 'unknown'), 2391 'title': "Platform: " + platformName 2392 }, platformName)); 2393 2394 span1.appendChild(make('span', { // Client 2395 'class': 'icon_only client client cl_' + (data._client ? data._client.id : 'unknown'), 2396 'title': "Client: " + clientName 2397 }, clientName)); 2398 }*/ 2399 2400 // identifier: 2401 if (data._type == BM_USERTYPE.KNOWN_BOT) { /* Bot only */ 2402 2403 const botName = ( data._bot && data._bot.n ? data._bot.n : "Unknown"); 2404 span1.appendChild(make('span', { /* Bot */ 2405 'class': 'has_icon bot bot_' + (data._bot ? data._bot.id : 'unknown'), 2406 'title': "Bot: " + botName 2407 }, botName)); 2408 2409 } else if (data._type == BM_USERTYPE.KNOWN_USER) { /* User only */ 2410 2411 span1.appendChild(make('span', { /* User */ 2412 'class': 'has_icon user_known', 2413 'title': "User: " + data.usr 2414 }, data.usr)); 2415 2416 } else { /* others */ 2417 2418 if (combineNets) { 2419 2420 const ispName = BotMon.live.data.ipRanges.getOwner( data._ipRange.g ) || data._ipRange.g; 2421 2422 span1.appendChild(make('span', { // IP-Address 2423 'class': 'has_icon ipaddr ipnet', 2424 'title': "IP-Range: " + data._ipRange.g 2425 }, ispName)); 2426 2427 } else { 2428 2429 span1.appendChild(make('span', { // IP-Address 2430 'class': 'has_icon ipaddr ip' + ipType, 2431 'title': "IP-Address: " + data.ip 2432 }, data.ip)); 2433 } 2434 } 2435 2436 span1.appendChild(make('span', { /* page views */ 2437 'class': 'has_icon pageseen', 2438 'title': data._pageViews.length + " page load(s)" 2439 }, data._pageViews.length)); 2440 2441 2442 // referer icons: 2443 if ((data._type == BM_USERTYPE.PROBABLY_HUMAN || data._type == BM_USERTYPE.LIKELY_BOT) && data.ref) { 2444 const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref); 2445 span1.appendChild(make('span', { 2446 'class': 'icon_only referer ref_' + refInfo.id, 2447 'title': "Referer: " + data.ref 2448 }, refInfo.n)); 2449 } 2450 2451 summary.appendChild(span1); 2452 const span2 = make('span'); /* right-hand group */ 2453 2454 // country flag: 2455 if (!combineNets) { // not for combined networks 2456 if (data.geo && data.geo !== 'ZZ') { 2457 span2.appendChild(make('span', { 2458 'class': 'icon_only country ctry_' + data.geo.toLowerCase(), 2459 'data-ctry': data.geo, 2460 'title': "Country: " + ( data._country || "Unknown") 2461 }, ( data._country || "Unknown") )); 2462 } 2463 } 2464 2465 span2.appendChild(make('span', { // seen-by icon: 2466 'class': 'icon_only seenby sb_' + data._seenBy.join(''), 2467 'title': "Seen by: " + data._seenBy.join('+') 2468 }, data._seenBy.join(', '))); 2469 2470 // captcha status: 2471 const cCode = ( data._captcha ? data._captcha._str() : ''); 2472 if (cCode !== '') { 2473 const cTitle = model._makeCaptchaTitle(data._captcha) 2474 span2.appendChild(make('span', { // captcha status 2475 'class': 'icon_only captcha cap_' + cCode, 2476 'title': "Captcha-status: " + cTitle 2477 }, cTitle)); 2478 } 2479 2480 summary.appendChild(span2); 2481 2482 // add details expandable section: 2483 details.appendChild(BotMon.live.gui.lists._makeVisitorDetails(data, type)); 2484 2485 li.appendChild(details); 2486 return li; 2487 }, 2488 2489 _makeVisitorDetails: function(data, type) { 2490 2491 // shortcuts for neater code: 2492 const make = BotMon.t._makeElement; 2493 const model = BotMon.live.data.model; 2494 2495 let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' ); 2496 if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0'; 2497 const platformName = (data._platform ? data._platform.n : 'Unknown'); 2498 const clientName = (data._client ? data._client.n: 'Unknown'); 2499 2500 const dl = make('dl', {'class': 'visitor_details'}); 2501 2502 if (data._type == BM_USERTYPE.KNOWN_BOT) { 2503 2504 dl.appendChild(make('dt', {}, "Bot name:")); /* bot info */ 2505 dl.appendChild(make('dd', {'class': 'icon_only bot bot_' + (data._bot ? data._bot.id : 'unknown')}, 2506 (data._bot ? data._bot.n : 'Unknown'))); 2507 2508 if (data._bot && data._bot.url) { 2509 dl.appendChild(make('dt', {}, "Bot info:")); /* bot info */ 2510 const botInfoDd = dl.appendChild(make('dd')); 2511 botInfoDd.appendChild(make('a', { 2512 'href': data._bot.url, 2513 'target': '_blank' 2514 }, data._bot.url)); /* bot info link*/ 2515 2516 } 2517 2518 } else { /* not for bots */ 2519 2520 dl.appendChild(make('dt', {}, "Client:")); /* client */ 2521 dl.appendChild(make('dd', {'class': 'has_icon client cl_' + (data._client ? data._client.id : 'unknown')}, 2522 clientName + ( data._client.v > 0 ? ' (' + data._client.v + ')' : '' ) )); 2523 2524 dl.appendChild(make('dt', {}, "Platform:")); /* platform */ 2525 dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')}, 2526 platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); 2527 2528 /*dl.appendChild(make('dt', {}, "ID:")); 2529 dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/ 2530 } 2531 2532 dl.appendChild(make('dt', {}, "IP-Address:")); 2533 const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType}); 2534 ipItem.appendChild(make('span', {'class': 'address'} , data.ip)); 2535 ipItem.appendChild(make('a', { 2536 'class': 'icon_only extlink ipinfo', 2537 'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`, 2538 'target': 'ipinfo', 2539 'title': "View this address on IPInfo.io" 2540 } , "DNS Info")); 2541 ipItem.appendChild(make('a', { 2542 'class': 'icon_only extlink abuseipdb', 2543 'href': `https://www.abuseipdb.com/check/${encodeURIComponent(data.ip)}`, 2544 'target': 'abuseipdb', 2545 'title': "Check this address on AbuseIPDB.com" 2546 } , "Check on AbuseIPDB")); 2547 dl.appendChild(ipItem); 2548 2549 if (Math.abs(data._lastSeen - data._firstSeen) < 100) { 2550 dl.appendChild(make('dt', {}, "Seen:")); 2551 dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString())); 2552 } else { 2553 dl.appendChild(make('dt', {}, "First seen:")); 2554 dl.appendChild(make('dd', {'class': 'firstSeen'}, data._firstSeen.toLocaleString())); 2555 dl.appendChild(make('dt', {}, "Last seen:")); 2556 dl.appendChild(make('dd', {'class': 'lastSeen'}, data._lastSeen.toLocaleString())); 2557 } 2558 2559 dl.appendChild(make('dt', {}, "Actions:")); 2560 dl.appendChild(make('dd', {'class': 'views'}, 2561 "Page loads: " + data._loadCount.toString() + 2562 ( data._captcha['Y'] > 0 || data._captcha['W'] || data._captcha['-'] > 0 ? ", captchas: " + (data._captcha['Y']+data._captcha['W']+data._captcha['-']).toString() : '') + 2563 ", views: " + data._viewCount.toString() 2564 )); 2565 2566 dl.appendChild(make('dt', {}, "User-Agent:")); 2567 dl.appendChild(make('dd', {'class': 'agent'}, data.agent)); 2568 2569 if (data.ref && data.ref !== '') { 2570 dl.appendChild(make('dt', {}, "Referrer:")); 2571 2572 const refInfo = BotMon.live.data.analytics.getRefererInfo(data.ref); 2573 const refDd = dl.appendChild(make('dd', { 2574 'class': 'has_icon referer ref_' + refInfo.id 2575 })); 2576 refDd.appendChild(make('a', { 2577 'href': data.ref, 2578 'target': 'refTarget' 2579 }, data.ref)); 2580 } 2581 2582 dl.appendChild(make('dt', {}, "Languages:")); 2583 dl.appendChild(make('dd', {'class': 'langs'}, ` [${data.accept}]`)); 2584 2585 if (data.geo && data.geo !=='') { 2586 dl.appendChild(make('dt', {}, "Location:")); 2587 dl.appendChild(make('dd', { 2588 'class': 'has_icon country ctry_' + data.geo.toLowerCase(), 2589 'data-ctry': data.geo, 2590 'title': "Country: " + data._country 2591 }, data._country + ' (' + data.geo + ')')); 2592 } 2593 2594 if (data.captcha && data.captcha !=='') { 2595 dl.appendChild(make('dt', {}, "Captcha-status:")); 2596 dl.appendChild(make('dd', { 2597 'class': 'captcha' 2598 }, model._makeCaptchaTitle(data._captcha))); 2599 } 2600 2601 dl.appendChild(make('dt', {}, "Session ID:")); 2602 dl.appendChild(make('dd', {'class': 'has_icon session typ_' + data.typ}, data.id)); 2603 2604 dl.appendChild(make('dt', {}, "Seen by:")); 2605 dl.appendChild(make('dd', {'class': 'has_icon seenby sb_' + data._seenBy.join('')}, data._seenBy.join(', ') )); 2606 2607 dl.appendChild(make('dt', {}, "Visited pages:")); 2608 const pagesDd = make('dd', {'class': 'pages'}); 2609 const pageList = make('ul'); 2610 2611 /* list all page views */ 2612 data._pageViews.sort( (a, b) => a._firstSeen - b._firstSeen ); 2613 data._pageViews.forEach( (page) => { 2614 pageList.appendChild(BotMon.live.gui.lists._makePageViewItem(page)); 2615 }); 2616 pagesDd.appendChild(pageList); 2617 dl.appendChild(pagesDd); 2618 2619 /* bot evaluation rating */ 2620 if (data._type !== BM_USERTYPE.KNOWN_BOT && data._type !== BM_USERTYPE.KNOWN_USER) { 2621 dl.appendChild(make('dt', undefined, "Bot rating:")); 2622 dl.appendChild(make('dd', {'class': 'bot-rating'}, ( data._botVal ? data._botVal : '–' ) + ' (of ' + BotMon.live.data.rules._threshold + ')')); 2623 2624 /* add bot evaluation details: */ 2625 if (data._eval) { 2626 dl.appendChild(make('dt', {}, "Bot evaluation:")); 2627 const evalDd = make('dd', {'class': 'eval'}); 2628 const testList = make('ul'); 2629 data._eval.forEach( test => { 2630 2631 const tObj = BotMon.live.data.rules.getRuleInfo(test); 2632 let tDesc = tObj ? tObj.desc : test; 2633 2634 // special case for Bot IP range test: 2635 if (tObj.func == 'fromKnownBotIP') { 2636 const rangeInfo = BotMon.live.data.ipRanges.match(data.ip); 2637 if (rangeInfo) { 2638 const owner = BotMon.live.data.ipRanges.getOwner(rangeInfo.g); 2639 tDesc += ' (range: “' + rangeInfo.cidr + '”, ' + owner + ')'; 2640 } 2641 } 2642 2643 // create the entry field 2644 const tstLi = make('li'); 2645 tstLi.appendChild(make('span', { 2646 'data-testid': test 2647 }, tDesc)); 2648 tstLi.appendChild(make('span', {}, ( tObj ? tObj.bot : '—') )); 2649 testList.appendChild(tstLi); 2650 }); 2651 2652 // add total row 2653 const tst2Li = make('li', { 2654 'class': 'total' 2655 }); 2656 /*tst2Li.appendChild(make('span', {}, "Total:")); 2657 tst2Li.appendChild(make('span', {}, data._botVal)); 2658 testList.appendChild(tst2Li);*/ 2659 2660 evalDd.appendChild(testList); 2661 dl.appendChild(evalDd); 2662 } 2663 } 2664 2665 // for debugging only. Disable on production: 2666 dl.appendChild(make('dt', {}, "Debug info:")); 2667 const dbgDd = make('dd', {'class': 'debug'}); 2668 dbgDd.innerHTML = '<pre>' + JSON.stringify(data, null, 4) + '</pre>'; 2669 dl.appendChild(dbgDd); 2670 2671 // return the element to add to the UI: 2672 return dl; 2673 }, 2674 2675 // make a page view item: 2676 _makePageViewItem: function(page) { 2677 //console.log("makePageViewItem:",page); 2678 2679 // shortcut for neater code: 2680 const make = BotMon.t._makeElement; 2681 2682 // the actual list item: 2683 const pgLi = make('li'); 2684 2685 const row1 = make('div', {'class': 'row'}); 2686 2687 row1.appendChild(make('a', { // page id is the left group 2688 'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(page.pg), 2689 'target': 'preview', 2690 'hreflang': page.lang, 2691 'title': "PageID: " + page.pg 2692 }, page.pg)); /* DW Page ID */ 2693 2694 const rightGroup = row1.appendChild(make('div')); // right-hand group 2695 2696 // get the time difference: 2697 rightGroup.appendChild(make('span', { 2698 'class': 'first-seen', 2699 'title': "First visited: " + page._firstSeen.toLocaleString() + " UTC" 2700 }, BotMon.t._formatTime(page._firstSeen))); 2701 2702 rightGroup.appendChild(make('span', { /* page loads */ 2703 'class': 'has_icon pageviews', 2704 'title': page._viewCount.toString() + " page load(s)" 2705 }, page._viewCount.toString())); 2706 2707 pgLi.appendChild(row1); 2708 2709 /* LINE 2 */ 2710 2711 const row2 = make('div', {'class': 'row'}); 2712 2713 // page referrer: 2714 if (page._ref) { 2715 row2.appendChild(make('span', { 2716 'class': 'referer', 2717 'title': "Referrer: " + page._ref.href 2718 }, page._ref.hostname)); 2719 } else { 2720 row2.appendChild(make('span', { 2721 'class': 'referer' 2722 }, "No referer")); 2723 } 2724 2725 // visit duration: 2726 let visitTimeStr = "Bounce"; 2727 const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime(); 2728 if (visitDuration > 0) { 2729 visitTimeStr = Math.floor(visitDuration / 1000) + "s"; 2730 } 2731 const tDiff = BotMon.t._formatTimeDiff(page._firstSeen, page._lastSeen); 2732 if (tDiff) { 2733 row2.appendChild(make('span', {'class': 'visit-length', 'title': 'Last seen: ' + page._lastSeen.toLocaleString()}, tDiff)); 2734 } else { 2735 row2.appendChild(make('span', { 2736 'class': 'bounce', 2737 'title': "Visitor bounced"}, "Bounce")); 2738 } 2739 2740 pgLi.appendChild(row2); 2741 2742 return pgLi; 2743 } 2744 } 2745 } 2746}; 2747 2748/* launch only if the BotMon admin panel is open: */ 2749if (document.getElementById('botmon__admin')) { 2750 BotMon.init(); 2751}